Skip to main content

orch/engine/
tick.rs

1//! Core tick loop phases.
2//!
3//! The engine ticks every ~10 seconds. Each tick runs six sequential phases:
4//! 1. Poll tmux for finished sessions
5//! 2. Recover stuck in-progress tasks
6//! 3. Route and dispatch:
7//!    - 3a. Route new tasks to agents
8//!    - 3b. Dispatch routed tasks (spawn agents)
9//! 4. Unblock parents whose children are all done
10//! 5. Run cron job scheduler
11
12use crate::backends::{ExternalBackend, ExternalId, ExternalTask, Status};
13use crate::channels::capture::CaptureService;
14use crate::config;
15use crate::engine::cooldown::{
16    github_circuit_remaining_secs, is_agent_in_cooldown, is_github_circuit_open,
17    is_model_in_cooldown, record_agent_failure_with_message, record_silence_detection,
18    set_agent_cooldown, set_model_cooldown, SILENCE_AGENT_COOLDOWN_SECS,
19    SILENCE_EXTENDED_COOLDOWN_SECS,
20};
21use crate::engine::dispatch_guard::DispatchGuard;
22use crate::engine::jobs;
23use crate::engine::router::{get_route_result, AllAgentsCooledError, RouteResultError, Router};
24use crate::engine::runner::{TaskRunner, WeightSignal};
25use crate::engine::tasks::{is_internal_id, TaskManager};
26use crate::repo_context::REPO_CONTEXT;
27use crate::store::TaskStore;
28use crate::tmux::TmuxManager;
29use dashmap::{DashMap, DashSet};
30use std::sync::atomic::{AtomicBool, Ordering};
31use std::sync::Arc;
32use std::sync::LazyLock;
33use std::time::{Instant, SystemTime};
34use tokio::sync::{mpsc, Semaphore};
35
36/// Unix timestamp when the engine process started.
37/// Used to identify tmux sessions from previous runs.
38static ENGINE_START_TIME: LazyLock<u64> = LazyLock::new(|| {
39    SystemTime::now()
40        .duration_since(SystemTime::UNIX_EPOCH)
41        .unwrap_or_default()
42        .as_secs()
43});
44
45/// Flag to ensure startup cleanup only runs once.
46static STARTUP_CLEANUP_DONE: AtomicBool = AtomicBool::new(false);
47
48/// Per outer engine-loop quota for expensive routing calls.
49///
50/// `tick()` runs once per active project. Without a quota shared by all those
51/// project ticks, `router.max_tasks_per_tick = 1` still permits one slow LLM
52/// routing call per project plus one global sweep route per project, multiplying
53/// 45s router timeouts into multi-minute watchdog stalls.
54pub(crate) struct RoutingTickQuota {
55    remaining: usize,
56}
57
58impl RoutingTickQuota {
59    pub(crate) fn new(max: usize) -> Self {
60        Self { remaining: max }
61    }
62
63    fn is_exhausted(&self) -> bool {
64        self.remaining == 0
65    }
66
67    fn remaining(&self) -> usize {
68        self.remaining
69    }
70
71    fn consume_one(&mut self) -> bool {
72        if self.remaining == 0 {
73            return false;
74        }
75        self.remaining -= 1;
76        true
77    }
78}
79
80/// Cleanup stale tmux sessions from previous engine runs.
81/// Called once at startup to kill sessions created before this process started.
82async fn startup_cleanup(tmux: &TmuxManager) {
83    // Use compare_exchange to ensure this only runs once across all threads
84    if STARTUP_CLEANUP_DONE
85        .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
86        .is_err()
87    {
88        return;
89    }
90
91    let start_time = *ENGINE_START_TIME;
92    tracing::info!(
93        start_time,
94        "running startup cleanup for stale tmux sessions"
95    );
96
97    match tmux.kill_stale_sessions(start_time).await {
98        Ok(killed) => {
99            if killed > 0 {
100                tracing::info!(killed, "startup cleanup killed stale sessions");
101            } else {
102                tracing::debug!("no stale tmux sessions found on startup");
103            }
104        }
105        Err(e) => {
106            tracing::warn!(error = %e, "failed to run startup cleanup");
107        }
108    }
109}
110
111use super::EngineConfig;
112use crate::store::{set_review_session_expected, store_set_by_id, store_touch_updated_at};
113
114async fn recover_routed_blocked_dispatch(
115    backend: &Arc<dyn ExternalBackend>,
116    store: &Arc<TaskStore>,
117    task_manager: &Arc<TaskManager>,
118    repo: &str,
119    task: &ExternalTask,
120    timing: &StuckTaskTiming,
121) {
122    tracing::warn!(
123        task_id = task.id.0,
124        age_mins = timing.age.num_minutes(),
125        threshold_mins = timing.threshold / 60,
126        "recovering routed task: dispatch blocked by stale tmux session → new"
127    );
128
129    let resolved_store_id = match store.resolve_task_id(repo, &task.id.0).await {
130        Ok(Some(id)) => Some(id),
131        Ok(None) => {
132            tracing::warn!(
133                task_id = %task.id.0,
134                repo,
135                "resolve_task_id returned None during routed-task recovery"
136            );
137            None
138        }
139        Err(e) => {
140            tracing::error!(
141                task_id = %task.id.0,
142                repo,
143                error = %e,
144                "resolve_task_id failed during routed-task recovery"
145            );
146            None
147        }
148    };
149
150    if let Some(store_id) = resolved_store_id {
151        if let Err(e) = store_set_by_id(
152            &Some(store),
153            store_id,
154            &[
155                ("agent", serde_json::Value::Null),
156                ("model", serde_json::Value::Null),
157                ("route_attempts", serde_json::json!(0)),
158            ],
159        )
160        .await
161        {
162            tracing::warn!(
163                task_id = task.id.0,
164                ?e,
165                "failed to clear routing fields for routed stuck task — will retry next tick"
166            );
167            return;
168        }
169        finalize_open_runs_for_recovery(
170            store,
171            store_id,
172            "stuck-task recovery: stale session blocking dispatch killed",
173        )
174        .await;
175    }
176
177    if let Err(e) = task_manager.update_task_status(&task.id, Status::New).await {
178        tracing::warn!(
179            task_id = task.id.0,
180            ?e,
181            "failed to reset routed stuck task status"
182        );
183        return;
184    }
185
186    crate::store::store_log_activity(
187        &Some(Arc::clone(store)),
188        repo,
189        &task.id.0,
190        "rerouted",
191        Some("routed"),
192        Some("new"),
193        None::<&str>,
194        None::<&str>,
195        Some(&serde_json::json!({
196            "failure_reason": "routed_blocked_by_existing_session",
197            "age_minutes": timing.age.num_minutes(),
198            "had_session": true,
199        })),
200    )
201    .await;
202
203    if !is_internal_id(&task.id.0) {
204        let backend_clone = Arc::clone(backend);
205        let task_id_clone = task.id.clone();
206        let body = format!(
207            "[{}] recovered: stuck routed task — existing tmux session blocked dispatch for {}m; cleared route for retry{}",
208            chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
209            timing.age.num_minutes(),
210            crate::engine::orch_footer()
211        );
212        tokio::spawn(async move {
213            if let Err(e) = backend_clone.post_comment(&task_id_clone, &body).await {
214                tracing::warn!(
215                    task_id = task_id_clone.0,
216                    ?e,
217                    "failed to post routed-task recovery comment"
218                );
219            }
220        });
221    }
222}
223
224/// Phase 1 of tick: poll tmux for finished sessions and clean them up.
225pub(crate) async fn tick_check_session_completions(
226    tmux: &Arc<TmuxManager>,
227    repo: &str,
228    capture: &Arc<CaptureService>,
229    store: &Arc<TaskStore>,
230) -> anyhow::Result<()> {
231    let _span = tracing::info_span!("engine.tick.phase1.sessions").entered();
232    let session_snapshot = tmux.snapshot().await;
233    // Derive the short project name from repo (owner/repo -> repo), matching
234    // TmuxManager::session_name's derivation exactly (including the ".git" strip)
235    // so a repo configured with a ".git" suffix doesn't silently fail to match
236    // its own live sessions here — leaving stuck-task recovery as the only
237    // (later, coarser-grained) safety net for this repo's session completions.
238    let repo_name = repo
239        .rsplit('/')
240        .next()
241        .unwrap_or(repo)
242        .trim_end_matches(".git");
243    for (session, active) in session_snapshot {
244        // Only handle sessions that belong to this repo/project.
245        if session.project != repo_name {
246            continue;
247        }
248
249        if !active {
250            tracing::info!(
251                session = %session.name,
252                task_id = %session.task_id,
253                "session completed, collecting results"
254            );
255            // Touch the store.updated_at immediately when we observe a completed
256            // session so the stuck-task recovery phase doesn't incorrectly
257            // reclaim this task while the runner is still finishing
258            // post-processing (race: session observed dead → tick reclaim).
259            // Best-effort: no-ops if the task isn't present in the store.
260            //
261            // session.task_id uses hyphens (e.g. "internal-63714") because the
262            // tmux session name sanitizes colons to hyphens. resolve_task_id only
263            // handles "internal:" prefix, so we must convert back before touching.
264            // Review sessions carry a "-review" suffix tracked under the main task.
265            let store_task_id = {
266                let base = session
267                    .task_id
268                    .strip_suffix("-review")
269                    .unwrap_or(&session.task_id);
270                if let Some(n) = base.strip_prefix("internal-") {
271                    format!("internal:{n}")
272                } else {
273                    base.to_string()
274                }
275            };
276            store_touch_updated_at(&Some(Arc::clone(store)), repo, &store_task_id).await;
277            // Unregister from capture service using the task id the capture service
278            // was registered under (task id without project prefix).
279            capture.unregister_session(repo, &session.task_id).await;
280            // Kill the actual tmux session name we discovered (do not reconstruct)
281            if let Err(e) = tmux.kill_session(&session.name).await {
282                tracing::debug!(
283                    session = %session.name,
284                    ?e,
285                    "kill_session failed (session may already be gone)"
286                );
287            }
288        }
289    }
290    Ok(())
291}
292
293#[derive(Debug)]
294struct StuckTaskTiming {
295    has_session: bool,
296    age: chrono::Duration,
297    threshold: u64,
298}
299
300#[allow(clippy::too_many_arguments)]
301fn stuck_task_timing_from_map(
302    tmux: &Arc<TmuxManager>,
303    repo: &str,
304    session_task_id: &str,
305    updated_at: &str,
306    config: &EngineConfig,
307    parse_error_message: &'static str,
308    session_map: &std::collections::HashMap<String, bool>,
309) -> Option<StuckTaskTiming> {
310    let session_name = tmux.session_name(repo, session_task_id);
311    // A session is "running" if it appears in the map with alive=true.
312    let has_session = session_map.get(&session_name).copied().unwrap_or(false);
313    let threshold = if has_session {
314        config.stuck_timeout
315    } else {
316        config.no_session_stuck_timeout
317    };
318
319    let updated = match chrono::DateTime::parse_from_rfc3339(updated_at) {
320        Ok(dt) => dt.with_timezone(&chrono::Utc),
321        Err(e) => {
322            tracing::warn!(
323                task_id = session_task_id,
324                updated_at,
325                ?e,
326                error_message = parse_error_message,
327                "cannot parse updated_at"
328            );
329            return None;
330        }
331    };
332
333    let raw_age = chrono::Utc::now() - updated;
334    // Host suspend (laptop sleep) freezes the process just like it freezes the agent
335    // running inside it — that frozen time is not evidence of a hang. Discount any
336    // detected suspend/resume gaps that occurred since this task was last touched.
337    let suspended = super::suspend::suspended_duration_since(updated);
338    let age = (raw_age - suspended).max(chrono::Duration::zero());
339    if age.num_seconds() <= threshold as i64 {
340        return None;
341    }
342
343    Some(StuckTaskTiming {
344        has_session,
345        age,
346        threshold,
347    })
348}
349
350/// Finalize any open `task_runs` rows for a task during recovery paths.
351/// Best-effort: logs a warning on failure but does not block recovery.
352async fn finalize_open_runs_for_recovery(store: &TaskStore, store_id: i64, reason: &str) {
353    if let Err(e) = store
354        .finalize_incomplete_runs(store_id, "aborted", reason)
355        .await
356    {
357        tracing::warn!(
358            store_id,
359            error = %e,
360            reason,
361            "failed to finalize incomplete runs during recovery"
362        );
363    }
364}
365
366/// Phase 1b of tick: detect agents that have produced no output since session start
367/// and have exceeded the silence grace period.
368///
369/// Only triggers on complete silence (agent never produced any output). Agents that
370/// produced output then went quiet (e.g. long tool calls) are NOT affected.
371///
372/// On silence: kills the tmux session, applies model-level cooldown, resets the task
373/// to `New` so the router picks a different agent/model.
374pub(crate) async fn tick_detect_silent_agents(
375    tmux: &Arc<TmuxManager>,
376    repo: &str,
377    capture: &Arc<CaptureService>,
378    backend: &Arc<dyn ExternalBackend>,
379    task_manager: &Arc<TaskManager>,
380    config: &EngineConfig,
381    store: &Arc<TaskStore>,
382) -> anyhow::Result<()> {
383    let _span = tracing::info_span!("engine.tick.phase1b.silence").entered();
384    let grace = std::time::Duration::from_secs(config.silence_grace_period);
385    let silent_sessions = capture.get_silent_sessions_for_repo(repo, grace).await;
386
387    for (task_id, session_name, silence_age_secs) in silent_sessions {
388        let use_backend = should_use_backend(&task_id);
389        // Look up agent + model from the store so we can cooldown the right model.
390        let store_task = match store.resolve_task_id(repo, &task_id).await {
391            Ok(Some(store_id)) => match store.get(store_id).await {
392                Ok(task) => Some(task),
393                Err(e) => {
394                    tracing::warn!(
395                        task_id,
396                        repo,
397                        store_id,
398                        operation = "store.get",
399                        error = %e,
400                        "silence detection: db error fetching task — agent/model cooldown will be skipped"
401                    );
402                    None
403                }
404            },
405            Ok(None) => {
406                tracing::debug!(
407                    task_id,
408                    repo,
409                    operation = "resolve_task_id",
410                    "silence detection: task not found in store (may have been cleaned up)"
411                );
412                None
413            }
414            Err(e) => {
415                tracing::warn!(
416                    task_id,
417                    repo,
418                    operation = "resolve_task_id",
419                    error = %e,
420                    "silence detection: db error resolving task id — agent/model cooldown will be skipped"
421                );
422                None
423            }
424        };
425        let agent_name = store_task
426            .as_ref()
427            .and_then(|t| t.agent.clone())
428            .unwrap_or_default();
429        let model_name = store_task
430            .as_ref()
431            .and_then(|t| t.model.clone())
432            .unwrap_or_default();
433
434        tracing::warn!(
435            task_id,
436            agent = %agent_name,
437            model = %model_name,
438            silence_age_secs,
439            grace_secs = config.silence_grace_period,
440            cooldown_secs = config.silence_cooldown,
441            "agent silent since session start — killing session, cooling down model + agent, failing over"
442        );
443
444        // 1. Kill the tmux session
445        if let Err(e) = tmux.kill_session(&session_name).await {
446            tracing::debug!(
447                task_id,
448                ?e,
449                "kill_session failed for silent agent (may already be gone)"
450            );
451        }
452
453        // 2. Unregister from capture
454        capture.unregister_session(repo, &task_id).await;
455
456        // Finalize any open audit run so the task_runs row is not orphaned
457        if let Some(store_id) = store_task.as_ref().map(|t| t.id) {
458            finalize_open_runs_for_recovery(store, store_id, "silence detection: session killed")
459                .await;
460        }
461
462        let mut extended_note = String::new();
463
464        // 3. Cooldown the specific model (not the whole agent)
465        if !agent_name.is_empty() && !model_name.is_empty() {
466            set_model_cooldown(
467                &agent_name,
468                &model_name,
469                config.silence_cooldown,
470                "silence_detected",
471            )
472            .await;
473            if let Some(result) = record_silence_detection(&agent_name, &model_name).await {
474                if result.extended_cooldown_applied {
475                    extended_note = format!(
476                        " ({} silences in 24h -> extended cooldown {}s)",
477                        result.count, SILENCE_EXTENDED_COOLDOWN_SECS
478                    );
479                }
480            }
481        }
482
483        // 3b. Short agent-level cooldown to force router to pick a different agent.
484        // Without this, the router picks the same agent with a different model,
485        // looping through all models (~2 min each) before the long agent cooldown kicks in.
486        if !agent_name.is_empty() {
487            set_agent_cooldown(&agent_name, SILENCE_AGENT_COOLDOWN_SECS).await;
488        }
489
490        // 4. Pick a fallback agent and set to Routed (not New) to preserve progress.
491        // Setting to New would clear routing state and trigger a full LLM re-routing
492        // cycle, losing intermediate context. Routed skips re-routing and re-dispatches
493        // directly with the chosen fallback agent.
494        let task_eid = ExternalId(task_id.clone());
495
496        // Build available agents list and reroute chain for failover
497        let available: Vec<String> = crate::engine::configured_agents()
498            .into_iter()
499            .filter(|a| crate::cmd_cache::command_exists(a))
500            .collect();
501        let chain = crate::engine::runner::response::get_reroute_chain(
502            &task_id,
503            &Some(Arc::clone(store)),
504            repo,
505        )
506        .await;
507        let chain = crate::engine::runner::response::update_reroute_chain(
508            &task_id,
509            &agent_name,
510            &chain,
511            &Some(Arc::clone(store)),
512            repo,
513        )
514        .await;
515
516        let (next_status, next_agent) = if let Some(fallback) =
517            crate::engine::runner::response::pick_fallback_agent(&agent_name, &chain, &available)
518        {
519            tracing::info!(
520                task_id,
521                from = %agent_name,
522                to = %fallback,
523                "silence detection: failover to different agent, setting routed"
524            );
525            (Status::Routed, Some(fallback))
526        } else {
527            tracing::warn!(
528                task_id,
529                "silence detection: no fallback agents available, resetting to new for re-routing"
530            );
531            (Status::New, None)
532        };
533
534        if use_backend {
535            if let Some(ref st) = store_task {
536                // Fire-and-forget: label removals are cosmetic — tick should not block on them.
537                let stale_labels: Vec<String> = st
538                    .labels
539                    .iter()
540                    .filter(|l| {
541                        l.starts_with("agent:")
542                            || l.starts_with("complexity:")
543                            || l.starts_with("model:")
544                    })
545                    .cloned()
546                    .collect();
547                if !stale_labels.is_empty() {
548                    let backend_clone = Arc::clone(backend);
549                    let task_eid_clone = task_eid.clone();
550                    tokio::spawn(async move {
551                        for label in &stale_labels {
552                            if let Err(e) = backend_clone.remove_label(&task_eid_clone, label).await
553                            {
554                                tracing::warn!(task_id = task_eid_clone.0, label, error = %e, "failed to remove label during silence detection re-route");
555                            }
556                        }
557                    });
558                }
559            }
560        }
561
562        if let Some(store_id) = store_task.as_ref().map(|t| t.id) {
563            if let Some(ref fallback) = next_agent {
564                // Write the fallback agent into the store so dispatch can proceed
565                // directly without an LLM re-routing cycle. Clearing model forces
566                // model_for_complexity to pick the best available model for the new agent.
567                let _ = store_set_by_id(
568                    &Some(store),
569                    store_id,
570                    &[
571                        ("agent", serde_json::json!(fallback)),
572                        ("model", serde_json::json!("")),
573                        (
574                            "last_error",
575                            serde_json::json!(format!(
576                                "silence detected after {}s, failing over to {}",
577                                config.silence_grace_period, fallback
578                            )),
579                        ),
580                    ],
581                )
582                .await;
583            } else {
584                let _ = store_set_by_id(
585                    &Some(store),
586                    store_id,
587                    &[
588                        ("agent", serde_json::Value::Null),
589                        ("model", serde_json::Value::Null),
590                        (
591                            "last_error",
592                            serde_json::json!(format!(
593                                "silence detected after {}s, no fallback agents available",
594                                config.silence_grace_period
595                            )),
596                        ),
597                    ],
598                )
599                .await;
600            }
601        }
602
603        if let Err(e) = task_manager
604            .update_task_status(&task_eid, next_status)
605            .await
606        {
607            tracing::warn!(task_id, ?e, "failed to update silent task status");
608            continue;
609        }
610
611        // Log reroute activity
612        let details = serde_json::json!({
613            "failure_reason": "agent_silence",
614            "silence_duration_secs": config.silence_grace_period,
615            "cooldown_applied": !agent_name.is_empty() && !model_name.is_empty(),
616            "fallback_available": next_agent.is_some(),
617        });
618        crate::store::store_log_activity(
619            &Some(Arc::clone(store)),
620            repo,
621            &task_id,
622            "rerouted",
623            Some("in_progress"),
624            Some(next_status.as_str()),
625            next_agent.as_deref(),
626            None,
627            Some(&details),
628        )
629        .await;
630
631        // 5. Post a comment explaining what happened
632        let action = if let Some(ref fallback) = next_agent {
633            format!("failing over to {fallback}")
634        } else {
635            "marking needs_review (no fallback agents)".to_string()
636        };
637        let comment = format!(
638            "[{}] agent silent for {}s since session start — killed session, cooled down model `{}:{}` for {}s{}, agent `{}` for {}s, {}{}",
639            chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
640            config.silence_grace_period,
641            agent_name,
642            model_name,
643            config.silence_cooldown,
644            extended_note,
645            agent_name,
646            SILENCE_AGENT_COOLDOWN_SECS,
647            action,
648            crate::engine::orch_footer(),
649        );
650        if use_backend {
651            // Fire-and-forget: comment is informational — tick should not block on it.
652            let backend_clone = Arc::clone(backend);
653            let task_eid_clone = task_eid.clone();
654            tokio::spawn(async move {
655                if let Err(e) = backend_clone.post_comment(&task_eid_clone, &comment).await {
656                    tracing::warn!(
657                        task_id = task_eid_clone.0,
658                        ?e,
659                        "failed to post silence detection comment"
660                    );
661                }
662            });
663        }
664    }
665
666    Ok(())
667}
668
669fn should_use_backend(task_id: &str) -> bool {
670    !is_internal_id(task_id)
671}
672
673/// Phase 2 of tick: recover tasks stranded by missing or stale tmux state.
674/// - `in_progress` tasks → reset to `New` when their session is missing or the run timed out
675/// - `routed` tasks      → reset to `New` when an old live session keeps blocking re-dispatch
676/// - `in_review` tasks   → reset to `NeedsReview` when the review session disappeared
677#[allow(clippy::too_many_arguments)]
678pub(crate) async fn tick_recover_stuck_tasks(
679    backend: &Arc<dyn ExternalBackend>,
680    tmux: &Arc<TmuxManager>,
681    repo: &str,
682    task_manager: &Arc<TaskManager>,
683    config: &EngineConfig,
684    store: &Arc<TaskStore>,
685    session_map: &std::collections::HashMap<String, bool>,
686    router: &Router,
687    dispatching: &Arc<DashMap<String, String>>,
688    auto_merge_in_flight: &Arc<DashSet<String>>,
689) -> anyhow::Result<()> {
690    let _span = tracing::info_span!("engine.tick.phase2.stuck_tasks").entered();
691
692    let in_progress = match task_manager
693        .list_external_by_status(Status::InProgress)
694        .await
695    {
696        Ok(tasks) => tasks,
697        Err(e) => {
698            tracing::warn!(?e, "failed to list in_progress tasks for stuck recovery");
699            vec![]
700        }
701    };
702    for task in &in_progress {
703        let Some(timing) = stuck_task_timing_from_map(
704            tmux,
705            repo,
706            &task.id.0,
707            &task.updated_at,
708            config,
709            "cannot parse updated_at, skipping stuck-task check",
710            session_map,
711        ) else {
712            continue;
713        };
714
715        // A dead tmux session with no `updated_at` refresh in a long time normally means
716        // the runner vanished (crashed, orch restarted). But it also matches the tail end
717        // of a *successful* run: the session exits the instant the agent finishes, while
718        // the runner is still polling for that exit (up to its own poll interval) and then
719        // writing results. If the runner's dispatch guard for this task is still held, the
720        // runner is alive and actively processing — let it finish rather than racing it.
721        if !timing.has_session {
722            let dispatch_key = format!("{repo}/{}", task.id.0);
723            if dispatching.contains_key(&dispatch_key) {
724                // Promoted from debug! to info!: the default log filter is "orch=info"
725                // (see main.rs), so this line — the only direct evidence that the guard
726                // check actually found and honored a held dispatch guard — was silently
727                // dropped in production. Every prior occurrence of the reclaim race
728                // (#3518/#3519/#3523/#3525/#3526) had to be diagnosed from timestamp
729                // correlation alone because this and the "dispatch guard
730                // claimed"/"released" lines never appeared in orch.log.
731                tracing::info!(
732                    task_id = task.id.0,
733                    dispatch_key,
734                    age_mins = timing.age.num_minutes(),
735                    "skipping stuck-task reclaim: runner still actively processing this task"
736                );
737                continue;
738            }
739        }
740
741        if timing.has_session {
742            tracing::warn!(
743                task_id = task.id.0,
744                age_mins = timing.age.num_minutes(),
745                threshold_mins = timing.threshold / 60,
746                "recovering stuck task: timed out with active session → new"
747            );
748        } else {
749            tracing::warn!(
750                task_id = task.id.0,
751                dispatch_key = format!("{repo}/{}", task.id.0),
752                age_mins = timing.age.num_minutes(),
753                threshold_mins = timing.threshold / 60,
754                "recovering stuck task: no session found — reclaiming early → new"
755            );
756        }
757        // For stuck tasks with an active session, record agent/model failure + cooldown so the
758        // router picks a different agent/model on the next attempt. Without this, the router
759        // sees the agent as healthy and selects it again, causing the hang to repeat.
760        // (Mirrors the pattern in tick_detect_silent_agents lines 219–284.)
761        let mut cached_store_id: Option<i64> = None;
762        if timing.has_session {
763            let store_task = match store.resolve_task_id(repo, &task.id.0).await {
764                Ok(Some(store_id)) => {
765                    cached_store_id = Some(store_id);
766                    match store.get(store_id).await {
767                        Ok(t) => Some(t),
768                        Err(e) => {
769                            tracing::warn!(task_id = task.id.0, error = %e, "failed to fetch task from store for stuck-task cooldown");
770                            None
771                        }
772                    }
773                }
774                Ok(None) => None,
775                Err(e) => {
776                    tracing::warn!(task_id = task.id.0, error = %e, "failed to resolve task id for stuck-task cooldown");
777                    None
778                }
779            };
780            let agent_name = store_task
781                .as_ref()
782                .and_then(|t| t.agent.clone())
783                .unwrap_or_default();
784            let model_name = store_task
785                .as_ref()
786                .and_then(|t| t.model.clone())
787                .unwrap_or_default();
788
789            if !agent_name.is_empty() && !model_name.is_empty() {
790                set_model_cooldown(
791                    &agent_name,
792                    &model_name,
793                    config.silence_cooldown,
794                    "silence_detected",
795                )
796                .await;
797                record_agent_failure_with_message(
798                    &agent_name,
799                    &format!(
800                        "stuck with active session after {}m",
801                        timing.age.num_minutes()
802                    ),
803                )
804                .await;
805            }
806            if !agent_name.is_empty() {
807                set_agent_cooldown(&agent_name, SILENCE_AGENT_COOLDOWN_SECS).await;
808            }
809            tracing::warn!(
810                task_id = task.id.0,
811                agent = %agent_name,
812                model = %model_name,
813                cooldown_secs = config.silence_cooldown,
814                "stuck-task cooldown applied — router will avoid this agent/model on next attempt"
815            );
816        }
817
818        // Remove stale agent/model labels so the LLM router re-routes properly.
819        // Fire-and-forget: cosmetic label operation, tick should not block on it.
820        {
821            let stale_labels: Vec<String> = task
822                .labels
823                .iter()
824                .filter(|l| l.starts_with("agent:") || l.starts_with("model:"))
825                .cloned()
826                .collect();
827            if !stale_labels.is_empty() {
828                let backend_clone = Arc::clone(backend);
829                let task_id_clone = task.id.clone();
830                tokio::spawn(async move {
831                    for label in &stale_labels {
832                        if let Err(e) = backend_clone.remove_label(&task_id_clone, label).await {
833                            tracing::warn!(task_id = task_id_clone.0, label, error = %e, "failed to remove stale routing label during stuck-task recovery");
834                        }
835                    }
836                });
837            }
838        }
839        let resolved_store_id = match cached_store_id {
840            Some(id) => Some(id),
841            None => match store.resolve_task_id(repo, &task.id.0).await {
842                Ok(Some(id)) => Some(id),
843                Ok(None) => {
844                    tracing::warn!(
845                        task_id = %task.id.0,
846                        repo,
847                        "resolve_task_id returned None during stuck-task recovery"
848                    );
849                    None
850                }
851                Err(e) => {
852                    tracing::error!(
853                        task_id = %task.id.0,
854                        repo,
855                        error = %e,
856                        "resolve_task_id failed during stuck-task recovery"
857                    );
858                    None
859                }
860            },
861        };
862        if let Some(store_id) = resolved_store_id {
863            if let Err(e) = store_set_by_id(
864                &Some(store),
865                store_id,
866                &[
867                    ("agent", serde_json::Value::Null),
868                    ("model", serde_json::Value::Null),
869                    ("route_attempts", serde_json::json!(0)),
870                ],
871            )
872            .await
873            {
874                tracing::warn!(
875                    task_id = task.id.0,
876                    ?e,
877                    "failed to clear routing fields for stuck task — will retry next tick"
878                );
879                continue;
880            }
881            finalize_open_runs_for_recovery(
882                store,
883                store_id,
884                if timing.has_session {
885                    "stuck-task recovery: active session killed"
886                } else {
887                    "stuck-task recovery: no session found"
888                },
889            )
890            .await;
891        }
892        if let Err(e) = task_manager.update_task_status(&task.id, Status::New).await {
893            tracing::warn!(task_id = task.id.0, ?e, "failed to reset stuck task status");
894            continue;
895        }
896
897        // Log reroute activity for stuck task recovery
898        let details = serde_json::json!({
899            "failure_reason": if timing.has_session { "stuck_with_session" } else { "stuck_no_session" },
900            "age_minutes": timing.age.num_minutes(),
901            "had_session": timing.has_session,
902        });
903        crate::store::store_log_activity(
904            &Some(Arc::clone(store)),
905            repo,
906            &task.id.0,
907            "rerouted",
908            Some("in_progress"),
909            Some("new"),
910            None::<&str>,
911            None::<&str>,
912            Some(&details),
913        )
914        .await;
915
916        let reason = if timing.has_session {
917            format!(
918                "timed out after {}m with active session (cleared agent for re-routing)",
919                timing.age.num_minutes()
920            )
921        } else {
922            format!(
923                "no session found — reclaiming early after {}m (cleared agent for re-routing)",
924                timing.age.num_minutes()
925            )
926        };
927        // Fire-and-forget: comment is informational — tick should not block on it.
928        {
929            let backend_clone = Arc::clone(backend);
930            let task_id_clone = task.id.clone();
931            let body = format!(
932                "[{}] recovered: stuck in_progress — {}{}",
933                chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
934                reason,
935                crate::engine::orch_footer()
936            );
937            tokio::spawn(async move {
938                if let Err(e) = backend_clone.post_comment(&task_id_clone, &body).await {
939                    tracing::warn!(
940                        task_id = task_id_clone.0,
941                        ?e,
942                        "failed to post stuck-task recovery comment"
943                    );
944                }
945            });
946        }
947    }
948
949    let mut routed = match task_manager.list_external_by_status(Status::Routed).await {
950        Ok(tasks) => tasks,
951        Err(e) => {
952            tracing::warn!(?e, "failed to list routed tasks for stuck recovery");
953            vec![]
954        }
955    };
956    match task_manager
957        .list_internal_by_status(crate::store::TaskStatus::Routed)
958        .await
959    {
960        Ok(tasks) => routed.extend(tasks),
961        Err(e) => tracing::warn!(
962            ?e,
963            "failed to list internal routed tasks for stuck recovery"
964        ),
965    }
966    for task in &routed {
967        let Some(timing) = stuck_task_timing_from_map(
968            tmux,
969            repo,
970            &task.id.0,
971            &task.updated_at,
972            config,
973            "cannot parse updated_at, skipping routed stuck-task check",
974            session_map,
975        ) else {
976            continue;
977        };
978
979        if !timing.has_session {
980            continue;
981        }
982
983        let session_name = tmux.session_name(repo, &task.id.0);
984        if let Err(e) = tmux.kill_session(&session_name).await {
985            tracing::warn!(
986                task_id = task.id.0,
987                session = %session_name,
988                error = %e,
989                "failed to kill stale tmux session blocking routed task"
990            );
991            continue;
992        }
993
994        recover_routed_blocked_dispatch(backend, store, task_manager, repo, task, &timing).await;
995    }
996
997    // Global recovery for external tasks from repos that are not currently being ticked.
998    // These tasks would otherwise remain in_progress forever if their project was removed
999    // from active config and never enters this tick loop again.
1000    let all_external_in_progress = match store
1001        .list_all_by_status_global(crate::store::TaskStatus::InProgress)
1002        .await
1003    {
1004        Ok(tasks) => tasks,
1005        Err(e) => {
1006            tracing::warn!(
1007                ?e,
1008                "failed to list global in_progress tasks for cross-repo stuck recovery"
1009            );
1010            vec![]
1011        }
1012    };
1013    for task in all_external_in_progress
1014        .iter()
1015        .filter(|t| t.origin != "internal" && t.repo != repo)
1016    {
1017        let external_id = task
1018            .external_id
1019            .clone()
1020            .unwrap_or_else(|| task.id.to_string());
1021        let Some(timing) = stuck_task_timing_from_map(
1022            tmux,
1023            &task.repo,
1024            &external_id,
1025            &task.updated_at,
1026            config,
1027            "cannot parse updated_at, skipping cross-repo stuck-task check",
1028            session_map,
1029        ) else {
1030            continue;
1031        };
1032
1033        // See the comment on the in-progress loop above: a dead session with a stale
1034        // `updated_at` also matches the tail end of a run that just finished successfully.
1035        // Skip the reclaim while the runner's dispatch guard for this task is still held.
1036        if !timing.has_session {
1037            let dispatch_key = format!("{}/{external_id}", task.repo);
1038            if dispatching.contains_key(&dispatch_key) {
1039                // See the same promotion note on the internal-task loop below: this was
1040                // debug! (silently dropped under the default "orch=info" filter), leaving
1041                // every prior reclaim-race occurrence undiagnosable from logs alone.
1042                tracing::info!(
1043                    task_id = task.id,
1044                    repo = %task.repo,
1045                    dispatch_key,
1046                    age_mins = timing.age.num_minutes(),
1047                    "skipping cross-repo stuck-task reclaim: runner still actively processing this task"
1048                );
1049                continue;
1050            }
1051        }
1052
1053        if timing.has_session {
1054            tracing::warn!(
1055                task_id = task.id,
1056                repo = %task.repo,
1057                age_mins = timing.age.num_minutes(),
1058                threshold_mins = timing.threshold / 60,
1059                "recovering cross-repo stuck task: timed out with active session → new"
1060            );
1061        } else {
1062            tracing::warn!(
1063                task_id = task.id,
1064                repo = %task.repo,
1065                dispatch_key = format!("{}/{external_id}", task.repo),
1066                age_mins = timing.age.num_minutes(),
1067                threshold_mins = timing.threshold / 60,
1068                "recovering cross-repo stuck task: no session found — reclaiming early → new"
1069            );
1070        }
1071
1072        if let Err(e) = store_set_by_id(
1073            &Some(store),
1074            task.id,
1075            &[
1076                ("agent", serde_json::Value::Null),
1077                ("model", serde_json::Value::Null),
1078                ("route_attempts", serde_json::json!(0)),
1079            ],
1080        )
1081        .await
1082        {
1083            tracing::warn!(
1084                task_id = task.id,
1085                repo = %task.repo,
1086                ?e,
1087                "failed to clear routing fields for cross-repo stuck task — will retry next tick"
1088            );
1089            continue;
1090        }
1091
1092        finalize_open_runs_for_recovery(
1093            store,
1094            task.id,
1095            if timing.has_session {
1096                "stuck-task recovery: cross-repo active session killed"
1097            } else {
1098                "stuck-task recovery: cross-repo no session found"
1099            },
1100        )
1101        .await;
1102
1103        if let Err(e) = store
1104            .update_status(task.id, crate::store::TaskStatus::New)
1105            .await
1106        {
1107            tracing::warn!(
1108                task_id = task.id,
1109                repo = %task.repo,
1110                ?e,
1111                "failed to reset cross-repo stuck task status"
1112            );
1113            continue;
1114        }
1115
1116        let details = serde_json::json!({
1117            "failure_reason": if timing.has_session { "stuck_with_session" } else { "stuck_no_session" },
1118            "age_minutes": timing.age.num_minutes(),
1119            "had_session": timing.has_session,
1120            "scope": "cross_repo_global_sweep",
1121        });
1122        crate::store::store_log_activity(
1123            &Some(Arc::clone(store)),
1124            &task.repo,
1125            &external_id,
1126            "rerouted",
1127            Some("in_progress"),
1128            Some("new"),
1129            None::<&str>,
1130            None::<&str>,
1131            Some(&details),
1132        )
1133        .await;
1134    }
1135
1136    // Recover internal (SQLite) tasks stuck in in_progress.
1137    // These have no GitHub labels or comments — just reset the DB status to New.
1138    use crate::store::TaskStatus as DbStatus;
1139    let internal_in_progress = match task_manager
1140        .list_internal_by_status(DbStatus::InProgress)
1141        .await
1142    {
1143        Ok(tasks) => tasks,
1144        Err(e) => {
1145            tracing::warn!(
1146                ?e,
1147                "failed to list internal in_progress tasks for stuck recovery"
1148            );
1149            vec![]
1150        }
1151    };
1152    for task in &internal_in_progress {
1153        let task_id = task.id.0.clone();
1154        let Some(timing) = stuck_task_timing_from_map(
1155            tmux,
1156            repo,
1157            &task_id,
1158            &task.updated_at,
1159            config,
1160            "cannot parse updated_at, skipping stuck internal-task check",
1161            session_map,
1162        ) else {
1163            continue;
1164        };
1165
1166        // See the comment on the external in-progress loop above: a dead session with a
1167        // stale `updated_at` also matches the tail end of a run that just finished
1168        // successfully. Skip the reclaim while the runner's dispatch guard for this task
1169        // is still held — this is the exact race that discards completed daily/scheduled
1170        // internal tasks whose total runtime exceeds `no_session_stuck_timeout`.
1171        if !timing.has_session {
1172            let dispatch_key = format!("{repo}/{task_id}");
1173            if dispatching.contains_key(&dispatch_key) {
1174                // Promoted from debug! to info! (issue #3526): the default log filter is
1175                // "orch=info" (main.rs), so this line — along with the DispatchGuard
1176                // claim/release lines in dispatch_guard.rs and dispatch_tasks_for_repo —
1177                // never reached orch.log in production. Every occurrence of the reclaim
1178                // race (#3518, #3519, #3523, #3525, #3526) had to be reasoned about from
1179                // timestamp correlation across unrelated log lines instead of directly
1180                // observing whether the guard was held, because the one log line that
1181                // would prove it was silently filtered out. Promoting these closes that
1182                // observability gap so the next occurrence carries conclusive evidence.
1183                tracing::info!(
1184                    task_id,
1185                    dispatch_key,
1186                    age_mins = timing.age.num_minutes(),
1187                    "skipping stuck-task reclaim: runner still actively processing this task"
1188                );
1189                continue;
1190            }
1191        }
1192
1193        if timing.has_session {
1194            tracing::warn!(
1195                task_id,
1196                age_mins = timing.age.num_minutes(),
1197                threshold_mins = timing.threshold / 60,
1198                "recovering stuck task: timed out with active session → new"
1199            );
1200        } else {
1201            tracing::warn!(
1202                task_id,
1203                dispatch_key = format!("{repo}/{task_id}"),
1204                age_mins = timing.age.num_minutes(),
1205                threshold_mins = timing.threshold / 60,
1206                "recovering stuck task: no session found — reclaiming early → new"
1207            );
1208        }
1209        // Apply agent/model cooldown for internal stuck tasks with active sessions,
1210        // mirroring the external path (lines 501-546). Without this, the router picks
1211        // the same agent/model that caused the hang, creating an infinite loop.
1212        let mut cached_store_id: Option<i64> = None;
1213        if timing.has_session {
1214            let store_task = match store.resolve_task_id(repo, &task_id).await {
1215                Ok(Some(store_id)) => {
1216                    cached_store_id = Some(store_id);
1217                    match store.get(store_id).await {
1218                        Ok(t) => Some(t),
1219                        Err(e) => {
1220                            tracing::warn!(task_id, error = %e, "failed to fetch task from store for stuck internal-task cooldown");
1221                            None
1222                        }
1223                    }
1224                }
1225                Ok(None) => None,
1226                Err(e) => {
1227                    tracing::warn!(task_id, error = %e, "failed to resolve task id for stuck internal-task cooldown");
1228                    None
1229                }
1230            };
1231            let agent_name = store_task
1232                .as_ref()
1233                .and_then(|t| t.agent.clone())
1234                .unwrap_or_default();
1235            let model_name = store_task
1236                .as_ref()
1237                .and_then(|t| t.model.clone())
1238                .unwrap_or_default();
1239
1240            if !agent_name.is_empty() && !model_name.is_empty() {
1241                set_model_cooldown(
1242                    &agent_name,
1243                    &model_name,
1244                    config.silence_cooldown,
1245                    "silence_detected",
1246                )
1247                .await;
1248                record_agent_failure_with_message(
1249                    &agent_name,
1250                    &format!(
1251                        "internal task stuck with active session after {}m",
1252                        timing.age.num_minutes()
1253                    ),
1254                )
1255                .await;
1256            }
1257            if !agent_name.is_empty() {
1258                set_agent_cooldown(&agent_name, SILENCE_AGENT_COOLDOWN_SECS).await;
1259            }
1260            tracing::warn!(
1261                task_id,
1262                agent = %agent_name,
1263                model = %model_name,
1264                cooldown_secs = config.silence_cooldown,
1265                "internal stuck-task cooldown applied — router will avoid this agent/model on next attempt"
1266            );
1267        }
1268        // Reset routing state so the LLM router is used on the next attempt
1269        // (same reset that external tasks perform).
1270        let resolved_store_id = match cached_store_id {
1271            Some(id) => Some(id),
1272            None => match store.resolve_task_id(repo, &task_id).await {
1273                Ok(Some(id)) => Some(id),
1274                Ok(None) => {
1275                    tracing::warn!(
1276                        task_id = %task_id,
1277                        repo,
1278                        "resolve_task_id returned None during stuck internal-task recovery"
1279                    );
1280                    None
1281                }
1282                Err(e) => {
1283                    tracing::error!(
1284                        task_id = %task_id,
1285                        repo,
1286                        error = %e,
1287                        "resolve_task_id failed during stuck internal-task recovery"
1288                    );
1289                    None
1290                }
1291            },
1292        };
1293        if let Some(store_id) = resolved_store_id {
1294            if let Err(e) = store_set_by_id(
1295                &Some(store),
1296                store_id,
1297                &[
1298                    ("agent", serde_json::Value::Null),
1299                    ("model", serde_json::Value::Null),
1300                    ("route_attempts", serde_json::json!(0)),
1301                ],
1302            )
1303            .await
1304            {
1305                tracing::warn!(
1306                    task_id,
1307                    ?e,
1308                    "failed to clear routing fields for stuck internal task — will retry next tick"
1309                );
1310                continue;
1311            }
1312            finalize_open_runs_for_recovery(
1313                store,
1314                store_id,
1315                if timing.has_session {
1316                    "stuck-task recovery: internal active session killed"
1317                } else {
1318                    "stuck-task recovery: internal no session found"
1319                },
1320            )
1321            .await;
1322        }
1323        if let Err(e) = task_manager
1324            .update_task_status(&ExternalId(task_id.clone()), Status::New)
1325            .await
1326        {
1327            tracing::warn!(task_id, ?e, "failed to reset stuck internal task status");
1328        }
1329    }
1330
1331    // Recover external tasks stuck in in_review.
1332    // Unlike in_progress recovery, we reset to NeedsReview (not New) so the review
1333    // agent re-triggers without clearing the routing state.
1334    let in_review = match task_manager.list_external_by_status(Status::InReview).await {
1335        Ok(tasks) => tasks,
1336        Err(e) => {
1337            tracing::warn!(?e, "failed to list in_review tasks for stuck recovery");
1338            vec![]
1339        }
1340    };
1341    // Read the stored in_review rows once to avoid per-task DB lookups for
1342    // `review_session_expected`. Build a set of external_ids that have the
1343    // flag set so the per-task loop can check it cheaply.
1344    let store_in_review = match store
1345        .list_by_status(repo, crate::store::TaskStatus::InReview)
1346        .await
1347    {
1348        Ok(v) => v,
1349        Err(e) => {
1350            tracing::warn!(
1351                ?e,
1352                "failed to list store in_review tasks for review_session_expected prefetch"
1353            );
1354            vec![]
1355        }
1356    };
1357    let review_expected_set: std::collections::HashSet<String> = store_in_review
1358        .into_iter()
1359        .filter(|t| t.review_session_expected)
1360        .filter_map(|t| t.external_id)
1361        .collect();
1362    for task in &in_review {
1363        if !review_expected_set.contains(&task.id.0) {
1364            tracing::debug!(
1365                task_id = task.id.0,
1366                "in_review task is waiting on PR review, skipping stuck-session recovery"
1367            );
1368            continue;
1369        }
1370
1371        let review_task_id = format!("{}-review", task.id.0);
1372        // Use in_review_no_session_stuck_timeout instead of the standard
1373        // no_session_stuck_timeout. Review agents exit their tmux session on
1374        // normal completion before delivering the result to the engine, so a
1375        // short no-session timeout causes a race: the stuck recovery fires,
1376        // resets the task to needs_review, and the completed result arriving
1377        // moments later is discarded as stale. A longer threshold gives the
1378        // result delivery enough time to complete.
1379        let in_review_config = crate::engine::EngineConfig {
1380            no_session_stuck_timeout: config.in_review_no_session_stuck_timeout,
1381            ..config.clone()
1382        };
1383        let Some(timing) = stuck_task_timing_from_map(
1384            tmux,
1385            repo,
1386            &review_task_id,
1387            &task.updated_at,
1388            &in_review_config,
1389            "cannot parse updated_at, skipping stuck in_review check",
1390            session_map,
1391        ) else {
1392            continue;
1393        };
1394
1395        tracing::warn!(
1396            task_id = task.id.0,
1397            age_mins = timing.age.num_minutes(),
1398            threshold_mins = timing.threshold / 60,
1399            "recovering stuck in_review task: no session found — resetting to needs_review"
1400        );
1401        if let Err(e) = task_manager
1402            .update_task_status(&task.id, Status::NeedsReview)
1403            .await
1404        {
1405            tracing::warn!(
1406                task_id = task.id.0,
1407                ?e,
1408                "failed to reset stuck in_review task status"
1409            );
1410            continue;
1411        } else {
1412            set_review_session_expected(store, repo, &task.id.0, false).await;
1413        }
1414
1415        // Finalize any open review run row
1416        if let Ok(Some(store_id)) = store.resolve_task_id(repo, &task.id.0).await {
1417            finalize_open_runs_for_recovery(
1418                store,
1419                store_id,
1420                "stuck-task recovery: in_review session killed",
1421            )
1422            .await;
1423        }
1424
1425        // Log activity for stuck in_review recovery
1426        let details = serde_json::json!({
1427            "failure_reason": "stuck_in_review",
1428            "age_minutes": timing.age.num_minutes(),
1429            "had_session": timing.has_session,
1430        });
1431        crate::store::store_log_activity(
1432            &Some(Arc::clone(store)),
1433            repo,
1434            &task.id.0,
1435            "rerouted",
1436            Some("in_review"),
1437            Some("needs_review"),
1438            None::<&str>,
1439            None::<&str>,
1440            Some(&details),
1441        )
1442        .await;
1443        if let Err(e) = backend
1444            .post_comment(
1445                &task.id,
1446                &format!(
1447                    "[{}] recovered: stuck in_review — no session found after {}m, resetting to needs_review{}",
1448                    chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
1449                    timing.age.num_minutes(),
1450                    crate::engine::orch_footer()
1451                ),
1452            )
1453            .await
1454        {
1455            tracing::warn!(
1456                task_id = task.id.0,
1457                ?e,
1458                "failed to post stuck in_review recovery comment"
1459            );
1460        }
1461    }
1462
1463    // Recover internal (SQLite) tasks stuck in in_review.
1464    let internal_in_review = match task_manager
1465        .list_internal_by_status(DbStatus::InReview)
1466        .await
1467    {
1468        Ok(tasks) => tasks,
1469        Err(e) => {
1470            tracing::warn!(
1471                ?e,
1472                "failed to list internal in_review tasks for stuck recovery"
1473            );
1474            vec![]
1475        }
1476    };
1477    for task in &internal_in_review {
1478        let task_id = task.id.0.clone();
1479        if !review_expected_set.contains(&task_id) {
1480            tracing::debug!(
1481                task_id,
1482                "internal in_review task is waiting on PR review, skipping stuck-session recovery"
1483            );
1484            continue;
1485        }
1486
1487        let review_task_id = format!("{}-review", task_id);
1488        let in_review_config = crate::engine::EngineConfig {
1489            no_session_stuck_timeout: config.in_review_no_session_stuck_timeout,
1490            ..config.clone()
1491        };
1492        let Some(timing) = stuck_task_timing_from_map(
1493            tmux,
1494            repo,
1495            &review_task_id,
1496            &task.updated_at,
1497            &in_review_config,
1498            "cannot parse updated_at, skipping stuck internal in_review check",
1499            session_map,
1500        ) else {
1501            continue;
1502        };
1503
1504        tracing::warn!(
1505            task_id,
1506            age_mins = timing.age.num_minutes(),
1507            threshold_mins = timing.threshold / 60,
1508            "recovering stuck internal in_review task: no session found — resetting to needs_review"
1509        );
1510        if let Err(e) = task_manager
1511            .update_task_status(&ExternalId(task_id.clone()), Status::NeedsReview)
1512            .await
1513        {
1514            tracing::warn!(
1515                task_id,
1516                ?e,
1517                "failed to reset stuck internal in_review task status"
1518            );
1519        } else {
1520            set_review_session_expected(store, repo, &task_id, false).await;
1521        }
1522
1523        // Finalize any open review run row
1524        if let Ok(Some(store_id)) = store.resolve_task_id(repo, &task_id).await {
1525            finalize_open_runs_for_recovery(
1526                store,
1527                store_id,
1528                "stuck-task recovery: internal in_review session killed",
1529            )
1530            .await;
1531        }
1532    }
1533
1534    // Global sweep for CI-failure blocked tasks from inactive or removed projects.
1535    // auto_unblock_blocked_tasks runs inside sync_tick and is scoped to the active repo.
1536    // Tasks blocked for projects no longer in orch config are never processed there.
1537    // This sweep covers all repos so stale CI-failure blocks are eventually resolved.
1538    if let Err(e) = crate::engine::sync::auto_unblock_ci_failure_blocked_tasks_global(
1539        task_manager,
1540        store,
1541        auto_merge_in_flight,
1542    )
1543    .await
1544    {
1545        tracing::warn!(err = %e, "global CI-failure blocked sweep failed");
1546    }
1547
1548    // Global sweep for stale NeedsReview tasks from inactive or removed projects.
1549    // The refire/escalation logic in sync_tick's "5d" step is scoped to the active
1550    // repo, so a task that reaches NeedsReview for a repo later removed from the
1551    // active `projects:` list never refires or escalates to Blocked. This sweep
1552    // covers all repos, writing directly through the store to avoid the repo-scoped
1553    // `TaskManager` resolution failure (same pattern as the CI-failure sweep above).
1554    if let Err(e) = crate::engine::sync::refire_and_escalate_stale_needs_review_global(
1555        task_manager,
1556        store,
1557        router,
1558        dispatching,
1559    )
1560    .await
1561    {
1562        tracing::warn!(err = %e, "global NeedsReview refire/escalation sweep failed");
1563    }
1564
1565    // Global sweep for rebroadcast-blocked tasks from inactive or removed projects.
1566    // The escalation sweep above (`refire_and_escalate_stale_needs_review_global`) that
1567    // *writes* this block_reason already runs across all repos, but the only recovery
1568    // path (`auto_recover_rebroadcast_blocked_tasks`) is scoped to the active repo via
1569    // `sync_tick` — so a task escalated for a repo later removed from the active
1570    // `projects:` list could never recover. This sweep covers all repos (same pattern
1571    // as the two sweeps above).
1572    if let Err(e) = crate::engine::sync::auto_recover_rebroadcast_blocked_tasks_global(
1573        router,
1574        task_manager,
1575        store,
1576    )
1577    .await
1578    {
1579        tracing::warn!(err = %e, "global rebroadcast-blocked recovery sweep failed");
1580    }
1581
1582    Ok(())
1583}
1584
1585/// Phase 3a of tick: route status:new tasks to an agent and transition them to status:routed.
1586pub(crate) async fn tick_route_tasks(
1587    backend: &Arc<dyn ExternalBackend>,
1588    task_manager: &Arc<TaskManager>,
1589    router: &mut Router,
1590    store: &Arc<TaskStore>,
1591    repo: &str,
1592    routing_quota: &mut RoutingTickQuota,
1593) -> anyhow::Result<()> {
1594    let _span = tracing::info_span!("engine.tick.phase3a.route").entered();
1595    if let Err(err) = crate::engine::dependencies::apply_claim_gates(store).await {
1596        tracing::warn!(err = %err, "dependency claim gate failed");
1597    }
1598    if routing_quota.is_exhausted() {
1599        tracing::debug!("routing quota exhausted for this tick — skipping repo routing phase");
1600        return Ok(());
1601    }
1602
1603    // Global GitHub 5xx circuit breaker — skip routing during sustained GitHub outages
1604    // to avoid routing-heavy retry storms. Tasks will remain in 'new' status and be
1605    // retried when the circuit closes.
1606    if is_github_circuit_open() {
1607        let remaining = github_circuit_remaining_secs();
1608        tracing::info!(
1609            remaining_secs = remaining,
1610            "GitHub 5xx circuit breaker open — skipping routing phase"
1611        );
1612        return Ok(());
1613    }
1614    let new_tasks = task_manager.list_routable().await?;
1615    let routable: Vec<&ExternalTask> = new_tasks
1616        .iter()
1617        .filter(|t| !skips_agent(&t.labels))
1618        .collect();
1619
1620    // Pre-emptive health check: refresh degraded-agent flags once per tick,
1621    // not per-task (previously this ran inside route() causing N redundant
1622    // DB queries when routing N tasks in the same tick).
1623    if !routable.is_empty() {
1624        let start = std::time::Instant::now();
1625        router.refresh_health(store).await;
1626        tracing::debug!(
1627            duration_ms = start.elapsed().as_millis(),
1628            "router.refresh_health completed"
1629        );
1630    }
1631
1632    // Limit routing to the remaining global quota for this outer engine tick.
1633    for task in routable.into_iter().take(routing_quota.remaining()) {
1634        if !routing_quota.consume_one() {
1635            break;
1636        }
1637        let _task_span = tracing::info_span!("engine.route", task_id = %task.id.0).entered();
1638
1639        let task_start = Instant::now();
1640        match router.route(task, store, repo).await {
1641            Ok(result) => {
1642                tracing::debug!(task_id = %task.id.0, duration_ms = task_start.elapsed().as_millis(), "route completed");
1643                // Store route result in store
1644                if let Err(e) = router
1645                    .store_route_result(&task.id.0, &result, store, repo)
1646                    .await
1647                {
1648                    tracing::warn!(
1649                        task_id = task.id.0,
1650                        ?e,
1651                        "failed to store route result; skipping Routed transition"
1652                    );
1653                    continue;
1654                }
1655
1656                if is_internal_id(&task.id.0) {
1657                    // Internal tasks: update DB status, skip GitHub label ops.
1658                    if let Err(e) = task_manager
1659                        .update_task_status(&task.id, Status::Routed)
1660                        .await
1661                    {
1662                        tracing::warn!(
1663                            task_id = task.id.0,
1664                            ?e,
1665                            "failed to set internal status:routed"
1666                        );
1667                        continue;
1668                    }
1669                } else {
1670                    // Fire-and-forget: label operations are cosmetic (routing already
1671                    // persisted in DB). Spawning them prevents blocking the tick loop
1672                    // on 6-9 sequential HTTP calls (~3-9s overhead per routed task).
1673                    let backend_clone = Arc::clone(backend);
1674                    let task_id_clone = task.id.clone();
1675                    let task_labels: Vec<String> = task.labels.clone();
1676                    let new_labels = {
1677                        let mut labels = vec![
1678                            format!("agent:{}", result.agent),
1679                            format!("complexity:{}", result.complexity),
1680                        ];
1681                        if let Some(ref model) = result.model {
1682                            labels.push(format!("model:{model}"));
1683                        }
1684                        labels
1685                    };
1686                    tokio::spawn(async move {
1687                        // Remove old agent/complexity/model labels
1688                        for label in &task_labels {
1689                            if label.starts_with("agent:")
1690                                || label.starts_with("complexity:")
1691                                || label.starts_with("model:")
1692                            {
1693                                if let Err(e) =
1694                                    backend_clone.remove_label(&task_id_clone, label).await
1695                                {
1696                                    tracing::warn!(task_id = task_id_clone.0, label, error = %e, "failed to remove stale routing label during re-route");
1697                                }
1698                            }
1699                        }
1700                        // Add new labels
1701                        if let Err(e) = backend_clone.set_labels(&task_id_clone, &new_labels).await
1702                        {
1703                            tracing::warn!(
1704                                task_id = task_id_clone.0,
1705                                ?e,
1706                                "failed to set routing labels"
1707                            );
1708                        }
1709                    });
1710
1711                    // Transition to routed
1712                    if let Err(e) = task_manager
1713                        .update_task_status(&task.id, Status::Routed)
1714                        .await
1715                    {
1716                        tracing::warn!(task_id = task.id.0, ?e, "failed to set status:routed");
1717                        continue;
1718                    }
1719                }
1720
1721                // Dual-write: upsert task + store route result in SQLite
1722                match store.ensure_external_task(repo, task).await {
1723                    Ok(store_id) => {
1724                        let profile_json =
1725                            serde_json::to_string(&result.profile).unwrap_or_default();
1726                        let skills_json =
1727                            serde_json::to_string(&result.selected_skills).unwrap_or_default();
1728                        if let Err(e) = store
1729                            .store_route(&crate::store::StoreRoute {
1730                                id: store_id,
1731                                agent: &result.agent,
1732                                model: result.model.as_deref(),
1733                                complexity: &result.complexity,
1734                                estimate: result.estimate,
1735                                reason: &result.reason,
1736                                profile: &profile_json,
1737                                skills: &skills_json,
1738                            })
1739                            .await
1740                        {
1741                            tracing::debug!(
1742                                task_id = task.id.0,
1743                                ?e,
1744                                "dual-write: store_route failed"
1745                            );
1746                        }
1747                        if let Err(e) = store
1748                            .update_status(store_id, crate::store::TaskStatus::Routed)
1749                            .await
1750                        {
1751                            tracing::debug!(
1752                                task_id = task.id.0,
1753                                ?e,
1754                                "dual-write: update_status failed"
1755                            );
1756                        }
1757                        // Sync the estimate to GitHub Projects if the router assigned one.
1758                        // Fire-and-forget: do NOT await inline — GraphQL mutations can take
1759                        // up to 30 s and would block the entire tick loop on every routed task.
1760                        if result.estimate > 0 {
1761                            let backend_clone = Arc::clone(backend);
1762                            let task_id = task.id.clone();
1763                            let estimate = result.estimate;
1764                            tokio::spawn(async move {
1765                                if let Err(e) = backend_clone
1766                                    .sync_estimate_to_project(&task_id, estimate)
1767                                    .await
1768                                {
1769                                    tracing::debug!(
1770                                        task_id = task_id.0,
1771                                        estimate,
1772                                        err = %e,
1773                                        "dual-write: estimate project sync failed"
1774                                    );
1775                                }
1776                            });
1777                        }
1778                    }
1779                    Err(e) => {
1780                        tracing::debug!(
1781                            task_id = task.id.0,
1782                            ?e,
1783                            "dual-write: ensure_external_task failed"
1784                        );
1785                    }
1786                }
1787
1788                if let Some(ref warning) = result.warning {
1789                    tracing::warn!(task_id = task.id.0, warning, "routing sanity warning");
1790                }
1791
1792                tracing::info!(
1793                    task_id = task.id.0,
1794                    agent = %result.agent,
1795                    complexity = %result.complexity,
1796                    reason = %result.reason,
1797                    "task routed"
1798                );
1799            }
1800            Err(e) => {
1801                if let Some(cooled) = e.downcast_ref::<AllAgentsCooledError>() {
1802                    tracing::error!(
1803                        task_id = %task.id.0,
1804                        scope = %cooled.scope(),
1805                        remaining_secs = ?cooled.remaining_secs(),
1806                        "ALL AGENTS COOLED — no agent available to route this task. \
1807                         Task stays in 'new' and will be retried on every tick. \
1808                         Check `orch cooldown list` to see what is blocking dispatch."
1809                    );
1810                    continue;
1811                }
1812                if let Some(waiting) = e.downcast_ref::<crate::store::WaitingOnDependency>() {
1813                    tracing::info!(
1814                        task_id = %task.id.0,
1815                        blockers = %waiting.blockers.join(", "),
1816                        "task is waiting on a dependency"
1817                    );
1818                    continue;
1819                }
1820                tracing::error!(task_id = task.id.0, ?e, "routing failed, skipping task");
1821            }
1822        }
1823    }
1824    Ok(())
1825}
1826
1827/// Global routing sweep: route status:new tasks from inactive or removed repos.
1828///
1829/// `tick_route_tasks` is scoped to the active repo per-tick and never processes
1830/// tasks whose `repo` field differs from the current project. When a project is
1831/// removed from the orch config, its tasks accumulate in `new` indefinitely
1832/// because no tick ever runs for that repo. This sweep covers all repos so
1833/// orphaned new tasks are eventually routed.
1834///
1835/// `active_repos` must be the full set of currently-configured project repos,
1836/// not just the repo of the tick that called this — mirroring
1837/// `dispatch_routed_tasks_global` (see its doc comment for the "racing the real
1838/// per-repo dispatch" failure mode this guards against). `tick()` runs once per
1839/// active project; filtering on `t.repo != current_repo` alone (the previous
1840/// behavior) made every *other* active repo's new tasks look "orphaned" too and
1841/// routed them a tick early, through the wrong repo's tick.
1842pub(crate) async fn route_new_tasks_global(
1843    router: &mut Router,
1844    store: &Arc<TaskStore>,
1845    active_repos: &std::collections::HashSet<String>,
1846    routing_quota: &mut RoutingTickQuota,
1847) -> anyhow::Result<()> {
1848    use crate::store::{StoreRoute, TaskStatus};
1849
1850    if routing_quota.is_exhausted() {
1851        tracing::debug!("routing quota exhausted for this tick — skipping global routing sweep");
1852        return Ok(());
1853    }
1854
1855    let all_new = store.list_all_by_status_global(TaskStatus::New).await?;
1856    let orphaned: Vec<&crate::store::Task> = all_new
1857        .iter()
1858        .filter(|t| !active_repos.contains(&t.repo))
1859        .filter(|t| !skips_agent(&t.labels))
1860        .collect();
1861
1862    if orphaned.is_empty() {
1863        return Ok(());
1864    }
1865
1866    tracing::info!(
1867        count = orphaned.len(),
1868        "global routing sweep: found new tasks from inactive repos"
1869    );
1870
1871    router.refresh_health(store).await;
1872
1873    for task in orphaned.into_iter().take(routing_quota.remaining()) {
1874        if !routing_quota.consume_one() {
1875            break;
1876        }
1877        let ext_task = crate::engine::tasks::store_task_to_external(task);
1878        let task_start = Instant::now();
1879        match router.route(&ext_task, store, &task.repo).await {
1880            Ok(result) => {
1881                tracing::debug!(
1882                    task_id = %ext_task.id.0,
1883                    repo = %task.repo,
1884                    duration_ms = task_start.elapsed().as_millis(),
1885                    "global sweep: route completed"
1886                );
1887                let profile_json = serde_json::to_string(&result.profile).unwrap_or_default();
1888                let skills_json =
1889                    serde_json::to_string(&result.selected_skills).unwrap_or_default();
1890                if let Err(e) = store
1891                    .store_route(&StoreRoute {
1892                        id: task.id,
1893                        agent: &result.agent,
1894                        model: result.model.as_deref(),
1895                        complexity: &result.complexity,
1896                        estimate: result.estimate,
1897                        reason: &result.reason,
1898                        profile: &profile_json,
1899                        skills: &skills_json,
1900                    })
1901                    .await
1902                {
1903                    tracing::warn!(
1904                        task_id = %ext_task.id.0,
1905                        repo = %task.repo,
1906                        err = %e,
1907                        "global sweep: failed to store route result; skipping status update"
1908                    );
1909                    continue;
1910                }
1911                if let Err(e) = store.update_status(task.id, TaskStatus::Routed).await {
1912                    tracing::warn!(
1913                        task_id = %ext_task.id.0,
1914                        repo = %task.repo,
1915                        err = %e,
1916                        "global sweep: failed to set status:routed"
1917                    );
1918                } else {
1919                    tracing::info!(
1920                        task_id = %ext_task.id.0,
1921                        repo = %task.repo,
1922                        agent = %result.agent,
1923                        complexity = %result.complexity,
1924                        "global sweep: task routed"
1925                    );
1926                }
1927                if let Some(ref warning) = result.warning {
1928                    tracing::warn!(task_id = %ext_task.id.0, warning, "global sweep: routing sanity warning");
1929                }
1930            }
1931            Err(e) => {
1932                if let Some(cooled) = e.downcast_ref::<AllAgentsCooledError>() {
1933                    tracing::error!(
1934                        task_id = %ext_task.id.0,
1935                        repo = %task.repo,
1936                        scope = %cooled.scope(),
1937                        remaining_secs = ?cooled.remaining_secs(),
1938                        "global sweep: ALL AGENTS COOLED — task stays in 'new'"
1939                    );
1940                    continue;
1941                }
1942                if let Some(waiting) = e.downcast_ref::<crate::store::WaitingOnDependency>() {
1943                    tracing::info!(
1944                        task_id = %ext_task.id.0,
1945                        repo = %task.repo,
1946                        blockers = %waiting.blockers.join(", "),
1947                        "global sweep: task is waiting on a dependency"
1948                    );
1949                    continue;
1950                }
1951                tracing::error!(
1952                    task_id = %ext_task.id.0,
1953                    repo = %task.repo,
1954                    err = ?e,
1955                    "global sweep: routing failed, skipping task"
1956                );
1957            }
1958        }
1959    }
1960
1961    Ok(())
1962}
1963
1964/// Phase 3b of tick: spawn agents for all status:routed tasks up to the parallel limit.
1965#[derive(Debug, Clone, Copy)]
1966pub(crate) struct DispatchMode {
1967    pub(crate) is_degraded: bool,
1968    pub(crate) healthy_agents: usize,
1969    pub(crate) threshold: usize,
1970}
1971
1972pub(crate) fn dispatch_mode_from_router(router: &Router) -> DispatchMode {
1973    let threshold = crate::engine::router::config::min_healthy_agents_threshold();
1974    let healthy_agents = router.healthy_agent_count("simple");
1975    let is_degraded = router.is_degraded(threshold);
1976    DispatchMode {
1977        is_degraded,
1978        healthy_agents,
1979        threshold,
1980    }
1981}
1982
1983/// Phase 3b of tick: spawn agents for all status:routed tasks up to the parallel limit.
1984#[allow(clippy::too_many_arguments)]
1985#[tracing::instrument(skip_all, name = "engine.tick.phase3b.dispatch")]
1986pub(crate) async fn tick_dispatch_tasks(
1987    backend: &Arc<dyn ExternalBackend>,
1988    tmux: &Arc<TmuxManager>,
1989    repo: &str,
1990    runner: &Arc<TaskRunner>,
1991    capture: &Arc<CaptureService>,
1992    semaphore: &Arc<Semaphore>,
1993    task_manager: &Arc<TaskManager>,
1994    weight_tx: &mpsc::Sender<WeightSignal>,
1995    dispatch_mode: DispatchMode,
1996    dispatching: &Arc<DashMap<String, String>>,
1997    store: &Arc<TaskStore>,
1998    session_map: &std::collections::HashMap<String, bool>,
1999) -> anyhow::Result<()> {
2000    // Phase 3a already drops no-agent and owner:human, this filter stays as defense in depth
2001    let mut routed_tasks = task_manager.list_external_by_status(Status::Routed).await?;
2002
2003    // Also include internal tasks in Routed status.
2004    use crate::store::TaskStatus as DbStatus;
2005    let internal_routed = task_manager
2006        .list_internal_by_status(DbStatus::Routed)
2007        .await?;
2008    routed_tasks.extend(internal_routed);
2009
2010    // Filter and collect owned tasks to avoid lifetime issues
2011    let dispatchable: Vec<ExternalTask> = routed_tasks
2012        .into_iter()
2013        .filter(|t| !skips_agent(&t.labels))
2014        .collect();
2015
2016    dispatch_tasks_for_repo(
2017        backend,
2018        tmux,
2019        repo,
2020        runner,
2021        capture,
2022        semaphore,
2023        task_manager,
2024        weight_tx,
2025        dispatch_mode,
2026        dispatching,
2027        store,
2028        session_map,
2029        dispatchable,
2030    )
2031    .await
2032}
2033
2034/// Dispatch a pre-fetched list of routed tasks for a single repo.
2035///
2036/// Shared by `tick_dispatch_tasks` (active-repo, per-tick dispatch) and
2037/// `dispatch_routed_tasks_global` (inactive-repo sweep) so both paths spawn
2038/// agents through the exact same worktree/tmux/runner logic.
2039#[allow(clippy::too_many_arguments)]
2040async fn dispatch_tasks_for_repo(
2041    backend: &Arc<dyn ExternalBackend>,
2042    tmux: &Arc<TmuxManager>,
2043    repo: &str,
2044    runner: &Arc<TaskRunner>,
2045    capture: &Arc<CaptureService>,
2046    semaphore: &Arc<Semaphore>,
2047    task_manager: &Arc<TaskManager>,
2048    weight_tx: &mpsc::Sender<WeightSignal>,
2049    dispatch_mode: DispatchMode,
2050    dispatching: &Arc<DashMap<String, String>>,
2051    store: &Arc<TaskStore>,
2052    session_map: &std::collections::HashMap<String, bool>,
2053    dispatchable: Vec<ExternalTask>,
2054) -> anyhow::Result<()> {
2055    if dispatchable.is_empty() {
2056        tracing::debug!(count = 0, "dispatchable tasks found");
2057        return Ok(());
2058    }
2059
2060    tracing::info!(count = dispatchable.len(), "dispatchable tasks found");
2061    if dispatch_mode.is_degraded {
2062        tracing::warn!(
2063            healthy_agents = dispatch_mode.healthy_agents,
2064            threshold = dispatch_mode.threshold,
2065            "degraded mode: using sequential dispatch"
2066        );
2067    }
2068
2069    let sequential_delay = if dispatch_mode.is_degraded {
2070        crate::engine::router::config::sequential_dispatch_delay_ms()
2071    } else {
2072        0
2073    };
2074
2075    for (idx, task) in dispatchable.into_iter().enumerate() {
2076        if crate::engine::dependencies::dispatch_is_waiting(store, repo, &task.id.0).await {
2077            continue;
2078        }
2079        // In degraded mode, add delay between dispatches to pace the system
2080        if idx > 0 && dispatch_mode.is_degraded {
2081            let delay_ms = sequential_delay;
2082            tracing::debug!(
2083                task_id = task.id.0,
2084                delay_ms = delay_ms,
2085                "sequential dispatch: waiting before dispatch"
2086            );
2087            tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
2088        }
2089        // In-memory guard: prevents double-dispatch due to GitHub API eventual consistency.
2090        // After update_status(InProgress), the label removal fires a webhook that can
2091        // trigger an immediate tick. GitHub's search index may not yet reflect the label
2092        // change, so list_by_status(Routed) can still return this task. The tmux session
2093        // does not exist until the runner completes worktree setup (~10s later), so the
2094        // session_exists check alone is insufficient.
2095        let dispatch_key = format!("{}/{}", repo, task.id.0);
2096        // Atomically claim the dispatch slot.  DashMap lets us distinguish:
2097        //   - same task already in-flight (expected): log debug and skip.
2098        //   - different task holds the same key (should never happen): log warning and skip.
2099        // A plain DashSet::insert would return false for both cases without distinction.
2100        {
2101            use dashmap::mapref::entry::Entry;
2102            match dispatching.entry(dispatch_key.clone()) {
2103                Entry::Occupied(existing) => {
2104                    let existing_id = existing.get().clone();
2105                    drop(existing); // release shard lock before logging
2106                    if existing_id == task.id.0 {
2107                        tracing::debug!(
2108                            task_id = task.id.0,
2109                            "task already dispatching, skipping duplicate"
2110                        );
2111                    } else {
2112                        tracing::warn!(
2113                            task_id = task.id.0,
2114                            existing_task_id = existing_id,
2115                            dispatch_key,
2116                            "dispatch key collision: unexpected task already holds this key"
2117                        );
2118                    }
2119                    continue;
2120                }
2121                Entry::Vacant(slot) => {
2122                    slot.insert(task.id.0.clone());
2123                    // Promoted from debug! to info! (issue #3526): paired with the
2124                    // "dispatch guard released" line in dispatch_guard.rs, this is the
2125                    // only direct evidence of how long a dispatch guard was actually held
2126                    // for a given key. Both were debug!, silently dropped under the
2127                    // default "orch=info" filter (main.rs) — see the reclaim-skip log
2128                    // promotions above for the full rationale.
2129                    tracing::info!(task_id = task.id.0, dispatch_key, "dispatch guard claimed");
2130                }
2131            }
2132        }
2133        // RAII guard — removes dispatch_key on drop even if the spawned task panics.
2134        let dispatch_guard = DispatchGuard::new(dispatching.clone(), dispatch_key.clone());
2135        let task_id = task.id.0.clone();
2136
2137        // Load routing result from store (stored during Phase 3a). Loaded before the
2138        // tmux/semaphore/in_progress steps so a stale cooldown can bail out cheaply.
2139        let route_result = match get_route_result(store, repo, &task_id).await {
2140            Ok(r) => Some(r),
2141            Err(RouteResultError::NoAgent { .. }) => {
2142                tracing::warn!(
2143                    task_id,
2144                    "routed task missing agent — resetting to new for re-routing (#1604)"
2145                );
2146                if let Err(e2) = task_manager.update_task_status(&task.id, Status::New).await {
2147                    tracing::error!(task_id, error = %e2, "failed to reset task to new");
2148                }
2149                continue; // dispatch_guard drops here, removing the key
2150            }
2151            Err(e) => {
2152                tracing::warn!(task_id, error = %e, "get_route_result failed — dispatching without route info");
2153                None
2154            }
2155        };
2156
2157        // Re-check the assigned agent/model against the cooldown store: routing and
2158        // dispatch are decoupled by ticks, and a failure recorded after this task was
2159        // routed but before it dispatched would otherwise burn a full run on an
2160        // already-cooled agent/model. Leave the task in Routed so the next tick's
2161        // dispatch pass re-evaluates it — no new per-task defer state (see #3243).
2162        if let Some(route) = route_result.as_ref() {
2163            let agent_cooled = is_agent_in_cooldown(&route.agent);
2164            let model_cooled = route
2165                .model
2166                .as_deref()
2167                .is_some_and(|model| is_model_in_cooldown(&route.agent, model));
2168            if agent_cooled || model_cooled {
2169                tracing::warn!(
2170                    task_id,
2171                    agent = route.agent,
2172                    model = route.model.as_deref().unwrap_or(""),
2173                    "routed agent/model entered cooldown after routing — deferring dispatch to next tick"
2174                );
2175                continue; // dispatch_guard drops here, removing the key
2176            }
2177        }
2178
2179        // Check if already running (has active session)
2180        let session_name = tmux.session_name(repo, &task_id);
2181        if tmux
2182            .session_blocks_dispatch_from_map(&session_name, session_map)
2183            .await
2184        {
2185            tracing::warn!(
2186                task_id,
2187                session_name,
2188                "task has existing tmux session, skipping dispatch"
2189            );
2190            continue; // dispatch_guard drops here, removing the key
2191        }
2192
2193        // Try to acquire a slot
2194        let permit = match semaphore.clone().try_acquire_owned() {
2195            Ok(p) => p,
2196            Err(_) => {
2197                tracing::debug!("all parallel slots busy, skipping remaining tasks");
2198                break; // dispatch_guard drops here, removing the key
2199            }
2200        };
2201
2202        // Mark in_progress BEFORE spawning to prevent double dispatch.
2203        let set_in_progress_result = task_manager
2204            .update_task_status(&task.id, Status::InProgress)
2205            .await;
2206        if let Err(e) = set_in_progress_result {
2207            tracing::error!(task_id, ?e, "failed to set in_progress, skipping dispatch");
2208            drop(permit);
2209            continue; // dispatch_guard drops here, removing the key
2210        }
2211        tracing::info!(task_id, "dispatching task");
2212
2213        // Register session for capture
2214        capture
2215            .register_session(repo, &task_id, &session_name)
2216            .await;
2217
2218        // Dispatch task
2219        let runner = runner.clone();
2220        let backend = backend.clone();
2221        let tmux = tmux.clone();
2222        let capture = capture.clone();
2223        let task_id_for_cleanup = task_id.clone();
2224        let task_owned = task;
2225        let weight_tx = weight_tx.clone();
2226        let repo_owned = repo.to_string();
2227        let task_manager_for_spawn = task_manager.clone();
2228        let store_for_spawn = store.clone();
2229
2230        let repo_ctx = repo_owned.clone();
2231        tokio::spawn(REPO_CONTEXT.scope(repo_ctx, async move {
2232            let _dispatch_guard = dispatch_guard; // released on drop (normal or panic)
2233            let dispatch_start = std::time::Instant::now();
2234            match runner
2235                .run_with_context(&task_owned, &backend, &tmux, route_result.as_ref())
2236                .await
2237            {
2238                Ok(signal) => {
2239                    tracing::info!(task_id, "task runner completed");
2240
2241                    let duration = dispatch_start.elapsed().as_secs_f64();
2242
2243                    // Derive a display status from the weight signal.
2244                    // When review agent is enabled, successful tasks go through
2245                    // review before being marked done — UNLESS the response
2246                    // handler already marked the task done (no-op tasks with
2247                    // no PR and no commits should not enter the review cycle).
2248                    let enable_review = config::get("workflow.enable_review_agent")
2249                        .map(|v| v != "false")
2250                        .unwrap_or(true);
2251                    let display_status = match &signal {
2252                        WeightSignal::Success { .. } if enable_review => {
2253                            // Check if response handler already set done (no-op task).
2254                            // The task_manager.update_task_status() was already called
2255                            // by response_handler, so read the actual status from store.
2256                            let has_pr = match store_for_spawn
2257                                .resolve_task_id(&repo_owned, &task_id)
2258                                .await
2259                            {
2260                                Ok(Some(store_id)) => match store_for_spawn
2261                                    .get(store_id)
2262                                    .await
2263                                {
2264                                    Ok(t) => t.pr_number.is_some(),
2265                                    Err(e) => {
2266                                        tracing::warn!(task_id, err = %e, "store.get() failed — defaulting to needs_review");
2267                                        true
2268                                    }
2269                                },
2270                                Ok(None) => false,
2271                                Err(e) => {
2272                                    tracing::warn!(task_id, err = %e, "failed to check PR status — defaulting to needs_review");
2273                                    true
2274                                }
2275                            };
2276                            if has_pr {
2277                                "needs_review"
2278                            } else {
2279                                "done"
2280                            }
2281                        }
2282                        WeightSignal::Success { .. } => "done",
2283                        WeightSignal::RateLimited { .. } => "new",
2284                        WeightSignal::Blocked => "blocked",
2285                        // Rerouted: silence detection or other non-rate-limit reset already
2286                        // moved the task to "new". Stay in "new" — never convert to
2287                        // needs_review, which would trigger the no-code shortcut and falsely
2288                        // close the task as done without a successful agent run.
2289                        WeightSignal::Rerouted => "new",
2290                        // None means the runner guard skipped the task (e.g. tmux session
2291                        // already exists) or the task completed with an unrecognised status.
2292                        // Route to needs_review only when the review agent is enabled;
2293                        // otherwise mark done so the task is not permanently stuck.
2294                        WeightSignal::None if enable_review => "needs_review",
2295                        WeightSignal::None => "done",
2296                    };
2297
2298                    // Send weight signal back to the router
2299                    let _ = weight_tx.send(signal).await;
2300
2301                    // All tasks: if needs_review, trigger the review agent.
2302                    // Status updates go through task_manager so internal tasks
2303                    // hit SQLite while external tasks hit GitHub labels.
2304                    if display_status == "needs_review" {
2305                        // Transition to NeedsReview — emits event so the review subscriber
2306                        // fires with the correct duration before the review agent starts.
2307                        if let Err(e) = task_manager_for_spawn
2308                            .update_task_status_with_duration(
2309                                &ExternalId(task_id.clone()),
2310                                Status::NeedsReview,
2311                                Some(duration),
2312                            )
2313                            .await
2314                        {
2315                            tracing::error!(task_id, err = %e, "update_task_status(NeedsReview) failed — task may be stuck");
2316                        }
2317                        // Review agent dispatch is handled by the event-driven subscriber.
2318                    } else {
2319                        // done, blocked, or new (rate-limited): update status directly.
2320                        let final_status = match display_status {
2321                            "done" => Status::Done,
2322                            "blocked" => Status::Blocked,
2323                            _ => Status::New,
2324                        };
2325                        if let Err(e) = task_manager_for_spawn
2326                            .update_task_status_with_duration(
2327                                &ExternalId(task_id.clone()),
2328                                final_status,
2329                                Some(duration),
2330                            )
2331                            .await
2332                        {
2333                            tracing::warn!(task_id, ?e, "failed to update task status after completion");
2334                        }
2335                    }
2336                }
2337                Err(e) => {
2338                    tracing::error!(task_id, ?e, "task runner failed");
2339                    let duration = dispatch_start.elapsed().as_secs_f64();
2340                    if is_internal_id(&task_id) {
2341                        // Internal tasks have no GitHub issue to comment on.
2342                        if let Err(ue) = task_manager_for_spawn
2343                            .update_task_status_with_duration(
2344                                &ExternalId(task_id.clone()),
2345                                Status::NeedsReview,
2346                                Some(duration),
2347                            )
2348                            .await
2349                        {
2350                            tracing::error!(task_id, err = %ue, "update_task_status(NeedsReview) failed — task may be stuck");
2351                        }
2352                    } else {
2353                        // Post comment (best-effort)
2354                        if let Err(comment_err) = backend
2355                            .post_comment(
2356                                &ExternalId(task_id.clone()),
2357                                &format!(
2358                                    "[{}] error: task runner failed: {e}{}",
2359                                    chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
2360                                    crate::engine::orch_footer()
2361                                ),
2362                            )
2363                            .await
2364                        {
2365                            tracing::warn!(
2366                                task_id,
2367                                ?comment_err,
2368                                "failed to post error comment to GitHub"
2369                            );
2370                        }
2371                        // Update status immediately to NeedsReview so external tasks
2372                        // don't remain stuck in InProgress until the no-session timeout.
2373                        // The notify subscriber will send the channel notification with duration.
2374                        if let Err(ue) = task_manager_for_spawn
2375                            .update_task_status_with_duration(
2376                                &ExternalId(task_id.clone()),
2377                                Status::NeedsReview,
2378                                Some(duration),
2379                            )
2380                            .await
2381                        {
2382                            tracing::error!(task_id, err = %ue, "update_task_status(NeedsReview) failed — task may be stuck");
2383                        }
2384                    }
2385                }
2386            }
2387
2388            // Unregister session from capture
2389            capture.unregister_session(&repo_owned, &task_id_for_cleanup).await;
2390
2391            // Release the semaphore permit
2392            drop(permit);
2393            // _dispatch_guard drops here, removing the key from the dispatching set.
2394        }));
2395    }
2396    Ok(())
2397}
2398
2399fn skips_agent(labels: &[String]) -> bool {
2400    labels
2401        .iter()
2402        .any(|label| label == "no-agent" || label == "owner:human")
2403}
2404
2405/// Filter routed tasks down to those belonging to a repo outside `active_repos`,
2406/// drop `no-agent` and `owner:human` tasks, and group the rest by repo.
2407///
2408/// Pure/side-effect-free so it can be unit tested without touching real tmux
2409/// sessions or spawning a runner — see `dispatch_routed_tasks_global`, which is
2410/// the only caller and does the actual (side-effectful) dispatch.
2411fn group_orphaned_routed_tasks(
2412    all_routed: Vec<crate::store::Task>,
2413    active_repos: &std::collections::HashSet<String>,
2414) -> std::collections::HashMap<String, Vec<crate::store::Task>> {
2415    let mut by_repo: std::collections::HashMap<String, Vec<crate::store::Task>> =
2416        std::collections::HashMap::new();
2417    for task in all_routed
2418        .into_iter()
2419        .filter(|t| !active_repos.contains(&t.repo))
2420        .filter(|t| !skips_agent(&t.labels))
2421    {
2422        by_repo.entry(task.repo.clone()).or_default().push(task);
2423    }
2424    by_repo
2425}
2426
2427/// Global dispatch sweep: dispatch status:routed tasks from inactive or removed repos.
2428///
2429/// `tick_dispatch_tasks` is scoped to the active repo per-tick (its `task_manager`
2430/// only queries that repo's tasks) and never processes tasks whose `repo` field
2431/// differs from the current project. When a project is removed from the orch
2432/// config, tasks that reach `routed` for it — including ones routed by the
2433/// `route_new_tasks_global` sweep — have no code path left to dispatch them, so
2434/// they stay `routed` forever. This sweep covers all repos so orphaned routed
2435/// tasks are eventually dispatched.
2436///
2437/// `active_repos` must be the full set of currently-configured project repos,
2438/// not just the repo of the tick that called this. `tick()` runs once per
2439/// active project; passing only that project's own repo would make every
2440/// other active repo's routed tasks look "orphaned" too and dispatch them a
2441/// second time through this fallback path, racing the real per-repo dispatch.
2442///
2443/// `TaskRunner::resolve_project_dir` already falls back to the deterministic
2444/// `~/.orch/projects/<owner>/<repo>.git` bare-clone convention when a repo is
2445/// not listed in the active config, so dispatch works without a `ProjectEngine`.
2446#[allow(clippy::too_many_arguments)]
2447pub(crate) async fn dispatch_routed_tasks_global(
2448    tmux: &Arc<TmuxManager>,
2449    capture: &Arc<CaptureService>,
2450    semaphore: &Arc<Semaphore>,
2451    weight_tx: &mpsc::Sender<WeightSignal>,
2452    dispatch_mode: DispatchMode,
2453    dispatching: &Arc<DashMap<String, String>>,
2454    store: &Arc<TaskStore>,
2455    session_map: &std::collections::HashMap<String, bool>,
2456    active_repos: &std::collections::HashSet<String>,
2457) -> anyhow::Result<()> {
2458    use crate::store::TaskStatus;
2459
2460    let all_routed = store.list_all_by_status_global(TaskStatus::Routed).await?;
2461    let by_repo = group_orphaned_routed_tasks(all_routed, active_repos);
2462
2463    if by_repo.is_empty() {
2464        return Ok(());
2465    }
2466
2467    tracing::info!(
2468        repos = by_repo.len(),
2469        "global dispatch sweep: found routed tasks from inactive repos"
2470    );
2471
2472    for (repo, tasks) in by_repo {
2473        let backend: Arc<dyn ExternalBackend> = match crate::backends::github::GitHubBackend::new(
2474            repo.clone(),
2475        ) {
2476            Ok(b) => Arc::new(b),
2477            Err(e) => {
2478                tracing::warn!(repo = %repo, err = %e, "global sweep: failed to construct backend, skipping repo");
2479                continue;
2480            }
2481        };
2482        let task_manager = Arc::new(TaskManager::with_store(
2483            backend.clone(),
2484            store.clone(),
2485            repo.clone(),
2486        ));
2487        let runner = Arc::new(TaskRunner::new(repo.clone()).with_store(store.clone()));
2488        let dispatchable: Vec<ExternalTask> = tasks
2489            .iter()
2490            .map(crate::engine::tasks::store_task_to_external)
2491            .collect();
2492
2493        if let Err(e) = dispatch_tasks_for_repo(
2494            &backend,
2495            tmux,
2496            &repo,
2497            &runner,
2498            capture,
2499            semaphore,
2500            &task_manager,
2501            weight_tx,
2502            dispatch_mode,
2503            dispatching,
2504            store,
2505            session_map,
2506            dispatchable,
2507        )
2508        .await
2509        {
2510            tracing::warn!(repo = %repo, err = %e, "global sweep: dispatch failed for repo");
2511        }
2512    }
2513
2514    Ok(())
2515}
2516
2517/// Phase 4 of tick: unblock parent tasks whose sub-issues are all done.
2518pub(crate) async fn tick_unblock_parents(
2519    backend: &Arc<dyn ExternalBackend>,
2520    task_manager: &Arc<TaskManager>,
2521    store: &Arc<TaskStore>,
2522    repo: &str,
2523) -> anyhow::Result<()> {
2524    let blocked = task_manager.list_all_by_status(Status::Blocked).await?;
2525
2526    // Pre-filter: skip tasks that are blocked for a known reason (e.g. CI failure,
2527    // escalation). These are not waiting on children, so there is no point making
2528    // a GraphQL call for them.
2529    let store_blocked = store
2530        .list_by_status(repo, crate::store::TaskStatus::Blocked)
2531        .await?;
2532    let has_block_reason: std::collections::HashSet<String> = store_blocked
2533        .iter()
2534        .filter(|t| t.block_reason.is_some())
2535        .filter_map(|t| t.external_id.clone())
2536        .collect();
2537
2538    // Spawn a concurrent task for each candidate blocked task.
2539    // Previously these ran sequentially; get_sub_issues is a GraphQL call taking 1-3 s each,
2540    // and with 4 blocked tasks that was 4-12 s of sequential latency every tick.
2541    let handles: Vec<_> = blocked
2542        .into_iter()
2543        .filter(|task| {
2544            if has_block_reason.contains(&task.id.0) {
2545                tracing::debug!(
2546                    task_id = task.id.0,
2547                    "skipping blocked task with known block_reason"
2548                );
2549                false
2550            } else {
2551                true
2552            }
2553        })
2554        .map(|task| {
2555            let backend_clone = Arc::clone(backend);
2556            let task_manager_clone = Arc::clone(task_manager);
2557            let store_clone = Arc::clone(store);
2558            let repo_owned = repo.to_string();
2559            tokio::spawn(async move {
2560                let task_id = &task.id;
2561
2562                let mut children: Vec<ExternalId> = Vec::new();
2563                let mut child_statuses: std::collections::HashMap<
2564                    String,
2565                    crate::store::TaskStatus,
2566                > = std::collections::HashMap::new();
2567
2568                match store_clone.resolve_task_id(&repo_owned, &task_id.0).await {
2569                    Ok(Some(parent_store_id)) => {
2570                        match store_clone.list_children(&repo_owned, parent_store_id).await {
2571                            Ok(store_children) => {
2572                                for child in store_children {
2573                                    let Some(child_external_id) = child.external_id else {
2574                                        continue;
2575                                    };
2576                                    child_statuses.insert(child_external_id.clone(), child.status);
2577                                    children.push(ExternalId(child_external_id));
2578                                }
2579                            }
2580                            Err(e) => {
2581                                tracing::debug!(
2582                                    task_id = task_id.0,
2583                                    ?e,
2584                                    "failed to list store-linked children"
2585                                );
2586                            }
2587                        }
2588                    }
2589                    Ok(None) => {}
2590                    Err(e) => {
2591                        tracing::debug!(
2592                            task_id = task_id.0,
2593                            ?e,
2594                            "failed to resolve blocked task in store for child lookup"
2595                        );
2596                    }
2597                }
2598
2599                if !is_internal_id(&task_id.0) {
2600                    match backend_clone.get_sub_issues(task_id).await {
2601                        Ok(ids) => {
2602                            let mut seen: std::collections::HashSet<String> =
2603                                children.iter().map(|child| child.0.clone()).collect();
2604                            for id in ids {
2605                                if seen.insert(id.0.clone()) {
2606                                    children.push(id);
2607                                }
2608                            }
2609                        }
2610                        Err(e) => {
2611                            tracing::debug!(task_id = task_id.0, ?e, "failed to get sub-issues");
2612                        }
2613                    }
2614                }
2615
2616                // No children means nothing to wait on — skip (may be blocked for other reasons)
2617                if children.is_empty() {
2618                    return;
2619                }
2620
2621                // Check if every child is done using the local store (batched) and
2622                // fall back to the backend only for children not present in the store.
2623                let mut all_done = true;
2624                // Build owned strings for the batch lookup (needed for &str slices below)
2625                let child_ext_strs: Vec<String> =
2626                    children.iter().map(|c| c.0.clone()).collect();
2627                let child_exts: Vec<&str> =
2628                    child_ext_strs.iter().map(|s| s.as_str()).collect();
2629                // Query store for statuses of any children present locally
2630                if let Ok(map) = store_clone
2631                    .get_statuses_by_external_ids(&repo_owned, &child_exts)
2632                    .await
2633                {
2634                    for (k, v) in map.into_iter() {
2635                        child_statuses.insert(k, v);
2636                    }
2637                } else {
2638                    tracing::debug!(
2639                        task_id = task_id.0,
2640                        "failed to batch-query store for child statuses; falling back to per-child lookups"
2641                    );
2642                }
2643
2644                for child_id in &children {
2645                    // Check store first
2646                    if let Some(status) = child_statuses.get(&child_id.0) {
2647                        if *status != crate::store::TaskStatus::Done {
2648                            all_done = false;
2649                            break;
2650                        }
2651                        continue;
2652                    }
2653
2654                    // Not found in store — fall back to backend check
2655                    match backend_clone.get_task(child_id).await {
2656                        Ok(child) => {
2657                            if !child.labels.iter().any(|l| l == Status::Done.as_label()) {
2658                                all_done = false;
2659                                break;
2660                            }
2661                        }
2662                        Err(e) => {
2663                            tracing::debug!(task_id = task_id.0, child = %child_id.0, ?e, "failed to fetch child task from backend — treating as not done");
2664                            all_done = false;
2665                            break;
2666                        }
2667                    }
2668                }
2669
2670                if all_done {
2671                    tracing::info!(
2672                        task_id = task_id.0,
2673                        children = children.len(),
2674                        "all children done, unblocking parent"
2675                    );
2676                    if let Err(e) =
2677                        task_manager_clone.update_task_status(task_id, Status::New).await
2678                    {
2679                        tracing::warn!(task_id = task_id.0, ?e, "failed to unblock parent");
2680                    }
2681                }
2682            })
2683        })
2684        .collect();
2685
2686    for handle in handles {
2687        if let Err(e) = handle.await {
2688            tracing::warn!(?e, "phase 4 sub-issue check panicked");
2689        }
2690    }
2691    Ok(())
2692}
2693
2694/// Phase 5 of tick: run cron job matching and fire any due jobs.
2695pub(crate) async fn tick_job_scheduler(
2696    jobs_path: &std::path::Path,
2697    backend: &Arc<dyn ExternalBackend>,
2698    store: Option<&Arc<TaskStore>>,
2699    repo: &str,
2700    transport: Option<&Arc<crate::channels::transport::Transport>>,
2701) -> anyhow::Result<()> {
2702    jobs::tick(jobs_path, backend, store, repo, transport).await
2703}
2704
2705/// Core tick — runs every 10s.
2706///
2707/// Delegates to named phase functions in order:
2708/// 1. `tick_check_session_completions` — poll tmux for finished sessions
2709/// 2. `tick_recover_stuck_tasks`       — reset in_progress/in_review tasks with no active session
2710/// 3. `tick_route_tasks`               — route status:new tasks to an agent
2711/// 4. `tick_dispatch_tasks`            — spawn agents for status:routed tasks
2712/// 5. `tick_unblock_parents`           — unblock parents whose sub-issues are all done
2713/// 6. `tick_job_scheduler`             — fire due cron jobs
2714#[allow(clippy::too_many_arguments)]
2715pub(crate) async fn tick(
2716    backend: &Arc<dyn ExternalBackend>,
2717    tmux: &Arc<TmuxManager>,
2718    repo: &str,
2719    runner: &Arc<TaskRunner>,
2720    capture: &Arc<CaptureService>,
2721    semaphore: &Arc<Semaphore>,
2722    config: &EngineConfig,
2723    jobs_path: &std::path::Path,
2724    router: &mut Router,
2725    task_manager: &Arc<TaskManager>,
2726    weight_tx: &mpsc::Sender<WeightSignal>,
2727    dispatching: &Arc<DashMap<String, String>>,
2728    store: &Arc<TaskStore>,
2729    transport: Option<&Arc<crate::channels::transport::Transport>>,
2730    active_repos: &std::collections::HashSet<String>,
2731    auto_merge_in_flight: &Arc<DashSet<String>>,
2732    routing_quota: &mut RoutingTickQuota,
2733) -> anyhow::Result<()> {
2734    let _tick_span = tracing::info_span!("engine.tick").entered();
2735
2736    // Run startup cleanup once to kill stale sessions from previous runs.
2737    startup_cleanup(tmux).await;
2738
2739    if let Err(e) = tick_check_session_completions(tmux, repo, capture, store).await {
2740        tracing::warn!(err = %e, "session completion phase failed");
2741    }
2742    if let Err(e) =
2743        tick_detect_silent_agents(tmux, repo, capture, backend, task_manager, config, store).await
2744    {
2745        tracing::warn!(err = %e, "silent-agent detection phase failed");
2746    }
2747    // Fetch session map once and share between stuck-task recovery and dispatch to avoid
2748    // spawning two `tmux list-panes -a` subprocesses per tick.
2749    let session_map = tmux.batch_session_active().await;
2750    if let Err(e) = tick_recover_stuck_tasks(
2751        backend,
2752        tmux,
2753        repo,
2754        task_manager,
2755        config,
2756        store,
2757        &session_map,
2758        router,
2759        dispatching,
2760        auto_merge_in_flight,
2761    )
2762    .await
2763    {
2764        tracing::warn!(err = %e, "stuck-task recovery phase failed");
2765    }
2766    if let Err(err) = crate::engine::dependencies::reconcile_dependencies(
2767        backend.as_ref(),
2768        store,
2769        transport.map(|arc| arc.as_ref()),
2770    )
2771    .await
2772    {
2773        tracing::warn!(err = %err, "dependency reconcile failed");
2774    }
2775    if let Err(e) =
2776        tick_route_tasks(backend, task_manager, router, store, repo, routing_quota).await
2777    {
2778        tracing::warn!(err = %e, "routing phase failed");
2779    }
2780    // Global routing sweep for tasks from inactive or removed repos.
2781    // tick_route_tasks is scoped to the active repo; tasks from projects no longer
2782    // in config are never returned by list_routable and would stay in 'new' indefinitely.
2783    if let Err(e) = route_new_tasks_global(router, store, active_repos, routing_quota).await {
2784        tracing::warn!(err = %e, "global new-task routing sweep failed");
2785    }
2786    let dispatch_mode = dispatch_mode_from_router(router);
2787    if let Err(e) = tick_dispatch_tasks(
2788        backend,
2789        tmux,
2790        repo,
2791        runner,
2792        capture,
2793        semaphore,
2794        task_manager,
2795        weight_tx,
2796        dispatch_mode,
2797        dispatching,
2798        store,
2799        &session_map,
2800    )
2801    .await
2802    {
2803        tracing::warn!(err = %e, "dispatch phase failed");
2804    }
2805    // Global dispatch sweep for routed tasks from inactive or removed repos.
2806    // tick_dispatch_tasks is scoped to the active repo; tasks from projects no longer
2807    // in config are never returned by list_external_by_status and would stay in
2808    // 'routed' indefinitely (see #3413). Excludes the full active-repo set, not just
2809    // `repo`, so this per-project tick doesn't treat other active repos as orphaned.
2810    if let Err(e) = dispatch_routed_tasks_global(
2811        tmux,
2812        capture,
2813        semaphore,
2814        weight_tx,
2815        dispatch_mode,
2816        dispatching,
2817        store,
2818        &session_map,
2819        active_repos,
2820    )
2821    .await
2822    {
2823        tracing::warn!(err = %e, "global routed-task dispatch sweep failed");
2824    }
2825    if let Err(e) = tick_unblock_parents(backend, task_manager, store, repo).await {
2826        tracing::warn!(err = %e, "parent-unblock phase failed");
2827    }
2828    if let Err(e) = tick_job_scheduler(jobs_path, backend, Some(store), repo, transport).await {
2829        tracing::error!(?e, "job scheduler tick failed");
2830    }
2831    Ok(())
2832}
2833
2834#[cfg(test)]
2835mod tests {
2836    use super::*;
2837    use crate::backends::{ExternalId, ExternalTask, Mention, Status};
2838    use crate::channels::transport::Transport;
2839    use crate::engine::tasks::TaskManager;
2840    use crate::store::TaskStore;
2841    use crate::tmux::{SessionGuard, TmuxManager};
2842    use async_trait::async_trait;
2843    use std::sync::{Arc, Mutex};
2844
2845    #[test]
2846    fn skips_agent_for_human_and_no_agent_labels() {
2847        assert!(skips_agent(&["owner:human".to_string()]));
2848        assert!(skips_agent(&["no-agent".to_string()]));
2849        assert!(!skips_agent(&["status:new".to_string()]));
2850    }
2851
2852    /// A router reporting one available agent with no healthy "review" model, so the
2853    /// global NeedsReview refire sweep inside `tick_recover_stuck_tasks` treats every
2854    /// review agent as cooled and skips — keeping it a no-op for tests unrelated to it.
2855    fn cooled_review_router_for_test() -> Router {
2856        Router::new_for_test(
2857            crate::engine::router::RouterConfig::default(),
2858            vec!["claude".to_string()],
2859        )
2860    }
2861
2862    // ── minimal mock backend ─────────────────────────────────────────────────
2863
2864    /// Configurable mock backend for tick tests.
2865    struct MockBackend {
2866        /// Tasks returned by `list_by_status(Blocked)`.
2867        blocked_tasks: Vec<ExternalTask>,
2868        /// Sub-issues map: task_id → list of child ExternalIds.
2869        sub_issues: std::collections::HashMap<String, Vec<ExternalId>>,
2870        /// Tasks returned by `get_task`. Keyed by id.0.
2871        tasks_by_id: std::collections::HashMap<String, ExternalTask>,
2872        /// Recorded `get_sub_issues` calls.
2873        get_sub_issues_calls: Arc<Mutex<Vec<String>>>,
2874        /// Recorded `update_status` calls.
2875        status_updates: Arc<Mutex<Vec<(String, Status)>>>,
2876        /// Recorded `remove_label` calls.
2877        removed_labels: Arc<Mutex<Vec<(String, String)>>>,
2878        /// Recorded `post_comment` calls.
2879        posted_comments: Arc<Mutex<Vec<(String, String)>>>,
2880    }
2881
2882    impl MockBackend {
2883        fn new() -> Self {
2884            Self {
2885                blocked_tasks: vec![],
2886                sub_issues: Default::default(),
2887                tasks_by_id: Default::default(),
2888                get_sub_issues_calls: Arc::new(Mutex::new(vec![])),
2889                status_updates: Arc::new(Mutex::new(vec![])),
2890                removed_labels: Arc::new(Mutex::new(vec![])),
2891                posted_comments: Arc::new(Mutex::new(vec![])),
2892            }
2893        }
2894    }
2895
2896    fn make_task(id: &str, labels: &[&str]) -> ExternalTask {
2897        ExternalTask {
2898            id: ExternalId(id.to_string()),
2899            title: format!("Task {id}"),
2900            body: "".to_string(),
2901            state: "open".to_string(),
2902            labels: labels.iter().map(|s| s.to_string()).collect(),
2903            author: "bot".to_string(),
2904            created_at: "2026-01-01T00:00:00Z".to_string(),
2905            updated_at: "2026-01-01T00:00:00Z".to_string(),
2906            url: format!("https://github.com/test/test/issues/{id}"),
2907        }
2908    }
2909
2910    #[async_trait]
2911    impl ExternalBackend for MockBackend {
2912        fn name(&self) -> &str {
2913            "mock"
2914        }
2915        async fn create_task(
2916            &self,
2917            _title: &str,
2918            _body: &str,
2919            _labels: &[String],
2920        ) -> anyhow::Result<ExternalId> {
2921            Ok(ExternalId("new".to_string()))
2922        }
2923        async fn get_task(&self, id: &ExternalId) -> anyhow::Result<ExternalTask> {
2924            self.tasks_by_id
2925                .get(&id.0)
2926                .cloned()
2927                .ok_or_else(|| anyhow::anyhow!("task not found: {}", id.0))
2928        }
2929        async fn list_by_status(&self, status: Status) -> anyhow::Result<Vec<ExternalTask>> {
2930            if status == Status::Blocked {
2931                Ok(self.blocked_tasks.clone())
2932            } else {
2933                Ok(vec![])
2934            }
2935        }
2936        async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
2937            Ok(vec![])
2938        }
2939        async fn post_comment(&self, _id: &ExternalId, _body: &str) -> anyhow::Result<()> {
2940            self.posted_comments
2941                .lock()
2942                .unwrap()
2943                .push((_id.0.clone(), _body.to_string()));
2944            Ok(())
2945        }
2946        async fn set_labels(&self, _id: &ExternalId, _labels: &[String]) -> anyhow::Result<()> {
2947            Ok(())
2948        }
2949        async fn remove_label(&self, _id: &ExternalId, _label: &str) -> anyhow::Result<()> {
2950            self.removed_labels
2951                .lock()
2952                .unwrap()
2953                .push((_id.0.clone(), _label.to_string()));
2954            Ok(())
2955        }
2956        async fn get_sub_issues(&self, id: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
2957            self.get_sub_issues_calls.lock().unwrap().push(id.0.clone());
2958            Ok(self.sub_issues.get(&id.0).cloned().unwrap_or_default())
2959        }
2960        async fn create_sub_task(
2961            &self,
2962            _parent: &ExternalId,
2963            _title: &str,
2964            _body: &str,
2965            _labels: &[String],
2966        ) -> anyhow::Result<ExternalId> {
2967            Ok(ExternalId("child".to_string()))
2968        }
2969        async fn ensure_status_label(&self, _label: &str) -> anyhow::Result<()> {
2970            Ok(())
2971        }
2972        async fn has_open_issue_with_title(
2973            &self,
2974            _title: &str,
2975            _label: &str,
2976        ) -> anyhow::Result<bool> {
2977            Ok(false)
2978        }
2979        async fn health_check(&self) -> anyhow::Result<()> {
2980            Ok(())
2981        }
2982        async fn is_pr_merged(&self, _branch: &str) -> anyhow::Result<bool> {
2983            Ok(false)
2984        }
2985        async fn get_authenticated_user(&self) -> anyhow::Result<Option<String>> {
2986            Ok(Some("testbot".to_string()))
2987        }
2988        async fn get_mentions(&self, _since: &str) -> anyhow::Result<Vec<Mention>> {
2989            Ok(vec![])
2990        }
2991        async fn update_status(&self, id: &ExternalId, status: Status) -> anyhow::Result<()> {
2992            self.status_updates
2993                .lock()
2994                .unwrap()
2995                .push((id.0.clone(), status));
2996            Ok(())
2997        }
2998    }
2999
3000    fn make_task_manager(backend: Arc<dyn ExternalBackend>) -> Arc<TaskManager> {
3001        Arc::new(TaskManager::new(backend))
3002    }
3003
3004    // ── tick_unblock_parents ─────────────────────────────────────────────────
3005
3006    #[tokio::test]
3007    async fn unblock_parents_unblocks_when_all_children_done() {
3008        let mut mock = MockBackend::new();
3009
3010        // Blocked parent with two done children
3011        let parent = make_task("10", &["status:blocked"]);
3012        mock.blocked_tasks.push(parent.clone());
3013        mock.sub_issues.insert(
3014            "10".to_string(),
3015            vec![ExternalId("11".to_string()), ExternalId("12".to_string())],
3016        );
3017        mock.tasks_by_id
3018            .insert("11".to_string(), make_task("11", &["status:done"]));
3019        mock.tasks_by_id
3020            .insert("12".to_string(), make_task("12", &["status:done"]));
3021
3022        let status_updates = mock.status_updates.clone();
3023        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3024        let task_manager = make_task_manager(backend.clone());
3025        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3026
3027        tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3028            .await
3029            .unwrap();
3030
3031        let updates = status_updates.lock().unwrap();
3032        assert_eq!(updates.len(), 1, "parent should be unblocked");
3033        assert_eq!(updates[0], ("10".to_string(), Status::New));
3034    }
3035
3036    #[tokio::test]
3037    async fn unblock_parents_skips_when_child_not_done() {
3038        let mut mock = MockBackend::new();
3039
3040        let parent = make_task("20", &["status:blocked"]);
3041        mock.blocked_tasks.push(parent.clone());
3042        mock.sub_issues.insert(
3043            "20".to_string(),
3044            vec![ExternalId("21".to_string()), ExternalId("22".to_string())],
3045        );
3046        // Child 21 is done, child 22 is still in_progress
3047        mock.tasks_by_id
3048            .insert("21".to_string(), make_task("21", &["status:done"]));
3049        mock.tasks_by_id
3050            .insert("22".to_string(), make_task("22", &["status:in_progress"]));
3051
3052        let status_updates = mock.status_updates.clone();
3053        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3054        let task_manager = make_task_manager(backend.clone());
3055        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3056
3057        tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3058            .await
3059            .unwrap();
3060
3061        let updates = status_updates.lock().unwrap();
3062        assert!(
3063            updates.is_empty(),
3064            "should not unblock when a child is still running"
3065        );
3066    }
3067
3068    #[tokio::test]
3069    async fn unblock_parents_skips_task_with_no_children() {
3070        let mut mock = MockBackend::new();
3071
3072        // Blocked task with no sub-issues (blocked for a different reason)
3073        let parent = make_task("30", &["status:blocked"]);
3074        mock.blocked_tasks.push(parent.clone());
3075        // No entry in sub_issues → get_sub_issues returns []
3076
3077        let status_updates = mock.status_updates.clone();
3078        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3079        let task_manager = make_task_manager(backend.clone());
3080        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3081
3082        tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3083            .await
3084            .unwrap();
3085
3086        let updates = status_updates.lock().unwrap();
3087        assert!(
3088            updates.is_empty(),
3089            "should not unblock tasks with no sub-issues"
3090        );
3091    }
3092
3093    #[tokio::test]
3094    async fn unblock_parents_skips_task_with_block_reason_in_store() {
3095        // A blocked task that has a known block_reason in the store should never
3096        // trigger a get_sub_issues GraphQL call — it is not waiting on children.
3097        let mut mock = MockBackend::new();
3098
3099        // Blocked parent in the backend list
3100        let parent = make_task("60", &["status:blocked"]);
3101        mock.blocked_tasks.push(parent.clone());
3102        // Give it children in the backend so that, if the skip logic is wrong,
3103        // the parent would get unblocked.
3104        mock.sub_issues
3105            .insert("60".to_string(), vec![ExternalId("61".to_string())]);
3106        mock.tasks_by_id
3107            .insert("61".to_string(), make_task("61", &["status:done"]));
3108
3109        let status_updates = mock.status_updates.clone();
3110        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3111        let task_manager = make_task_manager(backend.clone());
3112
3113        // Insert the blocked task into the store with a block_reason set.
3114        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3115        let task_id = store
3116            .upsert_external(&crate::store::UpsertExternal {
3117                repo: "test/repo",
3118                ext_id: "60",
3119                title: "Task 60",
3120                body: "",
3121                author: "bot",
3122                url: "https://github.com/test/test/issues/60",
3123                labels: &[],
3124                origin: "github",
3125            })
3126            .await
3127            .unwrap();
3128        store
3129            .update_status(task_id, crate::store::TaskStatus::Blocked)
3130            .await
3131            .unwrap();
3132        store
3133            .set_block_reason(task_id, Some("CI failure limit reached"))
3134            .await
3135            .unwrap();
3136
3137        tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3138            .await
3139            .unwrap();
3140
3141        let updates = status_updates.lock().unwrap();
3142        assert!(
3143            updates.is_empty(),
3144            "task with block_reason should be skipped — no API call, no unblock"
3145        );
3146    }
3147
3148    #[tokio::test]
3149    async fn unblock_parents_unblocks_internal_parent_from_store_children() {
3150        let mock = MockBackend::new();
3151        let get_sub_issues_calls = mock.get_sub_issues_calls.clone();
3152        let status_updates = mock.status_updates.clone();
3153        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3154        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3155        let task_manager = Arc::new(TaskManager::with_store(
3156            backend.clone(),
3157            store.clone(),
3158            "test/repo".to_string(),
3159        ));
3160
3161        let parent_id = store
3162            .create_internal("test/repo", "Parent", "", "job", "daily", None)
3163            .await
3164            .unwrap();
3165        store
3166            .update_status(parent_id, crate::store::TaskStatus::Blocked)
3167            .await
3168            .unwrap();
3169        let child_id = store
3170            .create(&crate::store::NewTask {
3171                external_id: Some("91".to_string()),
3172                repo: "test/repo".to_string(),
3173                origin: "github".to_string(),
3174                title: "Child".to_string(),
3175                body: String::new(),
3176                source: String::new(),
3177                source_id: String::new(),
3178                author: String::new(),
3179                url: String::new(),
3180                labels: vec![],
3181                parent_id: Some(parent_id),
3182            })
3183            .await
3184            .unwrap();
3185        store
3186            .update_status(child_id, crate::store::TaskStatus::Done)
3187            .await
3188            .unwrap();
3189
3190        tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3191            .await
3192            .unwrap();
3193
3194        let parent = store.get(parent_id).await.unwrap();
3195        assert_eq!(parent.status, crate::store::TaskStatus::New);
3196        assert!(
3197            status_updates.lock().unwrap().is_empty(),
3198            "internal parent should be updated in store without backend mirroring"
3199        );
3200        assert!(
3201            get_sub_issues_calls.lock().unwrap().is_empty(),
3202            "internal parent should not query backend sub-issues"
3203        );
3204    }
3205
3206    // ── tick_recover_stuck_tasks (InReview) ──────────────────────────────────
3207
3208    /// Set `updated_at` to a far-past date for a task in an in-memory store.
3209    #[cfg(test)]
3210    async fn set_task_updated_at_past(store: &TaskStore, task_id: i64) {
3211        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
3212            .bind(task_id)
3213            .execute(store.pool())
3214            .await
3215            .unwrap();
3216    }
3217
3218    #[tokio::test]
3219    async fn recover_stuck_tasks_resets_external_in_review_to_needs_review() {
3220        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3221        let mock = MockBackend::new();
3222        let status_updates = mock.status_updates.clone();
3223        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3224        let task_manager = Arc::new(TaskManager::with_store(
3225            backend.clone(),
3226            store.clone(),
3227            "owner/repo".to_string(),
3228        ));
3229        let tmux = Arc::new(TmuxManager::new());
3230        let config = EngineConfig {
3231            no_session_stuck_timeout: 600,
3232            stuck_timeout: 1800,
3233            ..EngineConfig::default()
3234        };
3235
3236        // Insert an external task and transition it to InReview with an old updated_at
3237        let id = store
3238            .upsert_external(&crate::store::UpsertExternal {
3239                repo: "owner/repo",
3240                ext_id: "99",
3241                title: "Stuck InReview",
3242                body: "",
3243                author: "",
3244                url: "",
3245                labels: &[],
3246                origin: "github",
3247            })
3248            .await
3249            .unwrap();
3250        store
3251            .update_status(id, crate::store::TaskStatus::InReview)
3252            .await
3253            .unwrap();
3254        set_review_session_expected(&store, "owner/repo", "99", true).await;
3255        set_task_updated_at_past(&store, id).await;
3256
3257        tick_recover_stuck_tasks(
3258            &backend,
3259            &tmux,
3260            "owner/repo",
3261            &task_manager,
3262            &config,
3263            &store,
3264            &std::collections::HashMap::new(),
3265            &cooled_review_router_for_test(),
3266            &Arc::new(DashMap::new()),
3267            &Arc::new(DashSet::new()),
3268        )
3269        .await
3270        .unwrap();
3271
3272        let updates = status_updates.lock().unwrap();
3273        assert_eq!(updates.len(), 1, "stuck InReview task should be recovered");
3274        assert_eq!(updates[0].0, "99");
3275        assert_eq!(updates[0].1, Status::NeedsReview);
3276    }
3277
3278    #[tokio::test]
3279    async fn recover_stuck_tasks_skips_recent_in_review() {
3280        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3281        let mock = MockBackend::new();
3282        let status_updates = mock.status_updates.clone();
3283        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3284        let task_manager = Arc::new(TaskManager::with_store(
3285            backend.clone(),
3286            store.clone(),
3287            "owner/repo".to_string(),
3288        ));
3289        let tmux = Arc::new(TmuxManager::new());
3290        let config = EngineConfig {
3291            no_session_stuck_timeout: 600, // 10 minutes — recent task won't trigger
3292            stuck_timeout: 1800,
3293            ..EngineConfig::default()
3294        };
3295
3296        // Insert a recently-updated InReview task (updated_at = now)
3297        let id = store
3298            .upsert_external(&crate::store::UpsertExternal {
3299                repo: "owner/repo",
3300                ext_id: "100",
3301                title: "Recent InReview",
3302                body: "",
3303                author: "",
3304                url: "",
3305                labels: &[],
3306                origin: "github",
3307            })
3308            .await
3309            .unwrap();
3310        store
3311            .update_status(id, crate::store::TaskStatus::InReview)
3312            .await
3313            .unwrap();
3314        // updated_at stays at now — age < 600s → should NOT be recovered
3315
3316        tick_recover_stuck_tasks(
3317            &backend,
3318            &tmux,
3319            "owner/repo",
3320            &task_manager,
3321            &config,
3322            &store,
3323            &std::collections::HashMap::new(),
3324            &cooled_review_router_for_test(),
3325            &Arc::new(DashMap::new()),
3326            &Arc::new(DashSet::new()),
3327        )
3328        .await
3329        .unwrap();
3330
3331        let updates = status_updates.lock().unwrap();
3332        assert!(
3333            updates.is_empty(),
3334            "recent InReview task should not be recovered"
3335        );
3336    }
3337
3338    #[tokio::test]
3339    async fn recover_stuck_tasks_resets_internal_in_review_to_needs_review() {
3340        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3341        let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
3342        let task_manager = Arc::new(TaskManager::with_store(
3343            backend.clone(),
3344            store.clone(),
3345            "owner/repo".to_string(),
3346        ));
3347        let tmux = Arc::new(TmuxManager::new());
3348        let config = EngineConfig {
3349            no_session_stuck_timeout: 600,
3350            stuck_timeout: 1800,
3351            ..EngineConfig::default()
3352        };
3353
3354        // Create an internal task in InReview with old updated_at
3355        let id = store
3356            .create_internal("owner/repo", "Internal InReview", "", "cron", "1", None)
3357            .await
3358            .unwrap();
3359        store
3360            .update_status(id, crate::store::TaskStatus::InReview)
3361            .await
3362            .unwrap();
3363        set_review_session_expected(&store, "owner/repo", "internal:1", true).await;
3364        set_task_updated_at_past(&store, id).await;
3365
3366        tick_recover_stuck_tasks(
3367            &backend,
3368            &tmux,
3369            "owner/repo",
3370            &task_manager,
3371            &config,
3372            &store,
3373            &std::collections::HashMap::new(),
3374            &cooled_review_router_for_test(),
3375            &Arc::new(DashMap::new()),
3376            &Arc::new(DashSet::new()),
3377        )
3378        .await
3379        .unwrap();
3380
3381        let task = store.get(id).await.unwrap();
3382        assert_eq!(
3383            task.status,
3384            crate::store::TaskStatus::NeedsReview,
3385            "stuck internal InReview task should be reset to NeedsReview"
3386        );
3387    }
3388
3389    #[tokio::test]
3390    async fn recover_stuck_tasks_resets_cross_repo_external_in_progress_to_new() {
3391        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3392        let mock = MockBackend::new();
3393        let status_updates = mock.status_updates.clone();
3394        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3395        let task_manager = Arc::new(TaskManager::with_store(
3396            backend.clone(),
3397            store.clone(),
3398            "owner/repo".to_string(),
3399        ));
3400        let tmux = Arc::new(TmuxManager::new());
3401        let config = EngineConfig {
3402            no_session_stuck_timeout: 600,
3403            stuck_timeout: 1800,
3404            ..EngineConfig::default()
3405        };
3406
3407        // Cross-repo task should still be recovered even when current tick repo is owner/repo.
3408        let id = store
3409            .upsert_external(&crate::store::UpsertExternal {
3410                repo: "owner/other",
3411                ext_id: "777",
3412                title: "Cross-repo stuck task",
3413                body: "",
3414                author: "",
3415                url: "",
3416                labels: &[],
3417                origin: "github",
3418            })
3419            .await
3420            .unwrap();
3421        store
3422            .update_status(id, crate::store::TaskStatus::InProgress)
3423            .await
3424            .unwrap();
3425        set_task_updated_at_past(&store, id).await;
3426
3427        tick_recover_stuck_tasks(
3428            &backend,
3429            &tmux,
3430            "owner/repo",
3431            &task_manager,
3432            &config,
3433            &store,
3434            &std::collections::HashMap::new(),
3435            &cooled_review_router_for_test(),
3436            &Arc::new(DashMap::new()),
3437            &Arc::new(DashSet::new()),
3438        )
3439        .await
3440        .unwrap();
3441
3442        let task = store.get(id).await.unwrap();
3443        assert_eq!(
3444            task.status,
3445            crate::store::TaskStatus::New,
3446            "cross-repo stale in_progress task should be reset to New"
3447        );
3448        assert_eq!(task.route_attempts, 0, "route attempts should be cleared");
3449        assert!(task.agent.is_none(), "agent should be cleared");
3450        assert!(task.model.is_none(), "model should be cleared");
3451
3452        // Cross-repo global sweep is store-only and must not call backend status updates.
3453        assert!(
3454            status_updates.lock().unwrap().is_empty(),
3455            "cross-repo recovery should not call backend update_status"
3456        );
3457    }
3458
3459    #[tokio::test]
3460    async fn recover_stuck_tasks_finalizes_open_task_runs_rows() {
3461        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3462        let mock = MockBackend::new();
3463        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3464        let task_manager = Arc::new(TaskManager::with_store(
3465            backend.clone(),
3466            store.clone(),
3467            "owner/repo".to_string(),
3468        ));
3469        let tmux = Arc::new(TmuxManager::new());
3470        let config = EngineConfig {
3471            no_session_stuck_timeout: 600,
3472            stuck_timeout: 1800,
3473            ..EngineConfig::default()
3474        };
3475
3476        let id = store
3477            .create_internal("owner/repo", "Stuck with open run", "", "cron", "1", None)
3478            .await
3479            .unwrap();
3480        store
3481            .set_fields(
3482                id,
3483                &[
3484                    ("agent", serde_json::json!("claude")),
3485                    ("model", serde_json::json!("sonnet")),
3486                    ("route_attempts", serde_json::json!(1)),
3487                ],
3488            )
3489            .await
3490            .unwrap();
3491        store
3492            .update_status(id, crate::store::TaskStatus::InProgress)
3493            .await
3494            .unwrap();
3495        set_task_updated_at_past(&store, id).await;
3496
3497        // Start an agent run that will be orphaned by recovery
3498        store
3499            .start_run(&crate::store::StartRun {
3500                task_id: id,
3501                attempt: 1,
3502                run_type: "agent",
3503                agent: "claude",
3504                model: "sonnet",
3505                command: "claude --model sonnet",
3506                prompt: "fix it",
3507            })
3508            .await
3509            .unwrap();
3510
3511        tick_recover_stuck_tasks(
3512            &backend,
3513            &tmux,
3514            "owner/repo",
3515            &task_manager,
3516            &config,
3517            &store,
3518            &std::collections::HashMap::new(),
3519            &cooled_review_router_for_test(),
3520            &Arc::new(DashMap::new()),
3521            &Arc::new(DashSet::new()),
3522        )
3523        .await
3524        .unwrap();
3525
3526        let task = store.get(id).await.unwrap();
3527        assert_eq!(task.status, crate::store::TaskStatus::New);
3528
3529        let runs = store.get_runs(id).await.unwrap();
3530        assert_eq!(runs.len(), 1);
3531        assert_eq!(runs[0].outcome, "aborted");
3532        assert!(runs[0].completed_at.is_some());
3533        assert_eq!(
3534            runs[0].error,
3535            "stuck-task recovery: internal no session found"
3536        );
3537    }
3538
3539    /// Regression test for #3518: a completed agent run must not be discarded by the
3540    /// stuck-task reclaim just because its tmux session has already exited.
3541    ///
3542    /// In production, sessions are created without `remain-on-exit`, so once the agent
3543    /// process exits tmux removes the session immediately — there is no observable
3544    /// "dead pane" state for `tick_check_session_completions` to catch, and the runner's
3545    /// own `store_touch_updated_at` only fires after its own poll notices the exit (up to
3546    /// its 5s poll interval) plus post-processing. For a task whose total runtime already
3547    /// exceeds `no_session_stuck_timeout`, the very next tick that observes the (now
3548    /// absent) session would otherwise reclaim it instantly — discarding a successful run.
3549    ///
3550    /// The fix: `tick_recover_stuck_tasks` skips the no-session reclaim while the task's
3551    /// dispatch guard (the in-memory `dispatching` map keyed by "{repo}/{task_id}",
3552    /// populated for the entire duration of `runner.run_with_context`) is still held —
3553    /// i.e. while this process is still actively running the task, regardless of tmux
3554    /// session state.
3555    #[tokio::test]
3556    async fn recover_stuck_tasks_skips_reclaim_while_dispatch_guard_held() {
3557        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3558        let mock = MockBackend::new();
3559        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3560        let task_manager = Arc::new(TaskManager::with_store(
3561            backend.clone(),
3562            store.clone(),
3563            "owner/repo".to_string(),
3564        ));
3565        let tmux = Arc::new(TmuxManager::new());
3566        let config = EngineConfig {
3567            no_session_stuck_timeout: 600,
3568            stuck_timeout: 1800,
3569            ..EngineConfig::default()
3570        };
3571
3572        let id = store
3573            .create_internal(
3574                "owner/repo",
3575                "Long-running daily job",
3576                "",
3577                "cron",
3578                "1",
3579                None,
3580            )
3581            .await
3582            .unwrap();
3583        store
3584            .set_fields(
3585                id,
3586                &[
3587                    ("agent", serde_json::json!("claude")),
3588                    ("model", serde_json::json!("sonnet")),
3589                    ("route_attempts", serde_json::json!(1)),
3590                ],
3591            )
3592            .await
3593            .unwrap();
3594        store
3595            .update_status(id, crate::store::TaskStatus::InProgress)
3596            .await
3597            .unwrap();
3598        // Simulate a task that has already run well past no_session_stuck_timeout,
3599        // as a legitimately long agent run would.
3600        set_task_updated_at_past(&store, id).await;
3601
3602        // Simulate the runner still actively processing this task: its dispatch guard
3603        // is held even though the tmux session (empty session_map below) is already gone.
3604        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3605        dispatching.insert(
3606            format!("owner/repo/internal:{id}"),
3607            format!("internal:{id}"),
3608        );
3609
3610        tick_recover_stuck_tasks(
3611            &backend,
3612            &tmux,
3613            "owner/repo",
3614            &task_manager,
3615            &config,
3616            &store,
3617            &std::collections::HashMap::new(),
3618            &cooled_review_router_for_test(),
3619            &dispatching,
3620            &Arc::new(DashSet::new()),
3621        )
3622        .await
3623        .unwrap();
3624
3625        let task = store.get(id).await.unwrap();
3626        assert_eq!(
3627            task.status,
3628            crate::store::TaskStatus::InProgress,
3629            "task should not be reclaimed while its dispatch guard is still held"
3630        );
3631
3632        // No run should have been finalized/aborted either.
3633        let runs = store.get_runs(id).await.unwrap();
3634        assert!(runs.is_empty());
3635
3636        // Once the guard is released (runner finished), the very same conditions should
3637        // reclaim the task as before — the fix must not disable the safety net entirely.
3638        dispatching.clear();
3639        tick_recover_stuck_tasks(
3640            &backend,
3641            &tmux,
3642            "owner/repo",
3643            &task_manager,
3644            &config,
3645            &store,
3646            &std::collections::HashMap::new(),
3647            &cooled_review_router_for_test(),
3648            &dispatching,
3649            &Arc::new(DashSet::new()),
3650        )
3651        .await
3652        .unwrap();
3653        let task = store.get(id).await.unwrap();
3654        assert_eq!(
3655            task.status,
3656            crate::store::TaskStatus::New,
3657            "task should still be reclaimed once the dispatch guard is released"
3658        );
3659    }
3660
3661    /// Concurrency regression test for issue #3526: the reclaim race recurred twice
3662    /// (internal:156563 in #3523, internal:156673 in #3526) *after* the #3519/#3525
3663    /// fix landed, in production under the real multi-threaded tokio runtime. The
3664    /// existing `recover_stuck_tasks_skips_reclaim_while_dispatch_guard_held` test
3665    /// above only exercises sequential `.await` calls on a single-threaded test
3666    /// runtime (`#[tokio::test]` defaults to `flavor = "current_thread"`), which can
3667    /// never observe genuine cross-thread races in the shared `dispatching` DashMap.
3668    ///
3669    /// This test claims the guard exactly the way `dispatch_tasks_for_repo` does
3670    /// (atomic `DashMap::entry` claim, `DispatchGuard` RAII release), hands it to a
3671    /// spawned task running on a *different* worker thread that holds it until
3672    /// explicitly told to release, and hammers the real `tick_recover_stuck_tasks`
3673    /// from the main test task hundreds of times while the guard is deterministically
3674    /// still held. If there were a visibility gap between `DashMap::entry().insert()`
3675    /// on one thread and `DashMap::contains_key()` on another, this test would catch it.
3676    #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
3677    async fn recover_stuck_tasks_never_reclaims_while_guard_held_under_real_concurrency() {
3678        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3679        let mock = MockBackend::new();
3680        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3681        let task_manager = Arc::new(TaskManager::with_store(
3682            backend.clone(),
3683            store.clone(),
3684            "owner/repo".to_string(),
3685        ));
3686        let tmux = Arc::new(TmuxManager::new());
3687        let config = EngineConfig {
3688            no_session_stuck_timeout: 600,
3689            stuck_timeout: 1800,
3690            ..EngineConfig::default()
3691        };
3692
3693        let id = store
3694            .create_internal(
3695                "owner/repo",
3696                "Long-running daily job",
3697                "",
3698                "cron",
3699                "1",
3700                None,
3701            )
3702            .await
3703            .unwrap();
3704        store
3705            .set_fields(
3706                id,
3707                &[
3708                    ("agent", serde_json::json!("claude")),
3709                    ("model", serde_json::json!("sonnet")),
3710                    ("route_attempts", serde_json::json!(1)),
3711                ],
3712            )
3713            .await
3714            .unwrap();
3715        store
3716            .update_status(id, crate::store::TaskStatus::InProgress)
3717            .await
3718            .unwrap();
3719        set_task_updated_at_past(&store, id).await;
3720
3721        let dispatch_key = format!("owner/repo/internal:{id}");
3722        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3723
3724        // Claim the guard synchronously, on this task, exactly like
3725        // dispatch_tasks_for_repo does: the atomic entry-claim always happens before
3726        // tokio::spawn hands the guard to the background task. (Claiming inside the
3727        // spawned task itself would race the hammer loop below against tokio's own
3728        // scheduling of that task onto a worker thread — a test artifact, not the
3729        // real production ordering, which guarantees claim-before-spawn.)
3730        let claim_task_id = format!("internal:{id}");
3731        {
3732            use dashmap::mapref::entry::Entry;
3733            match dispatching.entry(dispatch_key.clone()) {
3734                Entry::Vacant(slot) => {
3735                    slot.insert(claim_task_id);
3736                }
3737                Entry::Occupied(_) => panic!("dispatch key already claimed"),
3738            }
3739        }
3740        let guard = DispatchGuard::new(Arc::clone(&dispatching), dispatch_key.clone());
3741
3742        // Hold the guard on a spawned task — a different worker thread under the
3743        // multi-thread runtime — mirroring the real runner holding it across genuine
3744        // async work (worktree setup, agent session, response handling) inside
3745        // `tokio::spawn(... run_with_context ...)`. It only drops the guard once
3746        // explicitly told to via `release_rx`, so its lifetime is fully controlled
3747        // by this test rather than raced against a timer — the release-signal
3748        // ordering was itself a source of false positives in an earlier version of
3749        // this test (a `sleep`-then-drop holder can legitimately release the guard
3750        // in the gap between the hammer loop's last check and its next iteration,
3751        // which is a correct outcome, not a bug, but made the assertion flaky).
3752        let (release_tx, release_rx) = tokio::sync::oneshot::channel::<()>();
3753        let holder = tokio::spawn(async move {
3754            let _ = release_rx.await;
3755            drop(guard);
3756        });
3757
3758        // Hammer the real reclaim path concurrently from the main task while the
3759        // guard is deterministically still held (the holder task is parked on
3760        // `release_rx` and cannot drop the guard until we signal it below). Each
3761        // iteration's `tick_recover_stuck_tasks` call may itself run on any worker
3762        // thread the tokio scheduler picks, independent of the thread that claimed
3763        // the guard — this is what actually exercises cross-thread DashMap
3764        // visibility, not wall-clock racing against a timer.
3765        const HAMMER_ITERATIONS: u32 = 300;
3766        for i in 0..HAMMER_ITERATIONS {
3767            tick_recover_stuck_tasks(
3768                &backend,
3769                &tmux,
3770                "owner/repo",
3771                &task_manager,
3772                &config,
3773                &store,
3774                &std::collections::HashMap::new(),
3775                &cooled_review_router_for_test(),
3776                &dispatching,
3777                &Arc::new(DashSet::new()),
3778            )
3779            .await
3780            .unwrap();
3781
3782            let task = store.get(id).await.unwrap();
3783            assert_eq!(
3784                task.status,
3785                crate::store::TaskStatus::InProgress,
3786                "task must never be reclaimed while the dispatch guard is deterministically \
3787                 still held, even under real cross-thread concurrency (dispatch_key={dispatch_key}, \
3788                 iteration={i}, contains_key_now={}, map_len_now={})",
3789                dispatching.contains_key(&dispatch_key),
3790                dispatching.len(),
3791            );
3792            tokio::task::yield_now().await;
3793        }
3794
3795        // Now release the guard and confirm the holder task actually completed.
3796        let _ = release_tx.send(());
3797        holder.await.unwrap();
3798
3799        // Guard released — the safety net must still reclaim as before.
3800        tick_recover_stuck_tasks(
3801            &backend,
3802            &tmux,
3803            "owner/repo",
3804            &task_manager,
3805            &config,
3806            &store,
3807            &std::collections::HashMap::new(),
3808            &cooled_review_router_for_test(),
3809            &dispatching,
3810            &Arc::new(DashSet::new()),
3811        )
3812        .await
3813        .unwrap();
3814        let task = store.get(id).await.unwrap();
3815        assert_eq!(
3816            task.status,
3817            crate::store::TaskStatus::New,
3818            "task should be reclaimed once the dispatch guard is released"
3819        );
3820    }
3821
3822    #[tokio::test]
3823    async fn recover_stuck_tasks_skips_external_in_review_waiting_for_human_review() {
3824        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3825        let mock = MockBackend::new();
3826        let status_updates = mock.status_updates.clone();
3827        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3828        let task_manager = Arc::new(TaskManager::with_store(
3829            backend.clone(),
3830            store.clone(),
3831            "owner/repo".to_string(),
3832        ));
3833        let tmux = Arc::new(TmuxManager::new());
3834        let config = EngineConfig {
3835            no_session_stuck_timeout: 600,
3836            stuck_timeout: 1800,
3837            ..EngineConfig::default()
3838        };
3839
3840        let id = store
3841            .upsert_external(&crate::store::UpsertExternal {
3842                repo: "owner/repo",
3843                ext_id: "101",
3844                title: "Waiting For Human Review",
3845                body: "",
3846                author: "",
3847                url: "",
3848                labels: &[],
3849                origin: "github",
3850            })
3851            .await
3852            .unwrap();
3853        store
3854            .update_status(id, crate::store::TaskStatus::InReview)
3855            .await
3856            .unwrap();
3857        set_task_updated_at_past(&store, id).await;
3858
3859        tick_recover_stuck_tasks(
3860            &backend,
3861            &tmux,
3862            "owner/repo",
3863            &task_manager,
3864            &config,
3865            &store,
3866            &std::collections::HashMap::new(),
3867            &cooled_review_router_for_test(),
3868            &Arc::new(DashMap::new()),
3869            &Arc::new(DashSet::new()),
3870        )
3871        .await
3872        .unwrap();
3873
3874        let updates = status_updates.lock().unwrap();
3875        assert!(
3876            updates.is_empty(),
3877            "InReview task waiting for human review should not be reset"
3878        );
3879    }
3880
3881    #[tokio::test]
3882    async fn unblock_parents_no_blocked_tasks() {
3883        let mock = MockBackend::new(); // no blocked tasks
3884        let status_updates = mock.status_updates.clone();
3885        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3886        let task_manager = make_task_manager(backend.clone());
3887        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3888
3889        tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3890            .await
3891            .unwrap();
3892
3893        assert!(status_updates.lock().unwrap().is_empty());
3894    }
3895
3896    #[tokio::test]
3897    async fn unblock_parents_handles_multiple_blocked_tasks() {
3898        let mut mock = MockBackend::new();
3899
3900        // Task 40: all children done → should unblock
3901        let t40 = make_task("40", &["status:blocked"]);
3902        mock.blocked_tasks.push(t40);
3903        mock.sub_issues
3904            .insert("40".to_string(), vec![ExternalId("41".to_string())]);
3905        mock.tasks_by_id
3906            .insert("41".to_string(), make_task("41", &["status:done"]));
3907
3908        // Task 50: child not done → should stay blocked
3909        let t50 = make_task("50", &["status:blocked"]);
3910        mock.blocked_tasks.push(t50);
3911        mock.sub_issues
3912            .insert("50".to_string(), vec![ExternalId("51".to_string())]);
3913        mock.tasks_by_id
3914            .insert("51".to_string(), make_task("51", &["status:in_progress"]));
3915
3916        let status_updates = mock.status_updates.clone();
3917        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3918        let task_manager = make_task_manager(backend.clone());
3919        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3920
3921        tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3922            .await
3923            .unwrap();
3924
3925        let updates = status_updates.lock().unwrap();
3926        assert_eq!(updates.len(), 1, "only one parent should be unblocked");
3927        assert_eq!(updates[0].0, "40", "task 40 should be unblocked");
3928        assert_eq!(updates[0].1, Status::New);
3929    }
3930
3931    #[tokio::test]
3932    async fn detect_silent_agents_skips_backend_for_internal_tasks() {
3933        let transport = Arc::new(Transport::new());
3934        let capture = Arc::new(CaptureService::new(transport));
3935        let mock = Arc::new(MockBackend::new());
3936        let backend: Arc<dyn ExternalBackend> = mock.clone();
3937        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3938        let task_manager = Arc::new(TaskManager::with_store(
3939            backend.clone(),
3940            store.clone(),
3941            "owner/repo".to_string(),
3942        ));
3943        let tmux = Arc::new(TmuxManager::new());
3944        let config = EngineConfig {
3945            silence_grace_period: 0,
3946            ..EngineConfig::default()
3947        };
3948
3949        let internal_id = store
3950            .create_internal("owner/repo", "Silent internal", "", "cron", "1", None)
3951            .await
3952            .unwrap();
3953        store
3954            .update_status(internal_id, crate::store::TaskStatus::InProgress)
3955            .await
3956            .unwrap();
3957
3958        let task_id = format!("internal:{internal_id}");
3959        capture
3960            .register_session("owner/repo", &task_id, "orch-test-internal")
3961            .await;
3962        tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
3963
3964        tick_detect_silent_agents(
3965            &tmux,
3966            "owner/repo",
3967            &capture,
3968            &backend,
3969            &task_manager,
3970            &config,
3971            &store,
3972        )
3973        .await
3974        .unwrap();
3975
3976        assert!(
3977            mock.removed_labels.lock().unwrap().is_empty(),
3978            "internal tasks should skip backend label removal"
3979        );
3980        assert!(
3981            mock.posted_comments.lock().unwrap().is_empty(),
3982            "internal tasks should skip backend comments"
3983        );
3984
3985        let task = store.get(internal_id).await.unwrap();
3986        // Silence detection now uses failover: if a fallback agent is found → Routed,
3987        // otherwise → New (reset for re-routing when cooldowns expire). Either is
3988        // acceptable; the key invariant is that the task is no longer InProgress.
3989        assert!(
3990            matches!(
3991                task.status,
3992                crate::store::TaskStatus::Routed | crate::store::TaskStatus::New
3993            ),
3994            "internal task should be routed to fallback or reset to new, got {:?}",
3995            task.status
3996        );
3997    }
3998
3999    #[tokio::test]
4000    async fn detect_silent_agents_finalizes_open_task_runs_rows() {
4001        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4002        let mock = MockBackend::new();
4003        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4004        let task_manager = Arc::new(TaskManager::with_store(
4005            backend.clone(),
4006            store.clone(),
4007            "owner/repo".to_string(),
4008        ));
4009        let transport = Arc::new(Transport::new());
4010        let capture = Arc::new(CaptureService::new(transport));
4011        let tmux = Arc::new(TmuxManager::new());
4012        let config = EngineConfig {
4013            silence_grace_period: 60,
4014            silence_cooldown: 300,
4015            ..EngineConfig::default()
4016        };
4017
4018        let internal_id = store
4019            .create_internal("owner/repo", "Silent with open run", "", "cron", "1", None)
4020            .await
4021            .unwrap();
4022        store
4023            .set_fields(
4024                internal_id,
4025                &[
4026                    ("agent", serde_json::json!("claude")),
4027                    ("model", serde_json::json!("sonnet")),
4028                ],
4029            )
4030            .await
4031            .unwrap();
4032        store
4033            .update_status(internal_id, crate::store::TaskStatus::InProgress)
4034            .await
4035            .unwrap();
4036
4037        let task_id = format!("internal:{internal_id}");
4038        capture
4039            .register_session("owner/repo", &task_id, "orch-test-silent-run")
4040            .await;
4041        capture
4042            .set_buffer_state_for_test(
4043                "owner/repo",
4044                &task_id,
4045                false,
4046                false,
4047                chrono::Utc::now() - chrono::Duration::seconds(120),
4048            )
4049            .await;
4050
4051        // Start an agent run that will be orphaned by silence detection
4052        store
4053            .start_run(&crate::store::StartRun {
4054                task_id: internal_id,
4055                attempt: 1,
4056                run_type: "agent",
4057                agent: "claude",
4058                model: "sonnet",
4059                command: "claude --model sonnet",
4060                prompt: "fix it",
4061            })
4062            .await
4063            .unwrap();
4064
4065        tick_detect_silent_agents(
4066            &tmux,
4067            "owner/repo",
4068            &capture,
4069            &backend,
4070            &task_manager,
4071            &config,
4072            &store,
4073        )
4074        .await
4075        .unwrap();
4076
4077        let runs = store.get_runs(internal_id).await.unwrap();
4078        assert_eq!(runs.len(), 1);
4079        assert_eq!(runs[0].outcome, "aborted");
4080        assert!(runs[0].completed_at.is_some());
4081        assert_eq!(runs[0].error, "silence detection: session killed");
4082    }
4083
4084    /// Regression test for the race between Phase 1 (tick_check_session_completions)
4085    /// and Phase 2 (tick_recover_stuck_tasks).
4086    ///
4087    /// Before the fix, Phase 1 killed the tmux session without touching updated_at.
4088    /// Phase 2 could then see status=in_progress, has_session=false, and an old
4089    /// updated_at, and would reset the task to New — even though the runner's
4090    /// wait_for_completion() poll had not yet returned.
4091    ///
4092    /// The fix: Phase 1 touches updated_at as soon as it detects the session is done.
4093    /// This test verifies that after the touch, Phase 2 does NOT reclaim the task.
4094    #[tokio::test]
4095    async fn phase1_session_completion_touch_prevents_phase2_reclaim() {
4096        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4097        let mock = MockBackend::new();
4098        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4099        let task_manager = Arc::new(TaskManager::with_store(
4100            backend.clone(),
4101            store.clone(),
4102            "owner/repo".to_string(),
4103        ));
4104        let tmux = Arc::new(TmuxManager::new());
4105        let config = EngineConfig {
4106            no_session_stuck_timeout: 600,
4107            stuck_timeout: 1800,
4108            ..EngineConfig::default()
4109        };
4110
4111        // Create an in-progress internal task with old updated_at.
4112        // Without the fix this task would be reclaimed by Phase 2 (age >> threshold,
4113        // no session running in the test environment).
4114        let id = store
4115            .create_internal("owner/repo", "Completed task", "", "cron", "1", None)
4116            .await
4117            .unwrap();
4118        store
4119            .update_status(id, crate::store::TaskStatus::InProgress)
4120            .await
4121            .unwrap();
4122        set_task_updated_at_past(&store, id).await;
4123
4124        // Simulate Phase 1: touch updated_at when the session is detected as done.
4125        // In production this happens inside tick_check_session_completions.
4126        let task_id = format!("internal:{id}");
4127        store_touch_updated_at(&Some(Arc::clone(&store)), "owner/repo", &task_id).await;
4128
4129        // Phase 2 runs — updated_at is fresh so the task must NOT be reclaimed.
4130        tick_recover_stuck_tasks(
4131            &backend,
4132            &tmux,
4133            "owner/repo",
4134            &task_manager,
4135            &config,
4136            &store,
4137            &std::collections::HashMap::new(),
4138            &cooled_review_router_for_test(),
4139            &Arc::new(DashMap::new()),
4140            &Arc::new(DashSet::new()),
4141        )
4142        .await
4143        .unwrap();
4144
4145        let task = store.get(id).await.unwrap();
4146        assert_eq!(
4147            task.status,
4148            crate::store::TaskStatus::InProgress,
4149            "Phase 2 must not reclaim a task whose updated_at was just touched by Phase 1"
4150        );
4151    }
4152
4153    #[tokio::test]
4154    async fn stuck_task_timing_treats_dead_session_as_missing() {
4155        let tmux = Arc::new(TmuxManager::new());
4156        let repo = "owner/repo";
4157        let task_id = "internal:33498";
4158        let session_name = tmux.session_name(repo, task_id);
4159
4160        let create_result = tokio::process::Command::new("tmux")
4161            .args(["new-session", "-d", "-s", &session_name, "-c", "/tmp"])
4162            .output()
4163            .await;
4164
4165        match create_result {
4166            Ok(o) if o.status.success() => {}
4167            _ => {
4168                eprintln!("Skipping test: tmux not available or failed to create test session");
4169                return;
4170            }
4171        }
4172
4173        let _guard = SessionGuard(session_name.clone());
4174
4175        let set_option_result = tokio::process::Command::new("tmux")
4176            .args(["set-option", "-t", &session_name, "remain-on-exit", "on"])
4177            .output()
4178            .await;
4179        if !matches!(set_option_result, Ok(ref o) if o.status.success()) {
4180            eprintln!("Skipping test: unable to set tmux remain-on-exit option");
4181            return;
4182        }
4183
4184        let send_exit_result = tokio::process::Command::new("tmux")
4185            .args(["send-keys", "-t", &session_name, "exit", "Enter"])
4186            .output()
4187            .await;
4188        if !matches!(send_exit_result, Ok(ref o) if o.status.success()) {
4189            eprintln!("Skipping test: unable to exit tmux pane");
4190            return;
4191        }
4192
4193        // Poll until pane is dead (up to 2s) — fixed sleep is unreliable under load.
4194        for _ in 0..20 {
4195            tokio::time::sleep(std::time::Duration::from_millis(100)).await;
4196            if !tmux.session_is_running(&session_name).await {
4197                break;
4198            }
4199        }
4200
4201        // Also wait for batch_session_active to reflect the dead state.
4202        // session_is_running uses `list-panes -t <session>` which may settle
4203        // before the global `list-panes -a` snapshot used by the production code.
4204        // On CI under load these two tmux commands can transiently disagree.
4205        let mut session_map = tmux.batch_session_active().await;
4206        for _ in 0..20 {
4207            if !session_map.get(&session_name).copied().unwrap_or(false) {
4208                break;
4209            }
4210            tokio::time::sleep(std::time::Duration::from_millis(100)).await;
4211            session_map = tmux.batch_session_active().await;
4212        }
4213        if session_map.get(&session_name).copied().unwrap_or(false) {
4214            eprintln!("Skipping test: session still alive in batch_session_active after retries (CI tmux lag)");
4215            return;
4216        }
4217
4218        let config = EngineConfig {
4219            no_session_stuck_timeout: 600,
4220            stuck_timeout: 1800,
4221            ..EngineConfig::default()
4222        };
4223        let updated_at = (chrono::Utc::now()
4224            - chrono::Duration::seconds(config.no_session_stuck_timeout as i64 + 5))
4225        .to_rfc3339();
4226
4227        let timing = stuck_task_timing_from_map(
4228            &tmux,
4229            repo,
4230            task_id,
4231            &updated_at,
4232            &config,
4233            "parse failure",
4234            &session_map,
4235        )
4236        .expect("dead session should be treated as missing");
4237
4238        assert!(!timing.has_session);
4239        assert_eq!(timing.threshold, config.no_session_stuck_timeout);
4240    }
4241
4242    #[tokio::test]
4243    #[serial_test::serial(suspend_state)]
4244    async fn stuck_task_timing_discounts_host_suspend_gap() {
4245        super::super::suspend::clear_for_test();
4246
4247        let tmux = Arc::new(TmuxManager::new());
4248        let repo = "owner/repo";
4249        let task_id = "internal:99001";
4250        let session_map = std::collections::HashMap::new();
4251
4252        let config = EngineConfig {
4253            no_session_stuck_timeout: 1800,
4254            stuck_timeout: 1800,
4255            ..EngineConfig::default()
4256        };
4257
4258        // Task was last touched 33 minutes ago (age_mins=33 matches the reported
4259        // false-positive), but the host was suspended for 30 of those minutes —
4260        // real elapsed runtime is only 3 minutes, well under the 30-minute threshold.
4261        let updated_at = (chrono::Utc::now() - chrono::Duration::minutes(33)).to_rfc3339();
4262        let suspend_gap_detected_at = chrono::Utc::now() - chrono::Duration::minutes(1);
4263        super::super::suspend::inject_gap_for_test(
4264            suspend_gap_detected_at,
4265            chrono::Duration::minutes(30),
4266        );
4267
4268        let timing = stuck_task_timing_from_map(
4269            &tmux,
4270            repo,
4271            task_id,
4272            &updated_at,
4273            &config,
4274            "parse failure",
4275            &session_map,
4276        );
4277
4278        assert!(
4279            timing.is_none(),
4280            "task age should be discounted below the stuck threshold once suspend time is subtracted, got {timing:?}"
4281        );
4282
4283        super::super::suspend::clear_for_test();
4284    }
4285
4286    #[tokio::test]
4287    async fn recover_stuck_tasks_reclaims_routed_task_blocked_by_stale_session() {
4288        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4289        let mock = MockBackend::new();
4290        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4291        let task_manager = Arc::new(TaskManager::with_store(
4292            backend.clone(),
4293            store.clone(),
4294            "owner/repo".to_string(),
4295        ));
4296        let tmux = Arc::new(TmuxManager::new());
4297        let config = EngineConfig {
4298            no_session_stuck_timeout: 600,
4299            stuck_timeout: 1,
4300            ..EngineConfig::default()
4301        };
4302
4303        let id = store
4304            .create_internal("owner/repo", "Blocked routed task", "", "cron", "1", None)
4305            .await
4306            .unwrap();
4307        store
4308            .update_status(id, crate::store::TaskStatus::Routed)
4309            .await
4310            .unwrap();
4311        set_task_updated_at_past(&store, id).await;
4312
4313        let task_id = format!("internal:{id}");
4314        let session_name = tmux.session_name("owner/repo", &task_id);
4315        let create_result = tokio::process::Command::new("tmux")
4316            .args([
4317                "new-session",
4318                "-d",
4319                "-s",
4320                &session_name,
4321                "-c",
4322                "/tmp",
4323                "sleep 60",
4324            ])
4325            .output()
4326            .await;
4327
4328        match create_result {
4329            Ok(o) if o.status.success() => {}
4330            _ => {
4331                eprintln!("Skipping test: tmux not available or failed to create test session");
4332                return;
4333            }
4334        }
4335
4336        let session_map = tmux.batch_session_active().await;
4337        assert_eq!(session_map.get(&session_name).copied(), Some(true));
4338
4339        tick_recover_stuck_tasks(
4340            &backend,
4341            &tmux,
4342            "owner/repo",
4343            &task_manager,
4344            &config,
4345            &store,
4346            &session_map,
4347            &cooled_review_router_for_test(),
4348            &Arc::new(DashMap::new()),
4349            &Arc::new(DashSet::new()),
4350        )
4351        .await
4352        .unwrap();
4353
4354        let task = store.get(id).await.unwrap();
4355        assert_eq!(
4356            task.status,
4357            crate::store::TaskStatus::New,
4358            "routed task blocked by a stale session should be reset for re-routing"
4359        );
4360        assert!(
4361            !tmux.session_exists(&session_name).await,
4362            "recovery should kill the stale tmux session so dispatch can proceed"
4363        );
4364    }
4365
4366    // ── tick_dispatch_tasks: deadlock regression test (#1361) ──────────────
4367
4368    /// Regression test for #1361: tick_dispatch_tasks previously accepted
4369    /// `&Arc<RwLock<Router>>` and called `router_arc.read().await` internally.
4370    /// The main loop already held a write lock on the same RwLock, so the read
4371    /// could never be acquired — guaranteed deadlock on every tick with routed
4372    /// tasks.
4373    ///
4374    /// The fix changed the signature to accept `&Router` directly (the caller
4375    /// already has the dereferenced guard), eliminating the lock re-acquisition.
4376    ///
4377    /// This test verifies the function completes within 2 seconds when the
4378    /// caller holds the write lock — the old code would deadlock here.
4379    #[tokio::test]
4380    async fn dispatch_does_not_deadlock_under_write_lock() {
4381        use crate::engine::router::Router;
4382        use tokio::sync::RwLock;
4383
4384        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4385        let mock = MockBackend::new();
4386        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4387        let task_manager = Arc::new(TaskManager::with_store(
4388            backend.clone(),
4389            store.clone(),
4390            "owner/repo".to_string(),
4391        ));
4392        let tmux = Arc::new(TmuxManager::new());
4393        let semaphore = Arc::new(Semaphore::new(4));
4394        let (weight_tx, _weight_rx) = mpsc::channel(16);
4395        let dispatching = Arc::new(DashMap::new());
4396        let transport = Arc::new(Transport::new());
4397        let capture = Arc::new(CaptureService::new(transport));
4398
4399        let router_config = crate::engine::router::RouterConfig::default();
4400        let router = Router::new(router_config);
4401        let router_arc = Arc::new(RwLock::new(router));
4402
4403        // Simulate what the main loop does: hold a write lock, snapshot
4404        // dispatch mode, then call tick_dispatch_tasks without passing any
4405        // lock guard into async dispatch work.
4406        let write_guard = router_arc.write().await;
4407        let dispatch_mode = dispatch_mode_from_router(&write_guard);
4408
4409        let result = tokio::time::timeout(
4410            std::time::Duration::from_secs(2),
4411            tick_dispatch_tasks(
4412                &backend,
4413                &tmux,
4414                "owner/repo",
4415                &Arc::new(crate::engine::runner::TaskRunner::new(
4416                    "owner/repo".to_string(),
4417                )),
4418                &capture,
4419                &semaphore,
4420                &task_manager,
4421                &weight_tx,
4422                dispatch_mode,
4423                &dispatching,
4424                &store,
4425                &std::collections::HashMap::new(),
4426            ),
4427        )
4428        .await;
4429
4430        assert!(
4431            result.is_ok(),
4432            "tick_dispatch_tasks deadlocked! It tried to acquire a read lock \
4433             on router_arc while a write lock was already held (issue #1361)"
4434        );
4435    }
4436
4437    /// Regression test for #3599: a task routed to an agent/model that entered
4438    /// cooldown *after* routing but *before* dispatch must not be dispatched —
4439    /// the stale route was decided before the failure that triggered the
4440    /// cooldown was known. Dispatch must re-check the assigned agent/model
4441    /// against the same cooldown store used at routing time and, if stale,
4442    /// leave the task in `Routed` for the next tick to re-evaluate (no PR is
4443    /// spawned, no per-task defer timer is written).
4444    #[tokio::test]
4445    async fn dispatch_skips_task_whose_routed_model_entered_cooldown() {
4446        use crate::store::{StoreRoute, TaskStatus as DbStatus, UpsertExternal};
4447
4448        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4449        let mock = MockBackend::new();
4450        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4451        let task_manager = Arc::new(TaskManager::with_store(
4452            backend.clone(),
4453            store.clone(),
4454            "owner/repo".to_string(),
4455        ));
4456        let tmux = Arc::new(TmuxManager::new());
4457        let runner = Arc::new(TaskRunner::new("owner/repo".to_string()));
4458        let semaphore = Arc::new(Semaphore::new(4));
4459        let (weight_tx, _weight_rx) = mpsc::channel(16);
4460        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4461        let transport = Arc::new(Transport::new());
4462        let capture = Arc::new(CaptureService::new(transport));
4463
4464        let id = store
4465            .upsert_external(&UpsertExternal {
4466                repo: "owner/repo",
4467                ext_id: "999",
4468                title: "Stale routed task",
4469                body: "",
4470                author: "user",
4471                url: "https://github.com/owner/repo/issues/999",
4472                labels: &[],
4473                origin: "github",
4474            })
4475            .await
4476            .unwrap();
4477        store
4478            .store_route(&StoreRoute {
4479                id,
4480                agent: "minimax",
4481                model: Some("opus"),
4482                complexity: "medium",
4483                estimate: 0,
4484                reason: "test",
4485                profile: "{}",
4486                skills: "[]",
4487            })
4488            .await
4489            .unwrap();
4490        store.update_status(id, DbStatus::Routed).await.unwrap();
4491
4492        // Simulate a billing/rate-limit cooldown recorded for minimax:opus after
4493        // this task was routed but before it dispatched.
4494        crate::engine::cooldown::set_model_cooldown(
4495            "minimax",
4496            "opus",
4497            300,
4498            "rate_limit_usage_window",
4499        )
4500        .await;
4501
4502        let dispatch_mode = DispatchMode {
4503            is_degraded: false,
4504            healthy_agents: 1,
4505            threshold: 1,
4506        };
4507
4508        tick_dispatch_tasks(
4509            &backend,
4510            &tmux,
4511            "owner/repo",
4512            &runner,
4513            &capture,
4514            &semaphore,
4515            &task_manager,
4516            &weight_tx,
4517            dispatch_mode,
4518            &dispatching,
4519            &store,
4520            &std::collections::HashMap::new(),
4521        )
4522        .await
4523        .unwrap();
4524
4525        let task = store.get(id).await.unwrap();
4526        assert_eq!(
4527            task.status,
4528            DbStatus::Routed,
4529            "task whose routed model is now cooled down must stay Routed, not be dispatched"
4530        );
4531        assert!(
4532            dispatching.is_empty(),
4533            "dispatch guard must be released when skipping a stale-cooldown task"
4534        );
4535    }
4536
4537    #[tokio::test]
4538    async fn dispatch_skips_a_task_with_an_open_dependency() {
4539        use crate::store::{
4540            DepSource, NewDependency, StoreRoute, TaskStatus as DbStatus, UpsertExternal,
4541        };
4542
4543        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4544        let mock = MockBackend::new();
4545        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4546        let task_manager = Arc::new(TaskManager::with_store(
4547            backend.clone(),
4548            store.clone(),
4549            "owner/repo".to_string(),
4550        ));
4551        let tmux = Arc::new(TmuxManager::new());
4552        let runner = Arc::new(TaskRunner::new("owner/repo".to_string()));
4553        let semaphore = Arc::new(Semaphore::new(4));
4554        let (weight_tx, _weight_rx) = mpsc::channel(16);
4555        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4556        let transport = Arc::new(Transport::new());
4557        let capture = Arc::new(CaptureService::new(transport));
4558
4559        let id = store
4560            .upsert_external(&UpsertExternal {
4561                repo: "owner/repo",
4562                ext_id: "42",
4563                title: "Waiting routed task",
4564                body: "",
4565                author: "user",
4566                url: "https://github.com/owner/repo/issues/42",
4567                labels: &[],
4568                origin: "github",
4569            })
4570            .await
4571            .unwrap();
4572        store
4573            .replace_dependencies(
4574                id,
4575                &[NewDependency {
4576                    repo: "owner/repo".into(),
4577                    number: 7,
4578                    source: DepSource::Github,
4579                }],
4580            )
4581            .await
4582            .unwrap();
4583        store
4584            .store_route(&StoreRoute {
4585                id,
4586                agent: "claude",
4587                model: Some("haiku"),
4588                complexity: "simple",
4589                estimate: 0,
4590                reason: "test",
4591                profile: "{}",
4592                skills: "[]",
4593            })
4594            .await
4595            .unwrap();
4596        store.update_status(id, DbStatus::Routed).await.unwrap();
4597
4598        let dispatch_mode = DispatchMode {
4599            is_degraded: false,
4600            healthy_agents: 1,
4601            threshold: 1,
4602        };
4603        tick_dispatch_tasks(
4604            &backend,
4605            &tmux,
4606            "owner/repo",
4607            &runner,
4608            &capture,
4609            &semaphore,
4610            &task_manager,
4611            &weight_tx,
4612            dispatch_mode,
4613            &dispatching,
4614            &store,
4615            &std::collections::HashMap::new(),
4616        )
4617        .await
4618        .unwrap();
4619
4620        let task = store.get(id).await.unwrap();
4621        assert_eq!(task.status, DbStatus::Waiting);
4622        assert!(dispatching.is_empty());
4623    }
4624
4625    /// Regression test for the completed-session -> stuck-task reclaim race.
4626    ///
4627    /// Scenario:
4628    /// - Runner finishes a tmux session and will call store_touch_updated_at shortly after
4629    /// - Engine's tick may observe the dead session in Phase 1 and Phase 2 may run
4630    ///   stuck-task recovery before the runner touches the store, causing incorrect reclaim
4631    /// Fix: tick_check_session_completions now touches the store.updated_at when it
4632    /// observes a completed session, preventing reclaim. This test simulates the
4633    /// timing by creating a session, marking the corresponding task InProgress with
4634    /// an old updated_at, calling tick_check_session_completions (which should touch
4635    /// the store), then calling tick_recover_stuck_tasks and asserting the task
4636    /// remains InProgress.
4637    #[tokio::test]
4638    async fn completed_session_does_not_cause_stuck_reclaim_race() {
4639        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4640        let mock = MockBackend::new();
4641        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4642        let task_manager = Arc::new(TaskManager::with_store(
4643            backend.clone(),
4644            store.clone(),
4645            "owner/repo".to_string(),
4646        ));
4647        let tmux = Arc::new(TmuxManager::new());
4648        let transport = Arc::new(Transport::new());
4649        let capture = Arc::new(CaptureService::new(transport));
4650
4651        // Insert external task and set to InProgress with old updated_at
4652        let id = store
4653            .upsert_external(&crate::store::UpsertExternal {
4654                repo: "owner/repo",
4655                ext_id: "200",
4656                title: "Race Task",
4657                body: "",
4658                author: "",
4659                url: "",
4660                labels: &[],
4661                origin: "github",
4662            })
4663            .await
4664            .unwrap();
4665        store
4666            .update_status(id, crate::store::TaskStatus::InProgress)
4667            .await
4668            .unwrap();
4669
4670        // set updated_at to far past so it would normally be eligible for reclaim
4671        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4672            .bind(id)
4673            .execute(store.pool())
4674            .await
4675            .unwrap();
4676
4677        // Register a real tmux session for this task and then exit it so it's observed dead
4678        let task_id = "200";
4679        let session_name = tmux.session_name("owner/repo", task_id);
4680        let create_result = tokio::process::Command::new("tmux")
4681            .args(["new-session", "-d", "-s", &session_name, "-c", "/tmp"])
4682            .output()
4683            .await;
4684        match create_result {
4685            Ok(o) if o.status.success() => {}
4686            _ => {
4687                eprintln!("Skipping test: tmux not available or failed to create test session");
4688                return;
4689            }
4690        }
4691        let _guard = SessionGuard(session_name.clone());
4692
4693        // Ensure remain-on-exit so the session lingers as dead long enough to be snapshot
4694        let _ = tokio::process::Command::new("tmux")
4695            .args(["set-option", "-t", &session_name, "remain-on-exit", "on"])
4696            .output()
4697            .await;
4698        let _ = tokio::process::Command::new("tmux")
4699            .args(["send-keys", "-t", &session_name, "exit", "Enter"])
4700            .output()
4701            .await;
4702
4703        // Poll until pane is dead
4704        for _ in 0..20 {
4705            tokio::time::sleep(std::time::Duration::from_millis(100)).await;
4706            if !tmux.session_is_running(&session_name).await {
4707                break;
4708            }
4709        }
4710
4711        // Register the session in capture (engine does this during dispatch)
4712        let task_key = task_id.to_string();
4713        capture
4714            .register_session("owner/repo", &task_key, &session_name)
4715            .await;
4716
4717        // Call phase1 (now touches store.updated_at for completed session)
4718        let config = EngineConfig {
4719            no_session_stuck_timeout: 600,
4720            stuck_timeout: 1800,
4721            ..EngineConfig::default()
4722        };
4723
4724        tick_check_session_completions(&tmux, "owner/repo", &capture, &store)
4725            .await
4726            .unwrap();
4727
4728        // Verify phase1 actually touched updated_at. It only processes sessions
4729        // that appear in snapshot() (list_sessions + batch_session_active). On CI,
4730        // if the session was already fully removed before the snapshot runs (e.g.
4731        // remain-on-exit semantics differ across tmux versions), phase1 won't see
4732        // it and the store won't be touched. Skip in that case.
4733        {
4734            let task_state = store.get(id).await.unwrap();
4735            if task_state.updated_at.starts_with("2020") {
4736                eprintln!(
4737                    "Skipping test: tick_check_session_completions did not update stored \
4738                     updated_at — session was not in snapshot (CI tmux environment issue)"
4739                );
4740                return;
4741            }
4742        }
4743
4744        // Now run phase2 — the updated_at should have been touched so no reclaim occurs
4745        tick_recover_stuck_tasks(
4746            &backend,
4747            &tmux,
4748            "owner/repo",
4749            &task_manager,
4750            &config,
4751            &store,
4752            &std::collections::HashMap::new(),
4753            &cooled_review_router_for_test(),
4754            &Arc::new(DashMap::new()),
4755            &Arc::new(DashSet::new()),
4756        )
4757        .await
4758        .unwrap();
4759
4760        // Verify the task is still InProgress in the store
4761        let task = store.get(id).await.unwrap();
4762        assert_eq!(
4763            task.status,
4764            crate::store::TaskStatus::InProgress,
4765            "task should not be reclaimed to New"
4766        );
4767    }
4768
4769    /// Regression test: a repo configured with a ".git" suffix (e.g. "owner/repo.git")
4770    /// must still match its own live tmux sessions in `tick_check_session_completions`.
4771    ///
4772    /// `TmuxManager::session_name` strips ".git" when creating the session, so a live
4773    /// session's parsed `project` field never carries it. Before the fix, this
4774    /// function's `repo_name` derivation skipped the `.trim_end_matches(".git")` step,
4775    /// so `session.project` ("repo") never equaled `repo_name` ("repo.git") and the
4776    /// completed-session touch/kill was silently skipped for the entire repo — leaving
4777    /// the coarser-grained stuck-task reclaim as the only (racy) safety net.
4778    #[tokio::test]
4779    async fn session_completion_matches_repo_configured_with_git_suffix() {
4780        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4781        let tmux = Arc::new(TmuxManager::new());
4782        let transport = Arc::new(Transport::new());
4783        let capture = Arc::new(CaptureService::new(transport));
4784
4785        let repo = "owner/repo.git";
4786        let id = store
4787            .upsert_external(&crate::store::UpsertExternal {
4788                repo,
4789                ext_id: "201",
4790                title: "Git-suffix repo task",
4791                body: "",
4792                author: "",
4793                url: "",
4794                labels: &[],
4795                origin: "github",
4796            })
4797            .await
4798            .unwrap();
4799        store
4800            .update_status(id, crate::store::TaskStatus::InProgress)
4801            .await
4802            .unwrap();
4803        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4804            .bind(id)
4805            .execute(store.pool())
4806            .await
4807            .unwrap();
4808
4809        // session_name strips ".git" internally — this is the real session a dispatch
4810        // for this repo would have created.
4811        let task_id = "201";
4812        let session_name = tmux.session_name(repo, task_id);
4813        let create_result = tokio::process::Command::new("tmux")
4814            .args(["new-session", "-d", "-s", &session_name, "-c", "/tmp"])
4815            .output()
4816            .await;
4817        match create_result {
4818            Ok(o) if o.status.success() => {}
4819            _ => {
4820                eprintln!("Skipping test: tmux not available or failed to create test session");
4821                return;
4822            }
4823        }
4824        let _guard = SessionGuard(session_name.clone());
4825
4826        let _ = tokio::process::Command::new("tmux")
4827            .args(["set-option", "-t", &session_name, "remain-on-exit", "on"])
4828            .output()
4829            .await;
4830        let _ = tokio::process::Command::new("tmux")
4831            .args(["send-keys", "-t", &session_name, "exit", "Enter"])
4832            .output()
4833            .await;
4834
4835        for _ in 0..20 {
4836            tokio::time::sleep(std::time::Duration::from_millis(100)).await;
4837            if !tmux.session_is_running(&session_name).await {
4838                break;
4839            }
4840        }
4841
4842        capture.register_session(repo, task_id, &session_name).await;
4843
4844        tick_check_session_completions(&tmux, repo, &capture, &store)
4845            .await
4846            .unwrap();
4847
4848        let task_state = store.get(id).await.unwrap();
4849        if task_state.updated_at.starts_with("2020") {
4850            eprintln!(
4851                "Skipping test: tick_check_session_completions did not update stored \
4852                 updated_at — session was not in snapshot (CI tmux environment issue)"
4853            );
4854            return;
4855        }
4856        assert!(
4857            !task_state.updated_at.starts_with("2020"),
4858            "tick_check_session_completions must match its own repo's live sessions \
4859             even when the repo is configured with a \".git\" suffix"
4860        );
4861    }
4862
4863    // ── tick_job_scheduler: resilience to bad project configs ────────────────
4864
4865    /// A project with a duplicate job id (same id in .orch.yml AND prompts/jobs/)
4866    /// must not cause tick_job_scheduler to return Err — it should log an error
4867    /// and return Ok(()) so other projects are unaffected.
4868    #[tokio::test]
4869    async fn job_scheduler_returns_ok_on_duplicate_job_config() {
4870        let tmp = tempfile::tempdir().unwrap();
4871        let cfg = tmp.path().join(".orch.yml");
4872        std::fs::write(
4873            &cfg,
4874            "jobs:\n  - id: trading-scan\n    schedule: \"0 9 * * *\"\n    task:\n      title: Scan\n      body: do it\n",
4875        )
4876        .unwrap();
4877        let jobs_dir = tmp.path().join("prompts").join("jobs");
4878        std::fs::create_dir_all(&jobs_dir).unwrap();
4879        std::fs::write(
4880            jobs_dir.join("trading-scan.md"),
4881            "---\nid: trading-scan\nschedule: '0 9 * * *'\ntitle: Trading Scan\n---\n\nDo trading scan.\n",
4882        )
4883        .unwrap();
4884
4885        let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
4886        let result = tick_job_scheduler(&cfg, &backend, None, "test/repo", None).await;
4887        assert!(
4888            result.is_ok(),
4889            "duplicate job id must not propagate as Err: {result:?}"
4890        );
4891    }
4892
4893    /// When one project has a duplicate job id (bad config) and a second project
4894    /// has a valid config, the scheduler tick for the second project must still
4895    /// succeed — simulating the per-project loop in the engine.
4896    #[tokio::test]
4897    async fn job_scheduler_processes_valid_project_when_other_has_duplicate() {
4898        // Bad project: duplicate id across .orch.yml and prompts/jobs/
4899        let bad_tmp = tempfile::tempdir().unwrap();
4900        let bad_cfg = bad_tmp.path().join(".orch.yml");
4901        std::fs::write(
4902            &bad_cfg,
4903            "jobs:\n  - id: dup\n    schedule: \"0 9 * * *\"\n    task:\n      title: Dup\n      body: x\n",
4904        )
4905        .unwrap();
4906        let bad_jobs_dir = bad_tmp.path().join("prompts").join("jobs");
4907        std::fs::create_dir_all(&bad_jobs_dir).unwrap();
4908        std::fs::write(
4909            bad_jobs_dir.join("dup.md"),
4910            "---\nid: dup\nschedule: '0 9 * * *'\ntitle: Dup\n---\n\nbody\n",
4911        )
4912        .unwrap();
4913
4914        // Good project: valid config, no duplicate
4915        let good_tmp = tempfile::tempdir().unwrap();
4916        let good_cfg = good_tmp.path().join(".orch.yml");
4917        std::fs::write(
4918            &good_cfg,
4919            "jobs:\n  - id: valid-job\n    schedule: \"0 9 * * *\"\n    task:\n      title: Valid\n      body: ok\n",
4920        )
4921        .unwrap();
4922
4923        let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
4924
4925        let bad_result = tick_job_scheduler(&bad_cfg, &backend, None, "test/bad-repo", None).await;
4926        assert!(
4927            bad_result.is_ok(),
4928            "bad project must not abort scheduler: {bad_result:?}"
4929        );
4930
4931        let good_result =
4932            tick_job_scheduler(&good_cfg, &backend, None, "test/good-repo", None).await;
4933        assert!(
4934            good_result.is_ok(),
4935            "valid project must still run after bad project: {good_result:?}"
4936        );
4937    }
4938
4939    // ── route_new_tasks_global ───────────────────────────────────────────────
4940
4941    /// Regression test for #3407: tasks from inactive/removed repos were stuck
4942    /// in `new` indefinitely because `tick_route_tasks` is repo-scoped.
4943    ///
4944    /// `route_new_tasks_global` must route tasks whose repo != current_repo
4945    /// and leave current-repo tasks untouched.
4946    #[serial_test::serial(cooldown_state)]
4947    #[tokio::test]
4948    async fn global_sweep_routes_inactive_repo_tasks() {
4949        use crate::engine::router::{Router, RouterConfig};
4950        use crate::store::{TaskStatus, UpsertExternal};
4951
4952        crate::engine::cooldown::reset_global_state().await;
4953
4954        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4955
4956        // Active repo task — must NOT be touched by the global sweep.
4957        let active_id = store
4958            .upsert_external(&UpsertExternal {
4959                repo: "active/repo",
4960                ext_id: "1",
4961                title: "Active task",
4962                body: "",
4963                author: "user",
4964                url: "https://github.com/active/repo/issues/1",
4965                labels: &[],
4966                origin: "github",
4967            })
4968            .await
4969            .unwrap();
4970
4971        // Inactive repo task — must be routed by the global sweep.
4972        let inactive_id = store
4973            .upsert_external(&UpsertExternal {
4974                repo: "inactive/repo",
4975                ext_id: "42",
4976                title: "Orphaned task",
4977                body: "",
4978                author: "user",
4979                url: "https://github.com/inactive/repo/issues/42",
4980                labels: &[],
4981                origin: "github",
4982            })
4983            .await
4984            .unwrap();
4985
4986        assert_eq!(store.get(active_id).await.unwrap().status, TaskStatus::New);
4987        assert_eq!(
4988            store.get(inactive_id).await.unwrap().status,
4989            TaskStatus::New
4990        );
4991
4992        let mut model_map: std::collections::HashMap<
4993            String,
4994            std::collections::HashMap<String, Vec<String>>,
4995        > = std::collections::HashMap::new();
4996        let mut tier = std::collections::HashMap::new();
4997        tier.insert("claude".to_string(), vec!["haiku".to_string()]);
4998        model_map.insert("medium".to_string(), tier);
4999        let config = RouterConfig {
5000            mode: "round_robin".to_string(),
5001            agents: vec!["claude".to_string()],
5002            model_map,
5003            ..Default::default()
5004        };
5005        let mut router = Router::new_for_test(config, vec!["claude".to_string()]);
5006
5007        let active_repos: std::collections::HashSet<String> =
5008            ["active/repo".to_string()].into_iter().collect();
5009        let mut routing_quota = RoutingTickQuota::new(1);
5010        route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
5011            .await
5012            .unwrap();
5013
5014        // Active-repo task must remain new (global sweep skips it).
5015        assert_eq!(
5016            store.get(active_id).await.unwrap().status,
5017            TaskStatus::New,
5018            "global sweep must not touch tasks for the active repo"
5019        );
5020
5021        // Inactive-repo task must now be routed.
5022        let after = store.get(inactive_id).await.unwrap();
5023        assert_eq!(
5024            after.status,
5025            TaskStatus::Routed,
5026            "global sweep must route tasks from inactive repos"
5027        );
5028        assert!(
5029            matches!(&after.agent, Some(a) if !a.is_empty()),
5030            "agent must be set after routing"
5031        );
5032    }
5033
5034    /// Regression test for #3523: `route_new_tasks_global` previously filtered only
5035    /// `t.repo != current_repo`, so during repo A's tick it would route repo B's `new`
5036    /// tasks too — even though B is an active, non-orphaned project with its own tick
5037    /// scheduled later in the same pass. This is the exact "racing the real per-repo
5038    /// dispatch" failure mode `dispatch_routed_tasks_global` was fixed for in #3415;
5039    /// the route-side sweep needs the same `active_repos` scoping, not just the
5040    /// current tick's own repo.
5041    #[serial_test::serial(cooldown_state)]
5042    #[tokio::test]
5043    async fn global_sweep_does_not_touch_other_active_repo_tasks() {
5044        use crate::engine::router::{Router, RouterConfig};
5045        use crate::store::{TaskStatus, UpsertExternal};
5046
5047        crate::engine::cooldown::reset_global_state().await;
5048
5049        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5050
5051        // Belongs to a second active repo — NOT the repo whose tick is running,
5052        // but still a currently-configured project, not an orphan.
5053        let other_active_id = store
5054            .upsert_external(&UpsertExternal {
5055                repo: "other-active/repo",
5056                ext_id: "7",
5057                title: "Other active project's task",
5058                body: "",
5059                author: "user",
5060                url: "https://github.com/other-active/repo/issues/7",
5061                labels: &[],
5062                origin: "github",
5063            })
5064            .await
5065            .unwrap();
5066
5067        // Genuinely orphaned repo — not in the active set at all.
5068        let inactive_id = store
5069            .upsert_external(&UpsertExternal {
5070                repo: "inactive/repo",
5071                ext_id: "42",
5072                title: "Orphaned task",
5073                body: "",
5074                author: "user",
5075                url: "https://github.com/inactive/repo/issues/42",
5076                labels: &[],
5077                origin: "github",
5078            })
5079            .await
5080            .unwrap();
5081
5082        let mut model_map: std::collections::HashMap<
5083            String,
5084            std::collections::HashMap<String, Vec<String>>,
5085        > = std::collections::HashMap::new();
5086        let mut tier = std::collections::HashMap::new();
5087        tier.insert("claude".to_string(), vec!["haiku".to_string()]);
5088        model_map.insert("medium".to_string(), tier);
5089        let config = RouterConfig {
5090            mode: "round_robin".to_string(),
5091            agents: vec!["claude".to_string()],
5092            model_map,
5093            ..Default::default()
5094        };
5095        let mut router = Router::new_for_test(config, vec!["claude".to_string()]);
5096
5097        // Simulate "active/repo"'s tick running the global sweep. Both
5098        // "active/repo" and "other-active/repo" are currently-configured
5099        // projects; only "inactive/repo" is orphaned.
5100        let active_repos: std::collections::HashSet<String> =
5101            ["active/repo".to_string(), "other-active/repo".to_string()]
5102                .into_iter()
5103                .collect();
5104        let mut routing_quota = RoutingTickQuota::new(1);
5105        route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
5106            .await
5107            .unwrap();
5108
5109        assert_eq!(
5110            store.get(other_active_id).await.unwrap().status,
5111            TaskStatus::New,
5112            "global sweep run from a different active repo's tick must not route \
5113             another active repo's new tasks — that repo's own tick will route them"
5114        );
5115        assert_eq!(
5116            store.get(inactive_id).await.unwrap().status,
5117            TaskStatus::Routed,
5118            "global sweep must still route tasks from genuinely orphaned repos"
5119        );
5120    }
5121
5122    /// Tasks with the `no-agent` label must be skipped by the global sweep.
5123    #[serial_test::serial(cooldown_state)]
5124    #[tokio::test]
5125    async fn global_sweep_skips_no_agent_tasks() {
5126        use crate::engine::router::{Router, RouterConfig};
5127        use crate::store::{TaskStatus, UpsertExternal};
5128
5129        crate::engine::cooldown::reset_global_state().await;
5130
5131        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5132
5133        let id = store
5134            .upsert_external(&UpsertExternal {
5135                repo: "inactive/repo",
5136                ext_id: "99",
5137                title: "No-agent task",
5138                body: "",
5139                author: "user",
5140                url: "https://github.com/inactive/repo/issues/99",
5141                labels: &["no-agent".to_string()],
5142                origin: "github",
5143            })
5144            .await
5145            .unwrap();
5146
5147        let mut model_map: std::collections::HashMap<
5148            String,
5149            std::collections::HashMap<String, Vec<String>>,
5150        > = std::collections::HashMap::new();
5151        let mut tier = std::collections::HashMap::new();
5152        tier.insert("claude".to_string(), vec!["haiku".to_string()]);
5153        model_map.insert("medium".to_string(), tier);
5154        let config = RouterConfig {
5155            mode: "round_robin".to_string(),
5156            agents: vec!["claude".to_string()],
5157            model_map,
5158            ..Default::default()
5159        };
5160        let mut router = Router::new_for_test(config, vec!["claude".to_string()]);
5161
5162        let active_repos: std::collections::HashSet<String> =
5163            ["active/repo".to_string()].into_iter().collect();
5164        let mut routing_quota = RoutingTickQuota::new(1);
5165        route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
5166            .await
5167            .unwrap();
5168
5169        assert_eq!(
5170            store.get(id).await.unwrap().status,
5171            TaskStatus::New,
5172            "no-agent tasks must not be routed"
5173        );
5174    }
5175
5176    /// Regression test for #3615: the routing limit must be consumed by a
5177    /// shared per-engine-tick quota, not independently by every repo/global
5178    /// routing pass. Otherwise one slow router timeout per pass can stack into
5179    /// a multi-minute watchdog stall.
5180    #[serial_test::serial(cooldown_state)]
5181    #[tokio::test]
5182    async fn global_sweep_honors_shared_routing_quota() {
5183        use crate::engine::router::{Router, RouterConfig};
5184        use crate::store::{TaskStatus, UpsertExternal};
5185
5186        crate::engine::cooldown::reset_global_state().await;
5187
5188        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5189        for ext_id in ["1", "2"] {
5190            store
5191                .upsert_external(&UpsertExternal {
5192                    repo: "inactive/repo",
5193                    ext_id,
5194                    title: "Orphaned task",
5195                    body: "",
5196                    author: "user",
5197                    url: &format!("https://github.com/inactive/repo/issues/{ext_id}"),
5198                    labels: &[],
5199                    origin: "github",
5200                })
5201                .await
5202                .unwrap();
5203        }
5204
5205        let mut model_map: std::collections::HashMap<
5206            String,
5207            std::collections::HashMap<String, Vec<String>>,
5208        > = std::collections::HashMap::new();
5209        let mut tier = std::collections::HashMap::new();
5210        tier.insert("claude".to_string(), vec!["haiku".to_string()]);
5211        model_map.insert("medium".to_string(), tier);
5212        let config = RouterConfig {
5213            mode: "round_robin".to_string(),
5214            agents: vec!["claude".to_string()],
5215            model_map,
5216            ..Default::default()
5217        };
5218        let mut router = Router::new_for_test(config, vec!["claude".to_string()]);
5219
5220        let active_repos: std::collections::HashSet<String> =
5221            ["active/repo".to_string()].into_iter().collect();
5222        let mut routing_quota = RoutingTickQuota::new(1);
5223        route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
5224            .await
5225            .unwrap();
5226
5227        let all_tasks = store.list_all_global().await.unwrap();
5228        let routed = all_tasks
5229            .iter()
5230            .filter(|task| task.status == TaskStatus::Routed)
5231            .count();
5232        let still_new = all_tasks
5233            .iter()
5234            .filter(|task| task.status == TaskStatus::New)
5235            .count();
5236
5237        assert_eq!(routed, 1, "only one task may consume the shared quota");
5238        assert_eq!(still_new, 1, "remaining tasks wait for the next tick");
5239
5240        route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
5241            .await
5242            .unwrap();
5243        let after_exhausted = store.list_all_global().await.unwrap();
5244        assert_eq!(
5245            after_exhausted
5246                .iter()
5247                .filter(|task| task.status == TaskStatus::Routed)
5248                .count(),
5249            1,
5250            "an exhausted shared quota must not reset within the same tick"
5251        );
5252    }
5253
5254    // ── dispatch_routed_tasks_global ─────────────────────────────────────────
5255    //
5256    // `dispatch_routed_tasks_global` itself is deliberately not exercised
5257    // end-to-end here: once it finds an orphaned repo, it spawns the real
5258    // runner, which creates a real tmux session and touches the network.
5259    // The existing `dispatch_does_not_deadlock_under_write_lock` test avoids
5260    // this same hazard by keeping the dispatchable set empty. The selection
5261    // logic (which tasks belong to which repo) is pulled into the pure
5262    // `group_orphaned_routed_tasks` helper below so it can be tested without
5263    // those side effects.
5264
5265    /// Regression test for #3413: tasks routed for an inactive/removed repo were
5266    /// stuck in `routed` indefinitely because `tick_dispatch_tasks` is repo-scoped.
5267    ///
5268    /// `group_orphaned_routed_tasks` must select tasks whose repo != the active
5269    /// repo, group them by repo, and leave active-repo tasks out entirely.
5270    #[tokio::test]
5271    async fn group_orphaned_routed_tasks_selects_inactive_repo_tasks() {
5272        use crate::store::{StoreRoute, TaskStatus, UpsertExternal};
5273
5274        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5275
5276        async fn make_routed(store: &TaskStore, repo: &str, ext_id: &str) -> i64 {
5277            let id = store
5278                .upsert_external(&UpsertExternal {
5279                    repo,
5280                    ext_id,
5281                    title: "Routed task",
5282                    body: "",
5283                    author: "user",
5284                    url: &format!("https://github.com/{repo}/issues/{ext_id}"),
5285                    labels: &[],
5286                    origin: "github",
5287                })
5288                .await
5289                .unwrap();
5290            store
5291                .store_route(&StoreRoute {
5292                    id,
5293                    agent: "claude",
5294                    model: Some("sonnet"),
5295                    complexity: "medium",
5296                    estimate: 0,
5297                    reason: "test",
5298                    profile: "{}",
5299                    skills: "[]",
5300                })
5301                .await
5302                .unwrap();
5303            store.update_status(id, TaskStatus::Routed).await.unwrap();
5304            id
5305        }
5306
5307        let active_id = make_routed(&store, "active/repo", "1").await;
5308        let inactive_id_a = make_routed(&store, "inactive/repo", "2").await;
5309        let other_inactive_id = make_routed(&store, "other-inactive/repo", "3").await;
5310        let inactive_id_b = make_routed(&store, "inactive/repo", "4").await;
5311
5312        let all_routed = store
5313            .list_all_by_status_global(TaskStatus::Routed)
5314            .await
5315            .unwrap();
5316        let active_repos: std::collections::HashSet<String> =
5317            ["active/repo".to_string()].into_iter().collect();
5318        let by_repo = group_orphaned_routed_tasks(all_routed, &active_repos);
5319
5320        assert!(
5321            !by_repo.contains_key("active/repo"),
5322            "global sweep must not select tasks for the active repo"
5323        );
5324        let inactive_ids: std::collections::HashSet<i64> = by_repo
5325            .get("inactive/repo")
5326            .expect("inactive/repo must be selected")
5327            .iter()
5328            .map(|t| t.id)
5329            .collect();
5330        assert_eq!(
5331            inactive_ids,
5332            [inactive_id_a, inactive_id_b].into_iter().collect(),
5333            "global sweep must group all orphaned tasks for a repo together"
5334        );
5335        assert_eq!(
5336            by_repo
5337                .get("other-inactive/repo")
5338                .map(|tasks| tasks.iter().map(|t| t.id).collect::<Vec<_>>()),
5339            Some(vec![other_inactive_id])
5340        );
5341
5342        // Sanity: the active-repo task really was created and routed, it's just
5343        // excluded from the selection above.
5344        assert_eq!(
5345            store.get(active_id).await.unwrap().status,
5346            TaskStatus::Routed
5347        );
5348    }
5349
5350    /// Regression test for the PR #3413 review finding: `tick()` runs once per active
5351    /// project, so a single `current_repo` string is the wrong exclusion — repo B's
5352    /// routed tasks must not look "orphaned" during repo A's tick just because A != B.
5353    /// `group_orphaned_routed_tasks` must exclude every repo in the full active set.
5354    #[tokio::test]
5355    async fn group_orphaned_routed_tasks_excludes_all_active_repos_not_just_current() {
5356        use crate::store::{StoreRoute, TaskStatus, UpsertExternal};
5357
5358        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5359
5360        async fn make_routed(store: &TaskStore, repo: &str, ext_id: &str) -> i64 {
5361            let id = store
5362                .upsert_external(&UpsertExternal {
5363                    repo,
5364                    ext_id,
5365                    title: "Routed task",
5366                    body: "",
5367                    author: "user",
5368                    url: &format!("https://github.com/{repo}/issues/{ext_id}"),
5369                    labels: &[],
5370                    origin: "github",
5371                })
5372                .await
5373                .unwrap();
5374            store
5375                .store_route(&StoreRoute {
5376                    id,
5377                    agent: "claude",
5378                    model: Some("sonnet"),
5379                    complexity: "medium",
5380                    estimate: 0,
5381                    reason: "test",
5382                    profile: "{}",
5383                    skills: "[]",
5384                })
5385                .await
5386                .unwrap();
5387            store.update_status(id, TaskStatus::Routed).await.unwrap();
5388            id
5389        }
5390
5391        let repo_a_id = make_routed(&store, "active/repo-a", "1").await;
5392        let repo_b_id = make_routed(&store, "active/repo-b", "2").await;
5393        let orphaned_id = make_routed(&store, "inactive/repo", "3").await;
5394
5395        let all_routed = store
5396            .list_all_by_status_global(TaskStatus::Routed)
5397            .await
5398            .unwrap();
5399        // Simulate repo A's tick: current repo is "active/repo-a", but the full
5400        // active project set also includes "active/repo-b".
5401        let active_repos: std::collections::HashSet<String> =
5402            ["active/repo-a".to_string(), "active/repo-b".to_string()]
5403                .into_iter()
5404                .collect();
5405        let by_repo = group_orphaned_routed_tasks(all_routed, &active_repos);
5406
5407        assert!(
5408            !by_repo.contains_key("active/repo-a"),
5409            "the current repo's own tasks must not be swept"
5410        );
5411        assert!(
5412            !by_repo.contains_key("active/repo-b"),
5413            "another active repo's tasks must not be swept during repo A's tick"
5414        );
5415        assert_eq!(
5416            by_repo
5417                .get("inactive/repo")
5418                .map(|tasks| tasks.iter().map(|t| t.id).collect::<Vec<_>>()),
5419            Some(vec![orphaned_id]),
5420            "a genuinely inactive repo's tasks must still be swept"
5421        );
5422
5423        // Sanity: repo A and repo B tasks were really created and routed, just excluded.
5424        assert_eq!(
5425            store.get(repo_a_id).await.unwrap().status,
5426            TaskStatus::Routed
5427        );
5428        assert_eq!(
5429            store.get(repo_b_id).await.unwrap().status,
5430            TaskStatus::Routed
5431        );
5432    }
5433
5434    /// Tasks with the `no-agent` label must be skipped by the global dispatch sweep.
5435    #[tokio::test]
5436    async fn global_dispatch_sweep_skips_no_agent_tasks() {
5437        use crate::store::{StoreRoute, TaskStatus, UpsertExternal};
5438
5439        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5440
5441        let id = store
5442            .upsert_external(&UpsertExternal {
5443                repo: "inactive/repo",
5444                ext_id: "99",
5445                title: "No-agent routed task",
5446                body: "",
5447                author: "user",
5448                url: "https://github.com/inactive/repo/issues/99",
5449                labels: &["no-agent".to_string()],
5450                origin: "github",
5451            })
5452            .await
5453            .unwrap();
5454        store
5455            .store_route(&StoreRoute {
5456                id,
5457                agent: "claude",
5458                model: Some("sonnet"),
5459                complexity: "medium",
5460                estimate: 0,
5461                reason: "test",
5462                profile: "{}",
5463                skills: "[]",
5464            })
5465            .await
5466            .unwrap();
5467        store.update_status(id, TaskStatus::Routed).await.unwrap();
5468
5469        // Safe to call the full sweep here: with the only orphaned task filtered
5470        // out by the no-agent label, `group_orphaned_routed_tasks` returns an
5471        // empty map and `dispatch_routed_tasks_global` returns before ever
5472        // constructing a backend or spawning a runner.
5473        let tmux = Arc::new(TmuxManager::new());
5474        let transport = Arc::new(Transport::new());
5475        let capture = Arc::new(CaptureService::new(transport));
5476        let semaphore = Arc::new(Semaphore::new(4));
5477        let (weight_tx, _weight_rx) = mpsc::channel(16);
5478        let dispatching = Arc::new(DashMap::new());
5479        let dispatch_mode = DispatchMode {
5480            is_degraded: false,
5481            healthy_agents: 1,
5482            threshold: 1,
5483        };
5484
5485        let active_repos: std::collections::HashSet<String> =
5486            ["active/repo".to_string()].into_iter().collect();
5487        dispatch_routed_tasks_global(
5488            &tmux,
5489            &capture,
5490            &semaphore,
5491            &weight_tx,
5492            dispatch_mode,
5493            &dispatching,
5494            &store,
5495            &std::collections::HashMap::new(),
5496            &active_repos,
5497        )
5498        .await
5499        .unwrap();
5500
5501        assert_eq!(
5502            store.get(id).await.unwrap().status,
5503            TaskStatus::Routed,
5504            "no-agent tasks must not be dispatched"
5505        );
5506    }
5507}