1use super::{EngineConfig, ProjectEngine};
17use crate::backends::github::GitHubBackend;
18use crate::backends::{ExternalBackend, ExternalId, ExternalTask, Status};
19use crate::channels::capture::CaptureService;
20use crate::channels::transport::Transport;
21use crate::cmd_cache;
22use crate::config;
23use crate::engine;
24use crate::engine::cooldown::{
25 SILENCE_AGENT_COOLDOWN_SECS, SILENCE_EXTENDED_COOLDOWN_SECS, github_circuit_remaining_secs,
26 is_agent_in_cooldown, is_github_circuit_open, is_model_in_cooldown,
27 record_agent_failure_with_message, record_silence_detection, set_agent_cooldown,
28 set_model_cooldown,
29};
30use crate::engine::dependencies;
31use crate::engine::dispatch_guard::DispatchGuard;
32use crate::engine::jobs;
33use crate::engine::priority;
34use crate::engine::priority::Priority;
35use crate::engine::router::config as router_config;
36use crate::engine::router::{
37 AllAgentsCooledError, RouteResultError, Router, RouterConfig, get_route_result,
38};
39use crate::engine::runner::response;
40use crate::engine::runner::{NoProjectDir, TaskRunner, WeightSignal};
41use crate::engine::sync;
42use crate::engine::tasks;
43use crate::engine::tasks::{TaskManager, is_internal_id};
44use crate::github::http;
45use crate::repo_context::REPO_CONTEXT;
46use crate::store;
47use crate::store::StoreRoute;
48use crate::store::Task;
49use crate::store::TaskStatus;
50use crate::store::TaskStatus as DbStatus;
51use crate::store::TaskStore;
52use crate::store::WaitingOnDependency;
53use crate::store::{store_set_by_id, store_touch_updated_at};
54use crate::tmux::TmuxManager;
55use dashmap::mapref::entry::Entry;
56use dashmap::{DashMap, DashSet};
57use futures::future;
58use std::collections::HashMap;
59use std::collections::HashSet;
60use std::path::Path;
61use std::sync::Arc;
62use std::sync::OnceLock;
63use std::time::Duration;
64use std::time::{Instant, SystemTime};
65use tokio::sync::RwLock;
66use tokio::sync::{Semaphore, mpsc};
67use tokio::time;
68
69pub(crate) struct RoutingTickQuota {
75 remaining: usize,
76}
77
78impl RoutingTickQuota {
79 pub(crate) fn new(max: usize) -> Self {
80 Self { remaining: max }
81 }
82
83 fn is_exhausted(&self) -> bool {
84 self.remaining == 0
85 }
86
87 fn remaining(&self) -> usize {
88 self.remaining
89 }
90
91 fn consume_one(&mut self) -> bool {
92 if self.remaining == 0 {
93 return false;
94 }
95 self.remaining -= 1;
96 true
97 }
98}
99
100pub(crate) async fn startup_cleanup(tmux: &TmuxManager) {
103 let start_time = SystemTime::now()
104 .duration_since(SystemTime::UNIX_EPOCH)
105 .unwrap_or_default()
106 .as_secs();
107 tracing::info!(
108 start_time,
109 "running startup cleanup for stale tmux sessions"
110 );
111
112 match tmux.kill_stale_sessions(start_time).await {
113 Ok(killed) => {
114 if killed > 0 {
115 tracing::info!(killed, "startup cleanup killed stale sessions");
116 } else {
117 tracing::debug!("no stale tmux sessions found on startup");
118 }
119 }
120 Err(e) => {
121 tracing::warn!(error = %e, "failed to run startup cleanup");
122 }
123 }
124}
125
126async fn recover_routed_blocked_dispatch(
127 backend: &Arc<dyn ExternalBackend>,
128 store: &Arc<TaskStore>,
129 task_manager: &Arc<TaskManager>,
130 repo: &str,
131 task: &ExternalTask,
132 timing: &StuckTaskTiming,
133) {
134 tracing::warn!(
135 task_id = task.id.0,
136 age_mins = timing.age.num_minutes(),
137 threshold_mins = timing.threshold / 60,
138 "recovering routed task: dispatch blocked by stale tmux session → new"
139 );
140
141 let resolved_store_id = match store.resolve_task_id(repo, &task.id.0).await {
142 Ok(Some(id)) => Some(id),
143 Ok(None) => {
144 tracing::warn!(
145 task_id = %task.id.0,
146 repo,
147 "resolve_task_id returned None during routed-task recovery"
148 );
149 None
150 }
151 Err(e) => {
152 tracing::error!(
153 task_id = %task.id.0,
154 repo,
155 error = %e,
156 "resolve_task_id failed during routed-task recovery"
157 );
158 None
159 }
160 };
161
162 if let Some(store_id) = resolved_store_id {
163 if let Err(e) = store_set_by_id(
164 Some(store),
165 store_id,
166 &[
167 ("agent", serde_json::Value::Null),
168 ("model", serde_json::Value::Null),
169 ("route_attempts", serde_json::json!(0)),
170 ],
171 )
172 .await
173 {
174 tracing::warn!(
175 task_id = task.id.0,
176 ?e,
177 "failed to clear routing fields for routed stuck task — will retry next tick"
178 );
179 return;
180 }
181 finalize_open_runs_for_recovery(
182 store,
183 store_id,
184 "stuck-task recovery: stale session blocking dispatch killed",
185 )
186 .await;
187 }
188
189 if let Err(e) = task_manager.update_task_status(&task.id, Status::New).await {
190 tracing::warn!(
191 task_id = task.id.0,
192 ?e,
193 "failed to reset routed stuck task status"
194 );
195 return;
196 }
197
198 store::store_log_activity(
199 &Some(Arc::clone(store)),
200 repo,
201 &task.id.0,
202 "rerouted",
203 Some("routed"),
204 Some("new"),
205 None::<&str>,
206 None::<&str>,
207 Some(&serde_json::json!({
208 "failure_reason": "routed_blocked_by_existing_session",
209 "age_minutes": timing.age.num_minutes(),
210 "had_session": true,
211 })),
212 )
213 .await;
214
215 if !is_internal_id(&task.id.0) {
216 let backend_clone = Arc::clone(backend);
217 let task_id_clone = task.id.clone();
218 let body = format!(
219 "[{}] recovered: stuck routed task — existing tmux session blocked dispatch for {}m; cleared route for retry{}",
220 chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
221 timing.age.num_minutes(),
222 engine::orch_footer()
223 );
224 tokio::spawn(async move {
225 if let Err(e) = backend_clone.post_comment(&task_id_clone, &body).await {
226 tracing::warn!(
227 task_id = task_id_clone.0,
228 ?e,
229 "failed to post routed-task recovery comment"
230 );
231 }
232 });
233 }
234}
235
236pub(crate) async fn tick_check_session_completions(
238 tmux: &Arc<TmuxManager>,
239 repo: &str,
240 capture: &Arc<CaptureService>,
241 store: &Arc<TaskStore>,
242) -> anyhow::Result<()> {
243 let _span = tracing::info_span!("engine.tick.phase1.sessions").entered();
244 let session_snapshot = tmux.snapshot().await;
245 let repo_name = repo
251 .rsplit('/')
252 .next()
253 .unwrap_or(repo)
254 .trim_end_matches(".git");
255 for (session, active) in session_snapshot {
256 if session.project != repo_name {
258 continue;
259 }
260
261 if !active {
262 tracing::info!(
263 session = %session.name,
264 task_id = %session.task_id,
265 "session completed, collecting results"
266 );
267 let store_task_id = {
278 let base = session
279 .task_id
280 .strip_suffix("-review")
281 .unwrap_or(&session.task_id);
282 if let Some(n) = base.strip_prefix("internal-") {
283 format!("internal:{n}")
284 } else {
285 base.to_string()
286 }
287 };
288 store_touch_updated_at(&Some(Arc::clone(store)), repo, &store_task_id).await;
289 capture.unregister_session(repo, &session.task_id).await;
292 if let Err(e) = tmux.kill_session(&session.name).await {
294 tracing::debug!(
295 session = %session.name,
296 ?e,
297 "kill_session failed (session may already be gone)"
298 );
299 }
300 }
301 }
302 Ok(())
303}
304
305#[derive(Debug)]
306struct StuckTaskTiming {
307 has_session: bool,
308 age: chrono::Duration,
309 threshold: u64,
310}
311
312fn stuck_task_timing_from_map(
313 tmux: &Arc<TmuxManager>,
314 repo: &str,
315 session_task_id: &str,
316 updated_at: &str,
317 config: &EngineConfig,
318 parse_error_message: &'static str,
319 session_map: &HashMap<String, bool>,
320) -> Option<StuckTaskTiming> {
321 let has_session = session_alive(tmux, repo, session_task_id, session_map);
322 let threshold = if has_session {
323 config.stuck_timeout
324 } else {
325 config.no_session_stuck_timeout
326 };
327
328 let updated = match chrono::DateTime::parse_from_rfc3339(updated_at) {
329 Ok(dt) => dt.with_timezone(&chrono::Utc),
330 Err(e) => {
331 tracing::warn!(
332 task_id = session_task_id,
333 updated_at,
334 ?e,
335 error_message = parse_error_message,
336 "cannot parse updated_at"
337 );
338 return None;
339 }
340 };
341
342 let raw_age = chrono::Utc::now() - updated;
343 let suspended = super::suspend::suspended_duration_since(updated);
347 let age = (raw_age - suspended).max(chrono::Duration::zero());
348 if age.num_seconds() <= i64::try_from(threshold).unwrap_or(i64::MAX) {
349 return None;
350 }
351
352 Some(StuckTaskTiming {
353 has_session,
354 age,
355 threshold,
356 })
357}
358
359fn session_alive(
360 tmux: &TmuxManager,
361 repo: &str,
362 session_task_id: &str,
363 session_map: &HashMap<String, bool>,
364) -> bool {
365 session_map
366 .get(&tmux.session_name(repo, session_task_id))
367 .copied()
368 .unwrap_or(false)
369}
370
371fn run_tier(router: &RouterConfig, task: &Task) -> String {
374 match (task.agent.as_deref(), task.model.as_deref()) {
375 (Some(a), Some(m)) if !m.is_empty() => router.tier_of_model(a, m, &task.complexity),
376 _ => task.complexity.clone(),
377 }
378}
379
380fn tick_router(
383 cell: &OnceLock<RouterConfig>,
384 load: impl FnOnce() -> RouterConfig,
385) -> &RouterConfig {
386 cell.get_or_init(load)
387}
388
389async fn in_progress_stuck_clock(
394 store: &TaskStore,
395 repo: &str,
396 task_id: &str,
397 updated_at: &str,
398 has_session: bool,
399 config: &EngineConfig,
400 router: &OnceLock<RouterConfig>,
401) -> (String, EngineConfig) {
402 if !has_session {
403 return (updated_at.to_string(), config.clone());
404 }
405
406 let task = match store.resolve_task_id(repo, task_id).await {
407 Ok(Some(store_id)) => store.get(store_id).await.ok(),
408 _ => None,
409 };
410 let Some(task) = task else {
411 return (updated_at.to_string(), config.clone());
412 };
413
414 let entered_at = match store
415 .status_entered_at(task.id, TaskStatus::InProgress)
416 .await
417 {
418 Ok(Some(ts)) => ts,
419 Ok(None) => updated_at.to_string(),
420 Err(e) => {
421 tracing::warn!(
422 task_id,
423 error = %e,
424 "failed to read in_progress entry, falling back to updated_at"
425 );
426 updated_at.to_string()
427 }
428 };
429 let tier = run_tier(tick_router(router, RouterConfig::from_config), &task);
430 let task_config = EngineConfig {
431 stuck_timeout: config.live_session_stuck_limit(Some(&tier)),
432 ..config.clone()
433 };
434 (entered_at, task_config)
435}
436
437async fn in_review_stuck_entry(store: &TaskStore, store_id: i64, fallback: &str) -> (String, bool) {
440 match store.in_review_entry(store_id).await {
441 Ok(Some(entry)) => entry,
442 Ok(None) => (fallback.to_string(), false),
443 Err(e) => {
444 tracing::warn!(
445 store_id,
446 error = %e,
447 "failed to read in_review entry, falling back to updated_at"
448 );
449 (fallback.to_string(), false)
450 }
451 }
452}
453
454async fn finalize_open_runs_for_recovery(store: &TaskStore, store_id: i64, reason: &str) {
457 if let Err(e) = store
458 .finalize_incomplete_runs(store_id, "aborted", reason)
459 .await
460 {
461 tracing::warn!(
462 store_id,
463 error = %e,
464 reason,
465 "failed to finalize incomplete runs during recovery"
466 );
467 }
468}
469
470pub(crate) async fn tick_detect_silent_agents(
479 tmux: &Arc<TmuxManager>,
480 repo: &str,
481 capture: &Arc<CaptureService>,
482 backend: &Arc<dyn ExternalBackend>,
483 task_manager: &Arc<TaskManager>,
484 config: &EngineConfig,
485 store: &Arc<TaskStore>,
486) -> anyhow::Result<()> {
487 let _span = tracing::info_span!("engine.tick.phase1b.silence").entered();
488 let grace = Duration::from_secs(config.silence_grace_period);
489 let silent_sessions = capture.get_silent_sessions_for_repo(repo, grace).await;
490
491 for (task_id, session_name, silence_age_secs) in silent_sessions {
492 let use_backend = should_use_backend(&task_id);
493 let store_task = match store.resolve_task_id(repo, &task_id).await {
495 Ok(Some(store_id)) => match store.get(store_id).await {
496 Ok(task) => Some(task),
497 Err(e) => {
498 tracing::warn!(
499 task_id,
500 repo,
501 store_id,
502 operation = "store.get",
503 error = %e,
504 "silence detection: db error fetching task — agent/model cooldown will be skipped"
505 );
506 None
507 }
508 },
509 Ok(None) => {
510 tracing::debug!(
511 task_id,
512 repo,
513 operation = "resolve_task_id",
514 "silence detection: task not found in store (may have been cleaned up)"
515 );
516 None
517 }
518 Err(e) => {
519 tracing::warn!(
520 task_id,
521 repo,
522 operation = "resolve_task_id",
523 error = %e,
524 "silence detection: db error resolving task id — agent/model cooldown will be skipped"
525 );
526 None
527 }
528 };
529 let agent_name = store_task
530 .as_ref()
531 .and_then(|t| t.agent.clone())
532 .unwrap_or_default();
533 let model_name = store_task
534 .as_ref()
535 .and_then(|t| t.model.clone())
536 .unwrap_or_default();
537
538 tracing::warn!(
539 task_id,
540 agent = %agent_name,
541 model = %model_name,
542 silence_age_secs,
543 grace_secs = config.silence_grace_period,
544 cooldown_secs = config.silence_cooldown,
545 "agent silent since session start — killing session, cooling down model + agent, failing over"
546 );
547
548 if let Err(e) = tmux.kill_session(&session_name).await {
550 tracing::debug!(
551 task_id,
552 ?e,
553 "kill_session failed for silent agent (may already be gone)"
554 );
555 }
556
557 capture.unregister_session(repo, &task_id).await;
559
560 if let Some(store_id) = store_task.as_ref().map(|t| t.id) {
562 finalize_open_runs_for_recovery(store, store_id, "silence detection: session killed")
563 .await;
564 }
565
566 let mut extended_note = String::new();
567
568 if !agent_name.is_empty() && !model_name.is_empty() {
570 set_model_cooldown(
571 &agent_name,
572 &model_name,
573 config.silence_cooldown,
574 "silence_detected",
575 )
576 .await;
577 if let Some(result) = record_silence_detection(&agent_name, &model_name).await
578 && result.extended_cooldown_applied
579 {
580 extended_note = format!(
581 " ({} silences in 24h -> extended cooldown {}s)",
582 result.count, SILENCE_EXTENDED_COOLDOWN_SECS
583 );
584 }
585 }
586
587 if !agent_name.is_empty() {
591 set_agent_cooldown(&agent_name, SILENCE_AGENT_COOLDOWN_SECS).await;
592 }
593
594 let task_eid = ExternalId(task_id.clone());
599
600 let available: Vec<String> = engine::configured_agents()
602 .into_iter()
603 .filter(|a| cmd_cache::command_exists(a))
604 .collect();
605 let chain = response::get_reroute_chain(&task_id, &Some(Arc::clone(store)), repo).await;
606 let chain = response::update_reroute_chain(
607 &task_id,
608 &agent_name,
609 &chain,
610 &Some(Arc::clone(store)),
611 repo,
612 )
613 .await;
614
615 let (next_status, next_agent) = if let Some(fallback) =
616 response::pick_fallback_agent(&agent_name, &chain, &available)
617 {
618 tracing::info!(
619 task_id,
620 from = %agent_name,
621 to = %fallback,
622 "silence detection: failover to different agent, setting routed"
623 );
624 (Status::Routed, Some(fallback))
625 } else {
626 tracing::warn!(
627 task_id,
628 "silence detection: no fallback agents available, resetting to new for re-routing"
629 );
630 (Status::New, None)
631 };
632
633 if use_backend && let Some(ref st) = store_task {
634 let stale_labels: Vec<String> = st
636 .labels
637 .iter()
638 .filter(|l| {
639 l.starts_with("agent:")
640 || l.starts_with("complexity:")
641 || l.starts_with("model:")
642 })
643 .cloned()
644 .collect();
645 if !stale_labels.is_empty() {
646 let backend_clone = Arc::clone(backend);
647 let task_eid_clone = task_eid.clone();
648 tokio::spawn(async move {
649 for label in &stale_labels {
650 if let Err(e) = backend_clone.remove_label(&task_eid_clone, label).await {
651 tracing::warn!(task_id = task_eid_clone.0, label, error = %e, "failed to remove label during silence detection re-route");
652 }
653 }
654 });
655 }
656 }
657
658 if let Some(store_id) = store_task.as_ref().map(|t| t.id) {
659 if let Some(ref fallback) = next_agent {
660 let _ = store_set_by_id(
664 Some(store),
665 store_id,
666 &[
667 ("agent", serde_json::json!(fallback)),
668 ("model", serde_json::json!("")),
669 (
670 "last_error",
671 serde_json::json!(format!(
672 "silence detected after {}s, failing over to {}",
673 config.silence_grace_period, fallback
674 )),
675 ),
676 ],
677 )
678 .await;
679 } else {
680 let _ = store_set_by_id(
681 Some(store),
682 store_id,
683 &[
684 ("agent", serde_json::Value::Null),
685 ("model", serde_json::Value::Null),
686 (
687 "last_error",
688 serde_json::json!(format!(
689 "silence detected after {}s, no fallback agents available",
690 config.silence_grace_period
691 )),
692 ),
693 ],
694 )
695 .await;
696 }
697 }
698
699 if let Err(e) = task_manager
700 .update_task_status(&task_eid, next_status)
701 .await
702 {
703 tracing::warn!(task_id, ?e, "failed to update silent task status");
704 continue;
705 }
706
707 let details = serde_json::json!({
709 "failure_reason": "agent_silence",
710 "silence_duration_secs": config.silence_grace_period,
711 "cooldown_applied": !agent_name.is_empty() && !model_name.is_empty(),
712 "fallback_available": next_agent.is_some(),
713 });
714 store::store_log_activity(
715 &Some(Arc::clone(store)),
716 repo,
717 &task_id,
718 "rerouted",
719 Some("in_progress"),
720 Some(next_status.as_str()),
721 next_agent.as_deref(),
722 None,
723 Some(&details),
724 )
725 .await;
726
727 let action = if let Some(ref fallback) = next_agent {
729 format!("failing over to {fallback}")
730 } else {
731 "marking needs_review (no fallback agents)".to_string()
732 };
733 let comment = format!(
734 "[{}] agent silent for {}s since session start — killed session, cooled down model `{}:{}` for {}s{}, agent `{}` for {}s, {}{}",
735 chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
736 config.silence_grace_period,
737 agent_name,
738 model_name,
739 config.silence_cooldown,
740 extended_note,
741 agent_name,
742 SILENCE_AGENT_COOLDOWN_SECS,
743 action,
744 engine::orch_footer(),
745 );
746 if use_backend {
747 let backend_clone = Arc::clone(backend);
749 let task_eid_clone = task_eid.clone();
750 tokio::spawn(async move {
751 if let Err(e) = backend_clone.post_comment(&task_eid_clone, &comment).await {
752 tracing::warn!(
753 task_id = task_eid_clone.0,
754 ?e,
755 "failed to post silence detection comment"
756 );
757 }
758 });
759 }
760 }
761
762 Ok(())
763}
764
765fn should_use_backend(task_id: &str) -> bool {
766 !is_internal_id(task_id)
767}
768
769#[expect(
774 clippy::too_many_arguments,
775 reason = "each argument is engine state the tick passes down"
776)]
777pub(crate) async fn tick_recover_stuck_tasks(
778 backend: &Arc<dyn ExternalBackend>,
779 tmux: &Arc<TmuxManager>,
780 repo: &str,
781 task_manager: &Arc<TaskManager>,
782 config: &EngineConfig,
783 store: &Arc<TaskStore>,
784 session_map: &HashMap<String, bool>,
785 dispatching: &Arc<DashMap<String, String>>,
786) -> anyhow::Result<()> {
787 let _span = tracing::info_span!("engine.tick.phase2.stuck_tasks").entered();
788 let tick_router_cell = OnceLock::new();
789
790 let in_progress = match task_manager
791 .list_external_by_status(Status::InProgress)
792 .await
793 {
794 Ok(tasks) => tasks,
795 Err(e) => {
796 tracing::warn!(?e, "failed to list in_progress tasks for stuck recovery");
797 vec![]
798 }
799 };
800 for task in &in_progress {
801 let (since, task_config) = in_progress_stuck_clock(
802 store,
803 repo,
804 &task.id.0,
805 &task.updated_at,
806 session_alive(tmux, repo, &task.id.0, session_map),
807 config,
808 &tick_router_cell,
809 )
810 .await;
811 let Some(timing) = stuck_task_timing_from_map(
812 tmux,
813 repo,
814 &task.id.0,
815 &since,
816 &task_config,
817 "cannot parse updated_at, skipping stuck-task check",
818 session_map,
819 ) else {
820 continue;
821 };
822
823 if !timing.has_session {
830 let dispatch_key = format!("{repo}/{}", task.id.0);
831 if dispatching.contains_key(&dispatch_key) {
832 tracing::info!(
840 task_id = task.id.0,
841 dispatch_key,
842 age_mins = timing.age.num_minutes(),
843 "skipping stuck-task reclaim: runner still actively processing this task"
844 );
845 continue;
846 }
847 }
848
849 if timing.has_session {
850 tracing::warn!(
851 task_id = task.id.0,
852 age_mins = timing.age.num_minutes(),
853 threshold_mins = timing.threshold / 60,
854 "recovering stuck task: timed out with active session → new"
855 );
856 let session_name = tmux.session_name(repo, &task.id.0);
859 if let Err(e) = tmux.kill_session(&session_name).await {
860 tracing::warn!(
861 task_id = task.id.0,
862 session = %session_name,
863 error = %e,
864 "failed to kill stuck tmux session, will retry next tick"
865 );
866 continue;
867 }
868 } else {
869 tracing::warn!(
870 task_id = task.id.0,
871 dispatch_key = format!("{repo}/{}", task.id.0),
872 age_mins = timing.age.num_minutes(),
873 threshold_mins = timing.threshold / 60,
874 "recovering stuck task: no session found — reclaiming early → new"
875 );
876 }
877 let mut cached_store_id: Option<i64> = None;
882 if timing.has_session {
883 let store_task = match store.resolve_task_id(repo, &task.id.0).await {
884 Ok(Some(store_id)) => {
885 cached_store_id = Some(store_id);
886 match store.get(store_id).await {
887 Ok(t) => Some(t),
888 Err(e) => {
889 tracing::warn!(task_id = task.id.0, error = %e, "failed to fetch task from store for stuck-task cooldown");
890 None
891 }
892 }
893 }
894 Ok(None) => None,
895 Err(e) => {
896 tracing::warn!(task_id = task.id.0, error = %e, "failed to resolve task id for stuck-task cooldown");
897 None
898 }
899 };
900 let agent_name = store_task
901 .as_ref()
902 .and_then(|t| t.agent.clone())
903 .unwrap_or_default();
904 let model_name = store_task
905 .as_ref()
906 .and_then(|t| t.model.clone())
907 .unwrap_or_default();
908
909 if !agent_name.is_empty() && !model_name.is_empty() {
910 set_model_cooldown(
911 &agent_name,
912 &model_name,
913 config.silence_cooldown,
914 "silence_detected",
915 )
916 .await;
917 record_agent_failure_with_message(
918 &agent_name,
919 &format!(
920 "stuck with active session after {}m",
921 timing.age.num_minutes()
922 ),
923 )
924 .await;
925 }
926 if !agent_name.is_empty() {
927 set_agent_cooldown(&agent_name, SILENCE_AGENT_COOLDOWN_SECS).await;
928 }
929 tracing::warn!(
930 task_id = task.id.0,
931 agent = %agent_name,
932 model = %model_name,
933 cooldown_secs = config.silence_cooldown,
934 "stuck-task cooldown applied — router will avoid this agent/model on next attempt"
935 );
936 }
937
938 {
941 let stale_labels: Vec<String> = task
942 .labels
943 .iter()
944 .filter(|l| l.starts_with("agent:") || l.starts_with("model:"))
945 .cloned()
946 .collect();
947 if !stale_labels.is_empty() {
948 let backend_clone = Arc::clone(backend);
949 let task_id_clone = task.id.clone();
950 tokio::spawn(async move {
951 for label in &stale_labels {
952 if let Err(e) = backend_clone.remove_label(&task_id_clone, label).await {
953 tracing::warn!(task_id = task_id_clone.0, label, error = %e, "failed to remove stale routing label during stuck-task recovery");
954 }
955 }
956 });
957 }
958 }
959 let resolved_store_id = match cached_store_id {
960 Some(id) => Some(id),
961 None => match store.resolve_task_id(repo, &task.id.0).await {
962 Ok(Some(id)) => Some(id),
963 Ok(None) => {
964 tracing::warn!(
965 task_id = %task.id.0,
966 repo,
967 "resolve_task_id returned None during stuck-task recovery"
968 );
969 None
970 }
971 Err(e) => {
972 tracing::error!(
973 task_id = %task.id.0,
974 repo,
975 error = %e,
976 "resolve_task_id failed during stuck-task recovery"
977 );
978 None
979 }
980 },
981 };
982 if let Some(store_id) = resolved_store_id {
983 if let Err(e) = store_set_by_id(
984 Some(store),
985 store_id,
986 &[
987 ("agent", serde_json::Value::Null),
988 ("model", serde_json::Value::Null),
989 ("route_attempts", serde_json::json!(0)),
990 ],
991 )
992 .await
993 {
994 tracing::warn!(
995 task_id = task.id.0,
996 ?e,
997 "failed to clear routing fields for stuck task — will retry next tick"
998 );
999 continue;
1000 }
1001 finalize_open_runs_for_recovery(
1002 store,
1003 store_id,
1004 if timing.has_session {
1005 "stuck-task recovery: active session killed"
1006 } else {
1007 "stuck-task recovery: no session found"
1008 },
1009 )
1010 .await;
1011 }
1012 if let Err(e) = task_manager.update_task_status(&task.id, Status::New).await {
1013 tracing::warn!(task_id = task.id.0, ?e, "failed to reset stuck task status");
1014 continue;
1015 }
1016
1017 let details = serde_json::json!({
1019 "failure_reason": if timing.has_session { "stuck_with_session" } else { "stuck_no_session" },
1020 "age_minutes": timing.age.num_minutes(),
1021 "had_session": timing.has_session,
1022 });
1023 store::store_log_activity(
1024 &Some(Arc::clone(store)),
1025 repo,
1026 &task.id.0,
1027 "rerouted",
1028 Some("in_progress"),
1029 Some("new"),
1030 None::<&str>,
1031 None::<&str>,
1032 Some(&details),
1033 )
1034 .await;
1035
1036 let reason = if timing.has_session {
1037 format!(
1038 "timed out after {}m with active session (cleared agent for re-routing)",
1039 timing.age.num_minutes()
1040 )
1041 } else {
1042 format!(
1043 "no session found — reclaiming early after {}m (cleared agent for re-routing)",
1044 timing.age.num_minutes()
1045 )
1046 };
1047 {
1049 let backend_clone = Arc::clone(backend);
1050 let task_id_clone = task.id.clone();
1051 let body = format!(
1052 "[{}] recovered: stuck in_progress — {}{}",
1053 chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
1054 reason,
1055 engine::orch_footer()
1056 );
1057 tokio::spawn(async move {
1058 if let Err(e) = backend_clone.post_comment(&task_id_clone, &body).await {
1059 tracing::warn!(
1060 task_id = task_id_clone.0,
1061 ?e,
1062 "failed to post stuck-task recovery comment"
1063 );
1064 }
1065 });
1066 }
1067 }
1068
1069 let mut routed = match task_manager.list_external_by_status(Status::Routed).await {
1070 Ok(tasks) => tasks,
1071 Err(e) => {
1072 tracing::warn!(?e, "failed to list routed tasks for stuck recovery");
1073 vec![]
1074 }
1075 };
1076 match task_manager
1077 .list_internal_by_status(TaskStatus::Routed)
1078 .await
1079 {
1080 Ok(tasks) => routed.extend(tasks),
1081 Err(e) => tracing::warn!(
1082 ?e,
1083 "failed to list internal routed tasks for stuck recovery"
1084 ),
1085 }
1086 for task in &routed {
1087 let Some(timing) = stuck_task_timing_from_map(
1088 tmux,
1089 repo,
1090 &task.id.0,
1091 &task.updated_at,
1092 config,
1093 "cannot parse updated_at, skipping routed stuck-task check",
1094 session_map,
1095 ) else {
1096 continue;
1097 };
1098
1099 if !timing.has_session {
1100 continue;
1101 }
1102
1103 let session_name = tmux.session_name(repo, &task.id.0);
1104 if let Err(e) = tmux.kill_session(&session_name).await {
1105 tracing::warn!(
1106 task_id = task.id.0,
1107 session = %session_name,
1108 error = %e,
1109 "failed to kill stale tmux session blocking routed task"
1110 );
1111 continue;
1112 }
1113
1114 recover_routed_blocked_dispatch(backend, store, task_manager, repo, task, &timing).await;
1115 }
1116
1117 let internal_in_progress = match task_manager
1120 .list_internal_by_status(DbStatus::InProgress)
1121 .await
1122 {
1123 Ok(tasks) => tasks,
1124 Err(e) => {
1125 tracing::warn!(
1126 ?e,
1127 "failed to list internal in_progress tasks for stuck recovery"
1128 );
1129 vec![]
1130 }
1131 };
1132 for task in &internal_in_progress {
1133 let task_id = task.id.0.clone();
1134 let (since, task_config) = in_progress_stuck_clock(
1135 store,
1136 repo,
1137 &task_id,
1138 &task.updated_at,
1139 session_alive(tmux, repo, &task_id, session_map),
1140 config,
1141 &tick_router_cell,
1142 )
1143 .await;
1144 let Some(timing) = stuck_task_timing_from_map(
1145 tmux,
1146 repo,
1147 &task_id,
1148 &since,
1149 &task_config,
1150 "cannot parse updated_at, skipping stuck internal-task check",
1151 session_map,
1152 ) else {
1153 continue;
1154 };
1155
1156 if !timing.has_session {
1162 let dispatch_key = format!("{repo}/{task_id}");
1163 if dispatching.contains_key(&dispatch_key) {
1164 tracing::info!(
1174 task_id,
1175 dispatch_key,
1176 age_mins = timing.age.num_minutes(),
1177 "skipping stuck-task reclaim: runner still actively processing this task"
1178 );
1179 continue;
1180 }
1181 }
1182
1183 if timing.has_session {
1184 tracing::warn!(
1185 task_id,
1186 age_mins = timing.age.num_minutes(),
1187 threshold_mins = timing.threshold / 60,
1188 "recovering stuck task: timed out with active session → new"
1189 );
1190 let session_name = tmux.session_name(repo, &task_id);
1191 if let Err(e) = tmux.kill_session(&session_name).await {
1192 tracing::warn!(
1193 task_id,
1194 session = %session_name,
1195 error = %e,
1196 "failed to kill stuck tmux session, will retry next tick"
1197 );
1198 continue;
1199 }
1200 } else {
1201 tracing::warn!(
1202 task_id,
1203 dispatch_key = format!("{repo}/{task_id}"),
1204 age_mins = timing.age.num_minutes(),
1205 threshold_mins = timing.threshold / 60,
1206 "recovering stuck task: no session found — reclaiming early → new"
1207 );
1208 }
1209 let mut cached_store_id: Option<i64> = None;
1213 if timing.has_session {
1214 let store_task = match store.resolve_task_id(repo, &task_id).await {
1215 Ok(Some(store_id)) => {
1216 cached_store_id = Some(store_id);
1217 match store.get(store_id).await {
1218 Ok(t) => Some(t),
1219 Err(e) => {
1220 tracing::warn!(task_id, error = %e, "failed to fetch task from store for stuck internal-task cooldown");
1221 None
1222 }
1223 }
1224 }
1225 Ok(None) => None,
1226 Err(e) => {
1227 tracing::warn!(task_id, error = %e, "failed to resolve task id for stuck internal-task cooldown");
1228 None
1229 }
1230 };
1231 let agent_name = store_task
1232 .as_ref()
1233 .and_then(|t| t.agent.clone())
1234 .unwrap_or_default();
1235 let model_name = store_task
1236 .as_ref()
1237 .and_then(|t| t.model.clone())
1238 .unwrap_or_default();
1239
1240 if !agent_name.is_empty() && !model_name.is_empty() {
1241 set_model_cooldown(
1242 &agent_name,
1243 &model_name,
1244 config.silence_cooldown,
1245 "silence_detected",
1246 )
1247 .await;
1248 record_agent_failure_with_message(
1249 &agent_name,
1250 &format!(
1251 "internal task stuck with active session after {}m",
1252 timing.age.num_minutes()
1253 ),
1254 )
1255 .await;
1256 }
1257 if !agent_name.is_empty() {
1258 set_agent_cooldown(&agent_name, SILENCE_AGENT_COOLDOWN_SECS).await;
1259 }
1260 tracing::warn!(
1261 task_id,
1262 agent = %agent_name,
1263 model = %model_name,
1264 cooldown_secs = config.silence_cooldown,
1265 "internal stuck-task cooldown applied — router will avoid this agent/model on next attempt"
1266 );
1267 }
1268 let resolved_store_id = match cached_store_id {
1271 Some(id) => Some(id),
1272 None => match store.resolve_task_id(repo, &task_id).await {
1273 Ok(Some(id)) => Some(id),
1274 Ok(None) => {
1275 tracing::warn!(
1276 task_id = %task_id,
1277 repo,
1278 "resolve_task_id returned None during stuck internal-task recovery"
1279 );
1280 None
1281 }
1282 Err(e) => {
1283 tracing::error!(
1284 task_id = %task_id,
1285 repo,
1286 error = %e,
1287 "resolve_task_id failed during stuck internal-task recovery"
1288 );
1289 None
1290 }
1291 },
1292 };
1293 if let Some(store_id) = resolved_store_id {
1294 if let Err(e) = store_set_by_id(
1295 Some(store),
1296 store_id,
1297 &[
1298 ("agent", serde_json::Value::Null),
1299 ("model", serde_json::Value::Null),
1300 ("route_attempts", serde_json::json!(0)),
1301 ],
1302 )
1303 .await
1304 {
1305 tracing::warn!(
1306 task_id,
1307 ?e,
1308 "failed to clear routing fields for stuck internal task — will retry next tick"
1309 );
1310 continue;
1311 }
1312 finalize_open_runs_for_recovery(
1313 store,
1314 store_id,
1315 if timing.has_session {
1316 "stuck-task recovery: internal active session killed"
1317 } else {
1318 "stuck-task recovery: internal no session found"
1319 },
1320 )
1321 .await;
1322 }
1323 if let Err(e) = task_manager
1324 .update_task_status(&ExternalId(task_id.clone()), Status::New)
1325 .await
1326 {
1327 tracing::warn!(task_id, ?e, "failed to reset stuck internal task status");
1328 }
1329 }
1330
1331 let in_review = match store.list_by_status(repo, TaskStatus::InReview).await {
1338 Ok(v) => v,
1339 Err(e) => {
1340 tracing::warn!(?e, "failed to list in_review tasks for stuck recovery");
1341 vec![]
1342 }
1343 };
1344 let in_review_config = engine::EngineConfig {
1348 no_session_stuck_timeout: config.in_review_no_session_stuck_timeout,
1349 ..config.clone()
1350 };
1351 for task in &in_review {
1352 let Some(task_id) = task.external_id.clone() else {
1353 tracing::warn!(
1354 store_id = task.id,
1355 "in_review task has no external id, skipping stuck recovery"
1356 );
1357 continue;
1358 };
1359
1360 if dispatching.contains_key(&format!("{repo}/{task_id}")) {
1362 tracing::debug!(
1363 task_id,
1364 "task locked by dispatch flow, skipping stuck check"
1365 );
1366 continue;
1367 }
1368
1369 let (entered_at, decided) = in_review_stuck_entry(store, task.id, &task.updated_at).await;
1372 if decided {
1373 tracing::debug!(
1374 task_id,
1375 "in_review task already has a review decision, waiting on merge"
1376 );
1377 continue;
1378 }
1379
1380 let review_task_id = format!("{task_id}-review");
1381 let Some(timing) = stuck_task_timing_from_map(
1382 tmux,
1383 repo,
1384 &review_task_id,
1385 &entered_at,
1386 &in_review_config,
1387 "cannot parse in_review timestamp, skipping stuck in_review check",
1388 session_map,
1389 ) else {
1390 continue;
1391 };
1392
1393 tracing::warn!(
1394 task_id,
1395 age_mins = timing.age.num_minutes(),
1396 threshold_mins = timing.threshold / 60,
1397 has_session = timing.has_session,
1398 "recovering stuck in_review task — resetting to needs_review"
1399 );
1400
1401 match task_manager
1402 .update_task_status_if(
1403 &ExternalId(task_id.clone()),
1404 Status::NeedsReview,
1405 Status::InReview,
1406 )
1407 .await
1408 {
1409 Err(e) => {
1410 tracing::warn!(task_id, ?e, "failed to reset stuck in_review task status");
1411 continue;
1412 }
1413 Ok(false) => {
1414 tracing::debug!(
1415 task_id,
1416 "stuck in_review reset skipped — task already transitioned"
1417 );
1418 continue;
1419 }
1420 Ok(true) => {}
1421 }
1422
1423 if let Err(e) = store
1426 .set_fields(task.id, &[("review_agent_failures", serde_json::json!(0))])
1427 .await
1428 {
1429 tracing::warn!(task_id, ?e, "failed to reset review_agent_failures");
1430 }
1431
1432 finalize_open_runs_for_recovery(
1433 store,
1434 task.id,
1435 if timing.has_session {
1436 "stuck-task recovery: in_review session killed"
1437 } else {
1438 "stuck-task recovery: in_review no session found"
1439 },
1440 )
1441 .await;
1442
1443 let details = serde_json::json!({
1444 "failure_reason": "stuck_in_review",
1445 "age_minutes": timing.age.num_minutes(),
1446 "had_session": timing.has_session,
1447 });
1448 store::store_log_activity(
1449 &Some(Arc::clone(store)),
1450 repo,
1451 &task_id,
1452 "rerouted",
1453 Some("in_review"),
1454 Some("needs_review"),
1455 None::<&str>,
1456 None::<&str>,
1457 Some(&details),
1458 )
1459 .await;
1460
1461 if task.origin == "internal" {
1463 continue;
1464 }
1465
1466 if let Err(e) = backend
1467 .post_comment(
1468 &ExternalId(task_id.clone()),
1469 &format!(
1470 "[{}] recovered: stuck in_review — {} after {}m, resetting to needs_review{}",
1471 chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
1472 if timing.has_session {
1473 "review session hung"
1474 } else {
1475 "no session found"
1476 },
1477 timing.age.num_minutes(),
1478 engine::orch_footer()
1479 ),
1480 )
1481 .await
1482 {
1483 tracing::warn!(
1484 task_id,
1485 ?e,
1486 "failed to post stuck in_review recovery comment"
1487 );
1488 }
1489 }
1490
1491 Ok(())
1492}
1493
1494pub(crate) async fn recover_untracked_repo_stuck_tasks(
1497 tmux: &Arc<TmuxManager>,
1498 config: &EngineConfig,
1499 store: &Arc<TaskStore>,
1500 session_map: &HashMap<String, bool>,
1501 dispatching: &Arc<DashMap<String, String>>,
1502 active_repos: &HashSet<String>,
1503) {
1504 let all_external_in_progress = match store
1505 .list_all_by_status_global(TaskStatus::InProgress)
1506 .await
1507 {
1508 Ok(tasks) => tasks,
1509 Err(e) => {
1510 tracing::warn!(
1511 ?e,
1512 "failed to list global in_progress tasks for cross-repo stuck recovery"
1513 );
1514 vec![]
1515 }
1516 };
1517 for task in all_external_in_progress
1518 .iter()
1519 .filter(|t| t.origin != "internal" && !active_repos.contains(&t.repo))
1520 {
1521 let external_id = task
1522 .external_id
1523 .clone()
1524 .unwrap_or_else(|| task.id.to_string());
1525 let Some(timing) = stuck_task_timing_from_map(
1526 tmux,
1527 &task.repo,
1528 &external_id,
1529 &task.updated_at,
1530 config,
1531 "cannot parse updated_at, skipping cross-repo stuck-task check",
1532 session_map,
1533 ) else {
1534 continue;
1535 };
1536
1537 if !timing.has_session {
1541 let dispatch_key = format!("{}/{external_id}", task.repo);
1542 if dispatching.contains_key(&dispatch_key) {
1543 tracing::info!(
1547 task_id = task.id,
1548 repo = %task.repo,
1549 dispatch_key,
1550 age_mins = timing.age.num_minutes(),
1551 "skipping cross-repo stuck-task reclaim: runner still actively processing this task"
1552 );
1553 continue;
1554 }
1555 }
1556
1557 if timing.has_session {
1558 tracing::warn!(
1559 task_id = task.id,
1560 repo = %task.repo,
1561 age_mins = timing.age.num_minutes(),
1562 threshold_mins = timing.threshold / 60,
1563 "recovering cross-repo stuck task: timed out with active session → new"
1564 );
1565 } else {
1566 tracing::warn!(
1567 task_id = task.id,
1568 repo = %task.repo,
1569 dispatch_key = format!("{}/{external_id}", task.repo),
1570 age_mins = timing.age.num_minutes(),
1571 threshold_mins = timing.threshold / 60,
1572 "recovering cross-repo stuck task: no session found — reclaiming early → new"
1573 );
1574 }
1575
1576 if let Err(e) = store_set_by_id(
1577 Some(store),
1578 task.id,
1579 &[
1580 ("agent", serde_json::Value::Null),
1581 ("model", serde_json::Value::Null),
1582 ("route_attempts", serde_json::json!(0)),
1583 ],
1584 )
1585 .await
1586 {
1587 tracing::warn!(
1588 task_id = task.id,
1589 repo = %task.repo,
1590 ?e,
1591 "failed to clear routing fields for cross-repo stuck task — will retry next tick"
1592 );
1593 continue;
1594 }
1595
1596 finalize_open_runs_for_recovery(
1597 store,
1598 task.id,
1599 if timing.has_session {
1600 "stuck-task recovery: cross-repo active session killed"
1601 } else {
1602 "stuck-task recovery: cross-repo no session found"
1603 },
1604 )
1605 .await;
1606
1607 if let Err(e) = store.update_status(task.id, TaskStatus::New).await {
1608 tracing::warn!(
1609 task_id = task.id,
1610 repo = %task.repo,
1611 ?e,
1612 "failed to reset cross-repo stuck task status"
1613 );
1614 continue;
1615 }
1616
1617 let details = serde_json::json!({
1618 "failure_reason": if timing.has_session { "stuck_with_session" } else { "stuck_no_session" },
1619 "age_minutes": timing.age.num_minutes(),
1620 "had_session": timing.has_session,
1621 "scope": "cross_repo_global_sweep",
1622 });
1623 store::store_log_activity(
1624 &Some(Arc::clone(store)),
1625 &task.repo,
1626 &external_id,
1627 "rerouted",
1628 Some("in_progress"),
1629 Some("new"),
1630 None::<&str>,
1631 None::<&str>,
1632 Some(&details),
1633 )
1634 .await;
1635 }
1636}
1637
1638pub(crate) async fn tick_global_sweeps(
1647 task_manager: &Arc<TaskManager>,
1648 store: &Arc<TaskStore>,
1649 router: &Router,
1650 dispatching: &Arc<DashMap<String, String>>,
1651 auto_merge_in_flight: &Arc<DashSet<String>>,
1652 active_repos: &HashSet<String>,
1653) {
1654 if let Err(e) = sync::auto_unblock_ci_failure_blocked_tasks_global(
1659 task_manager,
1660 store,
1661 auto_merge_in_flight,
1662 )
1663 .await
1664 {
1665 tracing::warn!(err = %e, "global CI-failure blocked sweep failed");
1666 }
1667
1668 if let Err(e) = sync::refire_and_escalate_stale_needs_review_global(
1671 task_manager,
1672 store,
1673 router,
1674 dispatching,
1675 active_repos,
1676 )
1677 .await
1678 {
1679 tracing::warn!(err = %e, "global NeedsReview refire/escalation sweep failed");
1680 }
1681
1682 if let Err(e) = sync::auto_recover_rebroadcast_blocked_tasks_global(
1685 router,
1686 task_manager,
1687 store,
1688 active_repos,
1689 )
1690 .await
1691 {
1692 tracing::warn!(err = %e, "global rebroadcast-blocked recovery sweep failed");
1693 }
1694}
1695
1696pub(crate) async fn tick_route_tasks(
1698 backend: &Arc<dyn ExternalBackend>,
1699 task_manager: &Arc<TaskManager>,
1700 router: &mut Router,
1701 store: &Arc<TaskStore>,
1702 repo: &str,
1703 routing_quota: &mut RoutingTickQuota,
1704) -> anyhow::Result<()> {
1705 let _span = tracing::info_span!("engine.tick.phase3a.route").entered();
1706 if let Err(err) = dependencies::apply_claim_gates(store).await {
1707 tracing::warn!(err = %err, "dependency claim gate failed");
1708 }
1709 if routing_quota.is_exhausted() {
1710 tracing::debug!("routing quota exhausted for this tick — skipping repo routing phase");
1711 return Ok(());
1712 }
1713
1714 if is_github_circuit_open() {
1718 let remaining = github_circuit_remaining_secs();
1719 tracing::info!(
1720 remaining_secs = remaining,
1721 "GitHub 5xx circuit breaker open — skipping routing phase"
1722 );
1723 return Ok(());
1724 }
1725 let new_tasks = task_manager.list_routable().await?;
1726 let routable: Vec<&ExternalTask> = new_tasks
1727 .iter()
1728 .filter(|t| !skips_agent(&t.labels))
1729 .collect();
1730
1731 if !routable.is_empty() {
1735 let start = Instant::now();
1736 router.refresh_health(store).await;
1737 tracing::debug!(
1738 duration_ms = start.elapsed().as_millis(),
1739 "router.refresh_health completed"
1740 );
1741 }
1742
1743 for task in routable.into_iter().take(routing_quota.remaining()) {
1745 if !routing_quota.consume_one() {
1746 break;
1747 }
1748 let _task_span = tracing::info_span!("engine.route", task_id = %task.id.0).entered();
1749
1750 let task_start = Instant::now();
1751 match router.route(task, store, repo).await {
1752 Ok(result) => {
1753 tracing::debug!(task_id = %task.id.0, duration_ms = task_start.elapsed().as_millis(), "route completed");
1754 if let Err(e) = router
1756 .store_route_result(&task.id.0, &result, store, repo)
1757 .await
1758 {
1759 tracing::warn!(
1760 task_id = task.id.0,
1761 ?e,
1762 "failed to store route result; skipping Routed transition"
1763 );
1764 continue;
1765 }
1766
1767 if is_internal_id(&task.id.0) {
1768 if let Err(e) = task_manager
1770 .update_task_status(&task.id, Status::Routed)
1771 .await
1772 {
1773 tracing::warn!(
1774 task_id = task.id.0,
1775 ?e,
1776 "failed to set internal status:routed"
1777 );
1778 continue;
1779 }
1780 } else {
1781 let backend_clone = Arc::clone(backend);
1785 let task_id_clone = task.id.clone();
1786 let task_labels: Vec<String> = task.labels.clone();
1787 let new_labels = {
1788 let mut labels = vec![
1789 format!("agent:{}", result.agent),
1790 format!("complexity:{}", result.complexity),
1791 ];
1792 if let Some(ref model) = result.model {
1793 labels.push(format!("model:{model}"));
1794 }
1795 labels
1796 };
1797 tokio::spawn(async move {
1798 for label in &task_labels {
1800 if (label.starts_with("agent:")
1801 || label.starts_with("complexity:")
1802 || label.starts_with("model:"))
1803 && let Err(e) =
1804 backend_clone.remove_label(&task_id_clone, label).await
1805 {
1806 tracing::warn!(task_id = task_id_clone.0, label, error = %e, "failed to remove stale routing label during re-route");
1807 }
1808 }
1809 if let Err(e) = backend_clone.set_labels(&task_id_clone, &new_labels).await
1811 {
1812 tracing::warn!(
1813 task_id = task_id_clone.0,
1814 ?e,
1815 "failed to set routing labels"
1816 );
1817 }
1818 });
1819
1820 if let Err(e) = task_manager
1822 .update_task_status(&task.id, Status::Routed)
1823 .await
1824 {
1825 tracing::warn!(task_id = task.id.0, ?e, "failed to set status:routed");
1826 continue;
1827 }
1828 }
1829
1830 match store.ensure_external_task(repo, task).await {
1832 Ok(store_id) => {
1833 let profile_json =
1834 serde_json::to_string(&result.profile).unwrap_or_default();
1835 let skills_json =
1836 serde_json::to_string(&result.selected_skills).unwrap_or_default();
1837 if let Err(e) = store
1838 .store_route(&crate::store::StoreRoute {
1839 id: store_id,
1840 agent: &result.agent,
1841 model: result.model.as_deref(),
1842 complexity: &result.complexity,
1843 estimate: result.estimate,
1844 reason: &result.reason,
1845 profile: &profile_json,
1846 skills: &skills_json,
1847 })
1848 .await
1849 {
1850 tracing::debug!(
1851 task_id = task.id.0,
1852 ?e,
1853 "dual-write: store_route failed"
1854 );
1855 }
1856 if let Err(e) = store.update_status(store_id, TaskStatus::Routed).await {
1857 tracing::debug!(
1858 task_id = task.id.0,
1859 ?e,
1860 "dual-write: update_status failed"
1861 );
1862 }
1863 if result.estimate > 0 {
1867 let backend_clone = Arc::clone(backend);
1868 let task_id = task.id.clone();
1869 let estimate = result.estimate;
1870 tokio::spawn(async move {
1871 if let Err(e) = backend_clone
1872 .sync_estimate_to_project(&task_id, estimate)
1873 .await
1874 {
1875 tracing::debug!(
1876 task_id = task_id.0,
1877 estimate,
1878 err = %e,
1879 "dual-write: estimate project sync failed"
1880 );
1881 }
1882 });
1883 }
1884 }
1885 Err(e) => {
1886 tracing::debug!(
1887 task_id = task.id.0,
1888 ?e,
1889 "dual-write: ensure_external_task failed"
1890 );
1891 }
1892 }
1893
1894 if let Some(ref warning) = result.warning {
1895 tracing::warn!(task_id = task.id.0, warning, "routing sanity warning");
1896 }
1897
1898 tracing::info!(
1899 task_id = task.id.0,
1900 agent = %result.agent,
1901 complexity = %result.complexity,
1902 reason = %result.reason,
1903 "task routed"
1904 );
1905 }
1906 Err(e) => {
1907 if let Some(cooled) = e.downcast_ref::<AllAgentsCooledError>() {
1908 tracing::error!(
1909 task_id = %task.id.0,
1910 scope = %cooled.scope(),
1911 remaining_secs = ?cooled.remaining_secs(),
1912 degraded = ?cooled.degraded(),
1913 "ALL AGENTS COOLED — no agent available to route this task. \
1914 Task stays in 'new' and will be retried on every tick. \
1915 Check `orch cooldown list` to see what is blocking dispatch."
1916 );
1917 continue;
1918 }
1919 if let Some(waiting) = e.downcast_ref::<WaitingOnDependency>() {
1920 tracing::info!(
1921 task_id = %task.id.0,
1922 blockers = %waiting.blockers.join(", "),
1923 "task is waiting on a dependency"
1924 );
1925 continue;
1926 }
1927 tracing::error!(task_id = task.id.0, ?e, "routing failed, skipping task");
1928 }
1929 }
1930 }
1931 Ok(())
1932}
1933
1934pub(crate) async fn route_new_tasks_global(
1950 router: &mut Router,
1951 store: &Arc<TaskStore>,
1952 active_repos: &HashSet<String>,
1953 routing_quota: &mut RoutingTickQuota,
1954) -> anyhow::Result<()> {
1955 if routing_quota.is_exhausted() {
1956 tracing::debug!("routing quota exhausted for this tick — skipping global routing sweep");
1957 return Ok(());
1958 }
1959
1960 let all_new = store.list_all_by_status_global(TaskStatus::New).await?;
1961 let orphaned: Vec<&Task> = all_new
1962 .iter()
1963 .filter(|t| !active_repos.contains(&t.repo))
1964 .filter(|t| !skips_agent(&t.labels))
1965 .collect();
1966
1967 if orphaned.is_empty() {
1968 return Ok(());
1969 }
1970
1971 tracing::info!(
1972 count = orphaned.len(),
1973 "global routing sweep: found new tasks from inactive repos"
1974 );
1975
1976 router.refresh_health(store).await;
1977
1978 for task in orphaned.into_iter().take(routing_quota.remaining()) {
1979 if !routing_quota.consume_one() {
1980 break;
1981 }
1982 let ext_task = tasks::store_task_to_external(task);
1983 let task_start = Instant::now();
1984 match router.route(&ext_task, store, &task.repo).await {
1985 Ok(result) => {
1986 tracing::debug!(
1987 task_id = %ext_task.id.0,
1988 repo = %task.repo,
1989 duration_ms = task_start.elapsed().as_millis(),
1990 "global sweep: route completed"
1991 );
1992 let profile_json = serde_json::to_string(&result.profile).unwrap_or_default();
1993 let skills_json =
1994 serde_json::to_string(&result.selected_skills).unwrap_or_default();
1995 if let Err(e) = store
1996 .store_route(&StoreRoute {
1997 id: task.id,
1998 agent: &result.agent,
1999 model: result.model.as_deref(),
2000 complexity: &result.complexity,
2001 estimate: result.estimate,
2002 reason: &result.reason,
2003 profile: &profile_json,
2004 skills: &skills_json,
2005 })
2006 .await
2007 {
2008 tracing::warn!(
2009 task_id = %ext_task.id.0,
2010 repo = %task.repo,
2011 err = %e,
2012 "global sweep: failed to store route result; skipping status update"
2013 );
2014 continue;
2015 }
2016 match store.update_status(task.id, TaskStatus::Routed).await {
2017 Err(e) => {
2018 tracing::warn!(
2019 task_id = %ext_task.id.0,
2020 repo = %task.repo,
2021 err = %e,
2022 "global sweep: failed to set status:routed"
2023 );
2024 }
2025 _ => {
2026 tracing::info!(
2027 task_id = %ext_task.id.0,
2028 repo = %task.repo,
2029 agent = %result.agent,
2030 complexity = %result.complexity,
2031 "global sweep: task routed"
2032 );
2033 }
2034 }
2035 if let Some(ref warning) = result.warning {
2036 tracing::warn!(task_id = %ext_task.id.0, warning, "global sweep: routing sanity warning");
2037 }
2038 }
2039 Err(e) => {
2040 if let Some(cooled) = e.downcast_ref::<AllAgentsCooledError>() {
2041 tracing::error!(
2042 task_id = %ext_task.id.0,
2043 repo = %task.repo,
2044 scope = %cooled.scope(),
2045 remaining_secs = ?cooled.remaining_secs(),
2046 degraded = ?cooled.degraded(),
2047 "global sweep: ALL AGENTS COOLED — task stays in 'new'"
2048 );
2049 continue;
2050 }
2051 if let Some(waiting) = e.downcast_ref::<WaitingOnDependency>() {
2052 tracing::info!(
2053 task_id = %ext_task.id.0,
2054 repo = %task.repo,
2055 blockers = %waiting.blockers.join(", "),
2056 "global sweep: task is waiting on a dependency"
2057 );
2058 continue;
2059 }
2060 tracing::error!(
2061 task_id = %ext_task.id.0,
2062 repo = %task.repo,
2063 err = ?e,
2064 "global sweep: routing failed, skipping task"
2065 );
2066 }
2067 }
2068 }
2069
2070 Ok(())
2071}
2072
2073#[derive(Debug, Clone, Copy)]
2075pub(crate) struct DispatchMode {
2076 pub(crate) is_degraded: bool,
2077 pub(crate) healthy_agents: usize,
2078 pub(crate) threshold: usize,
2079}
2080
2081pub(crate) fn dispatch_mode_from_router(router: &Router) -> DispatchMode {
2082 let threshold = router_config::min_healthy_agents_threshold();
2083 let healthy_agents = router.healthy_agent_count("simple");
2084 let is_degraded = router.is_degraded(threshold);
2085 DispatchMode {
2086 is_degraded,
2087 healthy_agents,
2088 threshold,
2089 }
2090}
2091
2092#[expect(
2094 clippy::too_many_arguments,
2095 reason = "each argument is engine state the tick passes down"
2096)]
2097#[tracing::instrument(skip_all, name = "engine.tick.phase3b.dispatch")]
2098pub(crate) async fn tick_dispatch_tasks(
2099 backend: &Arc<dyn ExternalBackend>,
2100 tmux: &Arc<TmuxManager>,
2101 repo: &str,
2102 runner: &Arc<TaskRunner>,
2103 capture: &Arc<CaptureService>,
2104 semaphore: &Arc<Semaphore>,
2105 task_manager: &Arc<TaskManager>,
2106 weight_tx: &mpsc::Sender<WeightSignal>,
2107 dispatch_mode: DispatchMode,
2108 dispatching: &Arc<DashMap<String, String>>,
2109 store: &Arc<TaskStore>,
2110 session_map: &HashMap<String, bool>,
2111) -> anyhow::Result<()> {
2112 let mut routed_tasks = task_manager.list_external_by_status(Status::Routed).await?;
2114
2115 let internal_routed = task_manager
2117 .list_internal_by_status(DbStatus::Routed)
2118 .await?;
2119 routed_tasks.extend(internal_routed);
2120
2121 let dispatchable: Vec<ExternalTask> = routed_tasks
2123 .into_iter()
2124 .filter(|t| !skips_agent(&t.labels))
2125 .collect();
2126
2127 dispatch_tasks_for_repo(
2128 backend,
2129 tmux,
2130 repo,
2131 runner,
2132 capture,
2133 semaphore,
2134 task_manager,
2135 weight_tx,
2136 dispatch_mode,
2137 dispatching,
2138 store,
2139 session_map,
2140 dispatchable,
2141 )
2142 .await
2143}
2144
2145#[expect(
2151 clippy::too_many_arguments,
2152 reason = "each argument is engine state the tick passes down"
2153)]
2154async fn dispatch_tasks_for_repo(
2155 backend: &Arc<dyn ExternalBackend>,
2156 tmux: &Arc<TmuxManager>,
2157 repo: &str,
2158 runner: &Arc<TaskRunner>,
2159 capture: &Arc<CaptureService>,
2160 semaphore: &Arc<Semaphore>,
2161 task_manager: &Arc<TaskManager>,
2162 weight_tx: &mpsc::Sender<WeightSignal>,
2163 dispatch_mode: DispatchMode,
2164 dispatching: &Arc<DashMap<String, String>>,
2165 store: &Arc<TaskStore>,
2166 session_map: &HashMap<String, bool>,
2167 dispatchable: Vec<ExternalTask>,
2168) -> anyhow::Result<()> {
2169 if dispatchable.is_empty() {
2170 tracing::debug!(count = 0, "dispatchable tasks found");
2171 return Ok(());
2172 }
2173
2174 tracing::info!(count = dispatchable.len(), "dispatchable tasks found");
2175 if dispatch_mode.is_degraded {
2176 tracing::warn!(
2177 healthy_agents = dispatch_mode.healthy_agents,
2178 threshold = dispatch_mode.threshold,
2179 "degraded mode: using sequential dispatch"
2180 );
2181 }
2182
2183 let sequential_delay = if dispatch_mode.is_degraded {
2184 router_config::sequential_dispatch_delay_ms()
2185 } else {
2186 0
2187 };
2188
2189 let mut ranked: Vec<(Priority, ExternalTask)> = Vec::with_capacity(dispatchable.len());
2190 for task in dispatchable {
2191 let has_pr = match store.resolve_id_by_external(repo, &task.id.0).await {
2192 Ok(Some(id)) => store.get(id).await.is_ok_and(|t| t.pr_number.is_some()),
2193 _ => false,
2194 };
2195 ranked.push((Priority::from_labels(&task.labels).with_pr(has_pr), task));
2196 }
2197 priority::sort_by_priority(&mut ranked, |t| t.0);
2198
2199 for (idx, (priority, task)) in ranked.into_iter().enumerate() {
2200 if dependencies::dispatch_is_waiting(store, repo, &task.id.0).await {
2201 continue;
2202 }
2203 if idx > 0 && dispatch_mode.is_degraded {
2205 let delay_ms = sequential_delay;
2206 tracing::debug!(
2207 task_id = task.id.0,
2208 delay_ms = delay_ms,
2209 "sequential dispatch: waiting before dispatch"
2210 );
2211 time::sleep(Duration::from_millis(delay_ms)).await;
2212 }
2213 let dispatch_key = format!("{}/{}", repo, task.id.0);
2220 {
2225 match dispatching.entry(dispatch_key.clone()) {
2226 Entry::Occupied(existing) => {
2227 let existing_id = existing.get().clone();
2228 drop(existing); if existing_id == task.id.0 {
2230 tracing::debug!(
2231 task_id = task.id.0,
2232 "task already dispatching, skipping duplicate"
2233 );
2234 } else {
2235 tracing::warn!(
2236 task_id = task.id.0,
2237 existing_task_id = existing_id,
2238 dispatch_key,
2239 "dispatch key collision: unexpected task already holds this key"
2240 );
2241 }
2242 continue;
2243 }
2244 Entry::Vacant(slot) => {
2245 slot.insert(task.id.0.clone());
2246 tracing::info!(task_id = task.id.0, dispatch_key, "dispatch guard claimed");
2253 }
2254 }
2255 }
2256 let dispatch_guard = DispatchGuard::new(Arc::clone(dispatching), dispatch_key.clone());
2258 let task_id = task.id.0.clone();
2259
2260 let route_result = match get_route_result(store, repo, &task_id).await {
2263 Ok(r) => Some(r),
2264 Err(RouteResultError::NoAgent { .. }) => {
2265 tracing::warn!(
2266 task_id,
2267 "routed task missing agent — resetting to new for re-routing (#1604)"
2268 );
2269 if let Err(e2) = task_manager.update_task_status(&task.id, Status::New).await {
2270 tracing::error!(task_id, error = %e2, "failed to reset task to new");
2271 }
2272 continue; }
2274 Err(e) => {
2275 tracing::warn!(task_id, error = %e, "get_route_result failed — dispatching without route info");
2276 None
2277 }
2278 };
2279
2280 if let Some(route) = route_result.as_ref() {
2286 let agent_cooled = is_agent_in_cooldown(&route.agent);
2287 let model_cooled = route
2288 .model
2289 .as_deref()
2290 .is_some_and(|model| is_model_in_cooldown(&route.agent, model));
2291 if agent_cooled || model_cooled {
2292 tracing::warn!(
2293 task_id,
2294 agent = route.agent,
2295 model = route.model.as_deref().unwrap_or(""),
2296 "routed agent/model entered cooldown after routing — deferring dispatch to next tick"
2297 );
2298 continue; }
2300 }
2301
2302 let session_name = tmux.session_name(repo, &task_id);
2304 if tmux
2305 .session_blocks_dispatch_from_map(&session_name, session_map)
2306 .await
2307 {
2308 tracing::warn!(
2309 task_id,
2310 session_name,
2311 "task has existing tmux session, skipping dispatch"
2312 );
2313 continue; }
2315
2316 let Some(permit) = priority::acquire_dispatch_slot(semaphore, priority) else {
2318 tracing::debug!("all parallel slots busy, skipping remaining tasks");
2319 break; };
2321
2322 let set_in_progress_result = task_manager
2324 .update_task_status(&task.id, Status::InProgress)
2325 .await;
2326 if let Err(e) = set_in_progress_result {
2327 tracing::error!(task_id, ?e, "failed to set in_progress, skipping dispatch");
2328 drop(permit);
2329 continue; }
2331 tracing::info!(task_id, "dispatching task");
2332
2333 capture
2335 .register_session(repo, &task_id, &session_name)
2336 .await;
2337
2338 let runner = Arc::clone(runner);
2340 let backend = Arc::clone(backend);
2341 let tmux = Arc::clone(tmux);
2342 let capture = Arc::clone(capture);
2343 let task_id_for_cleanup = task_id.clone();
2344 let task_owned = task;
2345 let weight_tx = weight_tx.clone();
2346 let repo_owned = repo.to_string();
2347 let task_manager_for_spawn = Arc::clone(task_manager);
2348 let store_for_spawn = Arc::clone(store);
2349
2350 let repo_ctx = repo_owned.clone();
2351 tokio::spawn(REPO_CONTEXT.scope(repo_ctx, async move {
2352 let _dispatch_guard = dispatch_guard; let dispatch_start = Instant::now();
2354 match runner
2355 .run_with_context(&task_owned, &backend, &tmux, route_result.as_ref())
2356 .await
2357 {
2358 Ok(signal) => {
2359 tracing::info!(task_id, "task runner completed");
2360
2361 let duration = dispatch_start.elapsed().as_secs_f64();
2362
2363 let enable_review = config::get("workflow.enable_review_agent")
2369 .map_or(true, |v| v != "false");
2370 let has_pr = if enable_review && matches!(signal, WeightSignal::Success { .. })
2372 {
2373 match store_for_spawn.resolve_task_id(&repo_owned, &task_id).await {
2374 Ok(Some(store_id)) => match store_for_spawn.get(store_id).await {
2375 Ok(t) => t.pr_number.is_some(),
2376 Err(e) => {
2377 tracing::warn!(task_id, err = %e, "store.get() failed — defaulting to needs_review");
2378 true
2379 }
2380 },
2381 Ok(None) => false,
2382 Err(e) => {
2383 tracing::warn!(task_id, err = %e, "failed to check PR status — defaulting to needs_review");
2384 true
2385 }
2386 }
2387 } else {
2388 false
2389 };
2390 let display_status = display_status_for(&signal, enable_review, has_pr);
2391
2392 if display_status == "done" {
2393 tracing::info!(
2394 task_id,
2395 "marking done: the run succeeded with no PR to review"
2396 );
2397 }
2398 tracing::info!(task_id, display_status, "task run finished, applying status");
2399
2400 let _ = weight_tx.send(signal).await;
2402
2403 if display_status == "needs_review" {
2407 if let Err(e) = task_manager_for_spawn
2410 .update_task_status_with_duration(
2411 &ExternalId(task_id.clone()),
2412 Status::NeedsReview,
2413 Some(duration),
2414 )
2415 .await
2416 {
2417 tracing::error!(task_id, err = %e, "update_task_status(NeedsReview) failed — task may be stuck");
2418 }
2419 } else {
2421 let final_status = match display_status {
2423 "done" => Status::Done,
2424 "blocked" => Status::Blocked,
2425 _ => Status::New,
2426 };
2427 if let Err(e) = task_manager_for_spawn
2428 .update_task_status_with_duration(
2429 &ExternalId(task_id.clone()),
2430 final_status,
2431 Some(duration),
2432 )
2433 .await
2434 {
2435 tracing::warn!(task_id, ?e, "failed to update task status after completion");
2436 }
2437 }
2438 }
2439 Err(e) => {
2440 tracing::error!(task_id, ?e, "task runner failed");
2441 let duration = dispatch_start.elapsed().as_secs_f64();
2442 if e.downcast_ref::<NoProjectDir>().is_some() {
2443 block_task_without_project(
2445 &task_manager_for_spawn,
2446 &store_for_spawn,
2447 &repo_owned,
2448 &task_id,
2449 )
2450 .await;
2451 } else if is_internal_id(&task_id) {
2452 if let Err(ue) = task_manager_for_spawn
2454 .update_task_status_with_duration(
2455 &ExternalId(task_id.clone()),
2456 Status::NeedsReview,
2457 Some(duration),
2458 )
2459 .await
2460 {
2461 tracing::error!(task_id, err = %ue, "update_task_status(NeedsReview) failed — task may be stuck");
2462 }
2463 } else {
2464 if let Err(comment_err) = backend
2466 .post_comment(
2467 &ExternalId(task_id.clone()),
2468 &format!(
2469 "[{}] error: task runner failed: {e}{}",
2470 chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
2471 engine::orch_footer()
2472 ),
2473 )
2474 .await
2475 {
2476 tracing::warn!(
2477 task_id,
2478 ?comment_err,
2479 "failed to post error comment to GitHub"
2480 );
2481 }
2482 if let Err(ue) = task_manager_for_spawn
2486 .update_task_status_with_duration(
2487 &ExternalId(task_id.clone()),
2488 Status::NeedsReview,
2489 Some(duration),
2490 )
2491 .await
2492 {
2493 tracing::error!(task_id, err = %ue, "update_task_status(NeedsReview) failed — task may be stuck");
2494 }
2495 }
2496 }
2497 }
2498
2499 capture.unregister_session(&repo_owned, &task_id_for_cleanup).await;
2501
2502 drop(permit);
2504 }));
2506 }
2507 Ok(())
2508}
2509
2510async fn block_task_without_project(
2513 task_manager: &Arc<TaskManager>,
2514 store: &Arc<TaskStore>,
2515 repo: &str,
2516 task_id: &str,
2517) {
2518 let store_id = match store.resolve_task_id(repo, task_id).await {
2519 Ok(Some(id)) => id,
2520 Ok(None) => {
2521 tracing::error!(
2522 task_id,
2523 repo,
2524 "cannot block task without project: not in the store"
2525 );
2526 return;
2527 }
2528 Err(e) => {
2529 tracing::error!(task_id, repo, err = %e, "cannot block task without project: store lookup failed");
2530 return;
2531 }
2532 };
2533 let fields = [
2534 (
2535 "block_reason",
2536 serde_json::json!(NoProjectDir::BLOCK_REASON),
2537 ),
2538 (
2539 "last_error",
2540 serde_json::json!(format!("no project directory for repo {repo}")),
2541 ),
2542 ];
2543 if let Err(e) = task_manager
2544 .update_task_status_and_result_by_store_id(
2545 store_id,
2546 repo,
2547 &ExternalId(task_id.to_string()),
2548 Status::Blocked,
2549 &fields,
2550 )
2551 .await
2552 {
2553 tracing::error!(task_id, repo, err = %e, "failed to block task without project");
2554 }
2555}
2556
2557fn skips_agent(labels: &[String]) -> bool {
2558 labels
2559 .iter()
2560 .any(|label| label == "no-agent" || label == "owner:human")
2561}
2562
2563fn group_orphaned_routed_tasks(
2570 all_routed: Vec<Task>,
2571 active_repos: &HashSet<String>,
2572) -> HashMap<String, Vec<Task>> {
2573 let mut by_repo: HashMap<String, Vec<Task>> = HashMap::new();
2574 for task in all_routed
2575 .into_iter()
2576 .filter(|t| !active_repos.contains(&t.repo))
2577 .filter(|t| !skips_agent(&t.labels))
2578 {
2579 by_repo.entry(task.repo.clone()).or_default().push(task);
2580 }
2581 by_repo
2582}
2583
2584#[expect(
2603 clippy::too_many_arguments,
2604 reason = "each argument is engine state the tick passes down"
2605)]
2606pub(crate) async fn dispatch_routed_tasks_global(
2607 tmux: &Arc<TmuxManager>,
2608 capture: &Arc<CaptureService>,
2609 semaphore: &Arc<Semaphore>,
2610 weight_tx: &mpsc::Sender<WeightSignal>,
2611 dispatch_mode: DispatchMode,
2612 dispatching: &Arc<DashMap<String, String>>,
2613 store: &Arc<TaskStore>,
2614 session_map: &HashMap<String, bool>,
2615 active_repos: &HashSet<String>,
2616) -> anyhow::Result<()> {
2617 let all_routed = store.list_all_by_status_global(TaskStatus::Routed).await?;
2618 let by_repo = group_orphaned_routed_tasks(all_routed, active_repos);
2619
2620 if by_repo.is_empty() {
2621 return Ok(());
2622 }
2623
2624 tracing::info!(
2625 repos = by_repo.len(),
2626 "global dispatch sweep: found routed tasks from inactive repos"
2627 );
2628
2629 for (repo, tasks) in by_repo {
2630 let backend: Arc<dyn ExternalBackend> = match GitHubBackend::new(repo.clone()) {
2631 Ok(b) => Arc::new(b),
2632 Err(e) => {
2633 tracing::warn!(repo = %repo, err = %e, "global sweep: failed to construct backend, skipping repo");
2634 continue;
2635 }
2636 };
2637 let task_manager = Arc::new(TaskManager::with_store(
2638 Arc::clone(&backend),
2639 Arc::clone(store),
2640 repo.clone(),
2641 ));
2642 let runner = Arc::new(TaskRunner::new(repo.clone()).with_store(Arc::clone(store)));
2643
2644 if let Err(e) = runner.resolve_project_dir().await
2647 && e.downcast_ref::<NoProjectDir>().is_some()
2648 {
2649 tracing::warn!(repo = %repo, count = tasks.len(), "global sweep: repo has no project directory, blocking its routed tasks");
2650 for task in &tasks {
2651 let ext = task
2652 .external_id
2653 .clone()
2654 .unwrap_or_else(|| format!("internal:{}", task.id));
2655 block_task_without_project(&task_manager, store, &repo, &ext).await;
2656 }
2657 continue;
2658 }
2659
2660 let dispatchable: Vec<ExternalTask> =
2661 tasks.iter().map(tasks::store_task_to_external).collect();
2662
2663 if let Err(e) = dispatch_tasks_for_repo(
2664 &backend,
2665 tmux,
2666 &repo,
2667 &runner,
2668 capture,
2669 semaphore,
2670 &task_manager,
2671 weight_tx,
2672 dispatch_mode,
2673 dispatching,
2674 store,
2675 session_map,
2676 dispatchable,
2677 )
2678 .await
2679 {
2680 tracing::warn!(repo = %repo, err = %e, "global sweep: dispatch failed for repo");
2681 }
2682 }
2683
2684 Ok(())
2685}
2686
2687pub(crate) async fn tick_unblock_parents(
2689 backend: &Arc<dyn ExternalBackend>,
2690 task_manager: &Arc<TaskManager>,
2691 store: &Arc<TaskStore>,
2692 repo: &str,
2693) -> anyhow::Result<()> {
2694 let blocked = task_manager.list_all_by_status(Status::Blocked).await?;
2695
2696 let store_blocked = store.list_by_status(repo, TaskStatus::Blocked).await?;
2700 let has_block_reason: HashSet<String> = store_blocked
2701 .iter()
2702 .filter(|t| t.block_reason.is_some())
2703 .filter_map(|t| t.external_id.clone())
2704 .collect();
2705
2706 let handles: Vec<_> = blocked
2710 .into_iter()
2711 .filter(|task| {
2712 if has_block_reason.contains(&task.id.0) {
2713 tracing::debug!(
2714 task_id = task.id.0,
2715 "skipping blocked task with known block_reason"
2716 );
2717 false
2718 } else {
2719 true
2720 }
2721 })
2722 .map(|task| {
2723 let backend_clone = Arc::clone(backend);
2724 let task_manager_clone = Arc::clone(task_manager);
2725 let store_clone = Arc::clone(store);
2726 let repo_owned = repo.to_string();
2727 tokio::spawn(async move {
2728 let task_id = &task.id;
2729
2730 let mut children: Vec<ExternalId> = Vec::new();
2731 let mut child_statuses: HashMap<
2732 String,
2733 TaskStatus,
2734 > = HashMap::new();
2735
2736 match store_clone.resolve_task_id(&repo_owned, &task_id.0).await {
2737 Ok(Some(parent_store_id)) => {
2738 match store_clone.list_children(&repo_owned, parent_store_id).await {
2739 Ok(store_children) => {
2740 for child in store_children {
2741 let Some(child_external_id) = child.external_id else {
2742 continue;
2743 };
2744 child_statuses.insert(child_external_id.clone(), child.status);
2745 children.push(ExternalId(child_external_id));
2746 }
2747 }
2748 Err(e) => {
2749 tracing::debug!(
2750 task_id = task_id.0,
2751 ?e,
2752 "failed to list store-linked children"
2753 );
2754 }
2755 }
2756 }
2757 Ok(None) => {}
2758 Err(e) => {
2759 tracing::debug!(
2760 task_id = task_id.0,
2761 ?e,
2762 "failed to resolve blocked task in store for child lookup"
2763 );
2764 }
2765 }
2766
2767 if !is_internal_id(&task_id.0) {
2768 match backend_clone.get_sub_issues(task_id).await {
2769 Ok(ids) => {
2770 let mut seen: HashSet<String> =
2771 children.iter().map(|child| child.0.clone()).collect();
2772 for id in ids {
2773 if seen.insert(id.0.clone()) {
2774 children.push(id);
2775 }
2776 }
2777 }
2778 Err(e) => {
2779 tracing::debug!(task_id = task_id.0, ?e, "failed to get sub-issues");
2780 }
2781 }
2782 }
2783
2784 if children.is_empty() {
2786 return;
2787 }
2788
2789 let mut all_done = true;
2792 let child_ext_strs: Vec<String> =
2794 children.iter().map(|c| c.0.clone()).collect();
2795 let child_exts: Vec<&str> =
2796 child_ext_strs.iter().map(String::as_str).collect();
2797 match store_clone
2799 .get_statuses_by_external_ids(&repo_owned, &child_exts)
2800 .await
2801 { Ok(map) => {
2802 for (k, v) in map {
2803 child_statuses.insert(k, v);
2804 }
2805 } _ => {
2806 tracing::debug!(
2807 task_id = task_id.0,
2808 "failed to batch-query store for child statuses; falling back to per-child lookups"
2809 );
2810 }}
2811
2812 for child_id in &children {
2813 if let Some(status) = child_statuses.get(&child_id.0) {
2815 if *status != TaskStatus::Done {
2816 all_done = false;
2817 break;
2818 }
2819 continue;
2820 }
2821
2822 match backend_clone.get_task(child_id).await {
2824 Ok(child) => {
2825 if !child.labels.iter().any(|l| l == Status::Done.as_label()) {
2826 all_done = false;
2827 break;
2828 }
2829 }
2830 Err(e) => {
2831 tracing::debug!(task_id = task_id.0, child = %child_id.0, ?e, "failed to fetch child task from backend — treating as not done");
2832 all_done = false;
2833 break;
2834 }
2835 }
2836 }
2837
2838 if all_done {
2839 tracing::info!(
2840 task_id = task_id.0,
2841 children = children.len(),
2842 "all children done, unblocking parent"
2843 );
2844 if let Err(e) =
2845 task_manager_clone.update_task_status(task_id, Status::New).await
2846 {
2847 tracing::warn!(task_id = task_id.0, ?e, "failed to unblock parent");
2848 }
2849 }
2850 })
2851 })
2852 .collect();
2853
2854 for handle in handles {
2855 if let Err(e) = handle.await {
2856 tracing::warn!(?e, "phase 4 sub-issue check panicked");
2857 }
2858 }
2859 Ok(())
2860}
2861
2862pub(crate) async fn tick_job_scheduler(
2864 jobs_path: &Path,
2865 backend: &Arc<dyn ExternalBackend>,
2866 store: Option<&Arc<TaskStore>>,
2867 repo: &str,
2868 transport: Option<&Arc<Transport>>,
2869) -> anyhow::Result<()> {
2870 jobs::tick(jobs_path, backend, store, repo, transport).await
2871}
2872
2873#[expect(
2883 clippy::too_many_arguments,
2884 reason = "each argument is engine state the tick passes down"
2885)]
2886pub(crate) async fn tick_before_routing(
2887 backend: &Arc<dyn ExternalBackend>,
2888 tmux: &Arc<TmuxManager>,
2889 repo: &str,
2890 capture: &Arc<CaptureService>,
2891 config: &EngineConfig,
2892 task_manager: &Arc<TaskManager>,
2893 dispatching: &Arc<DashMap<String, String>>,
2894 store: &Arc<TaskStore>,
2895 session_map: &HashMap<String, bool>,
2896) {
2897 let mut phases: Vec<(&str, u128)> = Vec::new();
2898 let mut at = Instant::now();
2899
2900 if let Err(e) = tick_check_session_completions(tmux, repo, capture, store).await {
2901 tracing::warn!(err = %e, "session completion phase failed");
2902 }
2903 phases.push(("sessions", at.elapsed().as_millis()));
2904
2905 if !store.is_leader() {
2908 super::log_phase_timings("tick.pre_routing", repo, &phases);
2909 return;
2910 }
2911
2912 at = Instant::now();
2913 if let Err(e) =
2914 tick_detect_silent_agents(tmux, repo, capture, backend, task_manager, config, store).await
2915 {
2916 tracing::warn!(err = %e, "silent-agent detection phase failed");
2917 }
2918 phases.push(("silent_agents", at.elapsed().as_millis()));
2919
2920 at = Instant::now();
2921 if let Err(e) = tick_recover_stuck_tasks(
2922 backend,
2923 tmux,
2924 repo,
2925 task_manager,
2926 config,
2927 store,
2928 session_map,
2929 dispatching,
2930 )
2931 .await
2932 {
2933 tracing::warn!(err = %e, "stuck-task recovery phase failed");
2934 }
2935 phases.push(("stuck_tasks", at.elapsed().as_millis()));
2936
2937 super::log_phase_timings("tick.pre_routing", repo, &phases);
2938}
2939
2940#[expect(
2948 clippy::too_many_arguments,
2949 reason = "each argument is engine state the tick passes down"
2950)]
2951pub(crate) async fn tick_after_routing(
2952 backend: &Arc<dyn ExternalBackend>,
2953 tmux: &Arc<TmuxManager>,
2954 repo: &str,
2955 runner: &Arc<TaskRunner>,
2956 capture: &Arc<CaptureService>,
2957 semaphore: &Arc<Semaphore>,
2958 project_dir: &Path,
2959 task_manager: &Arc<TaskManager>,
2960 weight_tx: &mpsc::Sender<WeightSignal>,
2961 dispatch_mode: DispatchMode,
2962 dispatching: &Arc<DashMap<String, String>>,
2963 store: &Arc<TaskStore>,
2964 transport: Option<&Arc<Transport>>,
2965 session_map: &HashMap<String, bool>,
2966) {
2967 let mut phases: Vec<(&str, u128)> = Vec::new();
2968 let mut at = Instant::now();
2969
2970 if let Err(e) = tick_dispatch_tasks(
2971 backend,
2972 tmux,
2973 repo,
2974 runner,
2975 capture,
2976 semaphore,
2977 task_manager,
2978 weight_tx,
2979 dispatch_mode,
2980 dispatching,
2981 store,
2982 session_map,
2983 )
2984 .await
2985 {
2986 tracing::warn!(err = %e, "dispatch phase failed");
2987 }
2988 phases.push(("dispatch", at.elapsed().as_millis()));
2989
2990 at = Instant::now();
2991 if let Err(e) = tick_unblock_parents(backend, task_manager, store, repo).await {
2992 tracing::warn!(err = %e, "parent-unblock phase failed");
2993 }
2994 phases.push(("unblock_parents", at.elapsed().as_millis()));
2995
2996 at = Instant::now();
2997 let jobs_path = project_dir.join(".orch.yml");
2998 if let Err(e) = tick_job_scheduler(&jobs_path, backend, Some(store), repo, transport).await {
2999 tracing::error!(?e, "job scheduler tick failed");
3000 }
3001 phases.push(("jobs", at.elapsed().as_millis()));
3002
3003 super::log_phase_timings("tick.post_routing", repo, &phases);
3004}
3005
3006#[expect(
3016 clippy::too_many_arguments,
3017 reason = "each argument is engine state the tick passes down"
3018)]
3019pub(crate) async fn tick_all(
3020 engines: &[ProjectEngine],
3021 tmux: &Arc<TmuxManager>,
3022 capture: &Arc<CaptureService>,
3023 semaphore: &Arc<Semaphore>,
3024 config: &EngineConfig,
3025 router: &Arc<RwLock<Router>>,
3026 weight_tx: &mpsc::Sender<WeightSignal>,
3027 dispatching: &Arc<DashMap<String, String>>,
3028 transport: Option<&Arc<Transport>>,
3029 active_repos: &HashSet<String>,
3030 auto_merge_in_flight: &Arc<DashSet<String>>,
3031) {
3032 let _tick_span = tracing::info_span!("engine.tick").entered();
3033
3034 let Some(first) = engines.first() else {
3035 return;
3036 };
3037 let store = &first.store;
3040
3041 for (project, call_type, calls) in http::drain_call_counts().into_iter().take(10) {
3043 tracing::debug!(
3044 project,
3045 call_type,
3046 calls,
3047 "github api calls since last tick"
3048 );
3049 }
3050
3051 let session_map = tmux.batch_session_active().await;
3053
3054 let mut passes = Vec::with_capacity(engines.len());
3055 for engine in engines {
3056 passes.push(REPO_CONTEXT.scope(
3057 engine.repo.clone(),
3058 tick_before_routing(
3059 &engine.backend,
3060 tmux,
3061 &engine.repo,
3062 capture,
3063 config,
3064 &engine.task_manager,
3065 dispatching,
3066 &engine.store,
3067 &session_map,
3068 ),
3069 ));
3070 }
3071 future::join_all(passes).await;
3072
3073 if !store.is_leader() {
3074 return;
3075 }
3076
3077 let at = Instant::now();
3081 {
3082 let router_guard = router.read().await;
3083 tick_global_sweeps(
3084 &first.task_manager,
3085 store,
3086 &router_guard,
3087 dispatching,
3088 auto_merge_in_flight,
3089 active_repos,
3090 )
3091 .await;
3092 }
3093 let sweeps_ms = at.elapsed().as_millis();
3094
3095 let at = Instant::now();
3096 recover_untracked_repo_stuck_tasks(
3097 tmux,
3098 config,
3099 store,
3100 &session_map,
3101 dispatching,
3102 active_repos,
3103 )
3104 .await;
3105 let untracked_ms = at.elapsed().as_millis();
3106
3107 let at = Instant::now();
3108 if let Err(err) = dependencies::reconcile_dependencies(
3109 first.backend.as_ref(),
3110 store,
3111 transport.map(AsRef::as_ref),
3112 )
3113 .await
3114 {
3115 tracing::warn!(err = %err, "dependency reconcile failed");
3116 }
3117 super::log_phase_timings(
3118 "tick.global",
3119 &first.repo,
3120 &[
3121 ("global_sweeps", sweeps_ms),
3122 ("untracked_stuck", untracked_ms),
3123 ("dependencies", at.elapsed().as_millis()),
3124 ],
3125 );
3126
3127 {
3128 let mut router_guard = router.write().await;
3129 let mut routing_quota = RoutingTickQuota::new(router_config::max_tasks_per_routing_tick());
3130 for engine in engines {
3131 let at = Instant::now();
3132 REPO_CONTEXT
3133 .scope(engine.repo.clone(), async {
3134 if let Err(e) = tick_route_tasks(
3135 &engine.backend,
3136 &engine.task_manager,
3137 &mut router_guard,
3138 &engine.store,
3139 &engine.repo,
3140 &mut routing_quota,
3141 )
3142 .await
3143 {
3144 tracing::warn!(err = %e, "routing phase failed");
3145 }
3146 })
3147 .await;
3148 super::log_phase_timings(
3149 "tick.routing",
3150 &engine.repo,
3151 &[("route", at.elapsed().as_millis())],
3152 );
3153 }
3154
3155 if let Err(e) =
3159 route_new_tasks_global(&mut router_guard, store, active_repos, &mut routing_quota).await
3160 {
3161 tracing::warn!(err = %e, "global new-task routing sweep failed");
3162 }
3163 }
3164
3165 let dispatch_mode = dispatch_mode_from_router(&*router.read().await);
3166
3167 let session_map = tmux.batch_session_active().await;
3170
3171 let mut passes = Vec::with_capacity(engines.len());
3172 for engine in engines {
3173 passes.push(REPO_CONTEXT.scope(
3174 engine.repo.clone(),
3175 tick_after_routing(
3176 &engine.backend,
3177 tmux,
3178 &engine.repo,
3179 &engine.runner,
3180 capture,
3181 semaphore,
3182 &engine.project_dir,
3183 &engine.task_manager,
3184 weight_tx,
3185 dispatch_mode,
3186 dispatching,
3187 &engine.store,
3188 transport,
3189 &session_map,
3190 ),
3191 ));
3192 }
3193 future::join_all(passes).await;
3194
3195 if let Err(e) = dispatch_routed_tasks_global(
3201 tmux,
3202 capture,
3203 semaphore,
3204 weight_tx,
3205 dispatch_mode,
3206 dispatching,
3207 store,
3208 &session_map,
3209 active_repos,
3210 )
3211 .await
3212 {
3213 tracing::warn!(err = %e, "global routed-task dispatch sweep failed");
3214 }
3215}
3216
3217pub(crate) fn display_status_for(
3220 signal: &WeightSignal,
3221 enable_review: bool,
3222 has_pr: bool,
3223) -> &'static str {
3224 match signal {
3225 WeightSignal::Success { .. } if enable_review => {
3226 if has_pr {
3227 "needs_review"
3228 } else {
3229 "done"
3230 }
3231 }
3232 WeightSignal::RateLimited { .. } | WeightSignal::Rerouted => "new",
3235 WeightSignal::Blocked => "blocked",
3236 WeightSignal::Failed { .. } => "needs_review",
3238 WeightSignal::None if enable_review => "needs_review",
3240 WeightSignal::Success { .. } | WeightSignal::None => "done",
3241 }
3242}
3243
3244#[cfg(test)]
3245mod tests {
3246 use super::*;
3247 use crate::backends::{ExternalId, ExternalTask, Mention, Status};
3248 use crate::channels::transport::Transport;
3249 use crate::engine::RunTimeouts;
3250 use crate::engine::cooldown;
3251 use crate::engine::router::Router;
3252 use crate::engine::router::RouterConfig;
3253 use crate::engine::tasks::TaskManager;
3254 use crate::store::DepSource;
3255 use crate::store::NewDependency;
3256 use crate::store::NewTask;
3257 use crate::store::StoreRoute;
3258 use crate::store::TaskStatus as DbStatus;
3259 use crate::store::TaskStatus;
3260 use crate::store::TaskStore;
3261 use crate::store::UpsertExternal;
3262 use crate::tmux::{SessionGuard, TmuxManager};
3263 use async_trait::async_trait;
3264 use dashmap::mapref::entry::Entry;
3265 use std::cell::Cell;
3266 use std::fs;
3267 use std::path::PathBuf;
3268 use std::sync::{Arc, Mutex};
3269 use tokio::sync::RwLock;
3270 use tokio::sync::oneshot;
3271 use tokio::task;
3272
3273 #[test]
3274 fn timed_out_run_without_pr_never_becomes_done() {
3275 let failed = WeightSignal::Failed {
3276 agent: "claude".to_string(),
3277 };
3278 assert_eq!(display_status_for(&failed, true, false), "needs_review");
3279 assert_eq!(display_status_for(&failed, false, false), "needs_review");
3280 }
3281
3282 #[test]
3283 fn done_needs_a_successful_run_with_no_pr() {
3284 let ok = WeightSignal::Success {
3285 agent: "claude".to_string(),
3286 };
3287 assert_eq!(display_status_for(&ok, true, true), "needs_review");
3288 assert_eq!(display_status_for(&ok, true, false), "done");
3289 assert_eq!(
3290 display_status_for(&WeightSignal::Rerouted, true, false),
3291 "new"
3292 );
3293 assert_eq!(
3294 display_status_for(&WeightSignal::None, true, false),
3295 "needs_review"
3296 );
3297 }
3298 #[test]
3299 fn skips_agent_for_human_and_no_agent_labels() {
3300 assert!(skips_agent(&["owner:human".to_string()]));
3301 assert!(skips_agent(&["no-agent".to_string()]));
3302 assert!(!skips_agent(&["status:new".to_string()]));
3303 }
3304
3305 fn cooled_review_router_for_test() -> Router {
3309 Router::new_for_test(RouterConfig::default(), vec!["claude".to_string()])
3310 }
3311
3312 struct MockBackend {
3316 blocked_tasks: Vec<ExternalTask>,
3318 sub_issues: HashMap<String, Vec<ExternalId>>,
3320 tasks_by_id: HashMap<String, ExternalTask>,
3322 get_sub_issues_calls: Arc<Mutex<Vec<String>>>,
3324 status_updates: Arc<Mutex<Vec<(String, Status)>>>,
3326 removed_labels: Arc<Mutex<Vec<(String, String)>>>,
3328 posted_comments: Arc<Mutex<Vec<(String, String)>>>,
3330 }
3331
3332 impl MockBackend {
3333 fn new() -> Self {
3334 Self {
3335 blocked_tasks: vec![],
3336 sub_issues: HashMap::new(),
3337 tasks_by_id: HashMap::new(),
3338 get_sub_issues_calls: Arc::new(Mutex::new(vec![])),
3339 status_updates: Arc::new(Mutex::new(vec![])),
3340 removed_labels: Arc::new(Mutex::new(vec![])),
3341 posted_comments: Arc::new(Mutex::new(vec![])),
3342 }
3343 }
3344 }
3345
3346 fn make_task(id: &str, labels: &[&str]) -> ExternalTask {
3347 ExternalTask {
3348 id: ExternalId(id.to_string()),
3349 title: format!("Task {id}"),
3350 body: String::new(),
3351 state: "open".to_string(),
3352 labels: labels.iter().map(ToString::to_string).collect(),
3353 author: "bot".to_string(),
3354 created_at: "2026-01-01T00:00:00Z".to_string(),
3355 updated_at: "2026-01-01T00:00:00Z".to_string(),
3356 url: format!("https://github.com/test/test/issues/{id}"),
3357 }
3358 }
3359
3360 #[async_trait]
3361 impl ExternalBackend for MockBackend {
3362 fn name(&self) -> &'static str {
3363 "mock"
3364 }
3365 async fn create_task(
3366 &self,
3367 _title: &str,
3368 _body: &str,
3369 _labels: &[String],
3370 ) -> anyhow::Result<ExternalId> {
3371 Ok(ExternalId("new".to_string()))
3372 }
3373 async fn get_task(&self, id: &ExternalId) -> anyhow::Result<ExternalTask> {
3374 self.tasks_by_id
3375 .get(&id.0)
3376 .cloned()
3377 .ok_or_else(|| anyhow::anyhow!("task not found: {}", id.0))
3378 }
3379 async fn list_by_status(&self, status: Status) -> anyhow::Result<Vec<ExternalTask>> {
3380 if status == Status::Blocked {
3381 Ok(self.blocked_tasks.clone())
3382 } else {
3383 Ok(vec![])
3384 }
3385 }
3386 async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
3387 Ok(vec![])
3388 }
3389 async fn post_comment(&self, _id: &ExternalId, _body: &str) -> anyhow::Result<()> {
3390 self.posted_comments
3391 .lock()
3392 .unwrap()
3393 .push((_id.0.clone(), _body.to_string()));
3394 Ok(())
3395 }
3396 async fn set_labels(&self, _id: &ExternalId, _labels: &[String]) -> anyhow::Result<()> {
3397 Ok(())
3398 }
3399 async fn remove_label(&self, _id: &ExternalId, _label: &str) -> anyhow::Result<()> {
3400 self.removed_labels
3401 .lock()
3402 .unwrap()
3403 .push((_id.0.clone(), _label.to_string()));
3404 Ok(())
3405 }
3406 async fn get_sub_issues(&self, id: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
3407 self.get_sub_issues_calls.lock().unwrap().push(id.0.clone());
3408 Ok(self.sub_issues.get(&id.0).cloned().unwrap_or_default())
3409 }
3410 async fn create_sub_task(
3411 &self,
3412 _parent: &ExternalId,
3413 _title: &str,
3414 _body: &str,
3415 _labels: &[String],
3416 ) -> anyhow::Result<ExternalId> {
3417 Ok(ExternalId("child".to_string()))
3418 }
3419 async fn ensure_status_label(&self, _label: &str) -> anyhow::Result<()> {
3420 Ok(())
3421 }
3422 async fn has_open_issue_with_title(
3423 &self,
3424 _title: &str,
3425 _label: &str,
3426 ) -> anyhow::Result<bool> {
3427 Ok(false)
3428 }
3429 async fn health_check(&self) -> anyhow::Result<()> {
3430 Ok(())
3431 }
3432 async fn is_pr_merged(&self, _branch: &str) -> anyhow::Result<bool> {
3433 Ok(false)
3434 }
3435 async fn get_authenticated_user(&self) -> anyhow::Result<Option<String>> {
3436 Ok(Some("testbot".to_string()))
3437 }
3438 async fn get_mentions(&self, _since: &str) -> anyhow::Result<Vec<Mention>> {
3439 Ok(vec![])
3440 }
3441 async fn update_status(&self, id: &ExternalId, status: Status) -> anyhow::Result<()> {
3442 self.status_updates
3443 .lock()
3444 .unwrap()
3445 .push((id.0.clone(), status));
3446 Ok(())
3447 }
3448 }
3449
3450 fn make_task_manager(backend: Arc<dyn ExternalBackend>) -> Arc<TaskManager> {
3451 Arc::new(TaskManager::new(backend))
3452 }
3453
3454 #[tokio::test]
3457 async fn unblock_parents_unblocks_when_all_children_done() {
3458 let mut mock = MockBackend::new();
3459
3460 let parent = make_task("10", &["status:blocked"]);
3462 mock.blocked_tasks.push(parent.clone());
3463 mock.sub_issues.insert(
3464 "10".to_string(),
3465 vec![ExternalId("11".to_string()), ExternalId("12".to_string())],
3466 );
3467 mock.tasks_by_id
3468 .insert("11".to_string(), make_task("11", &["status:done"]));
3469 mock.tasks_by_id
3470 .insert("12".to_string(), make_task("12", &["status:done"]));
3471
3472 let status_updates = Arc::clone(&mock.status_updates);
3473 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3474 let task_manager = make_task_manager(Arc::clone(&backend));
3475 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3476
3477 tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3478 .await
3479 .unwrap();
3480
3481 let updates = status_updates.lock().unwrap();
3482 assert_eq!(updates.len(), 1, "parent should be unblocked");
3483 assert_eq!(updates[0], ("10".to_string(), Status::New));
3484 }
3485
3486 #[tokio::test]
3487 async fn unblock_parents_skips_when_child_not_done() {
3488 let mut mock = MockBackend::new();
3489
3490 let parent = make_task("20", &["status:blocked"]);
3491 mock.blocked_tasks.push(parent.clone());
3492 mock.sub_issues.insert(
3493 "20".to_string(),
3494 vec![ExternalId("21".to_string()), ExternalId("22".to_string())],
3495 );
3496 mock.tasks_by_id
3498 .insert("21".to_string(), make_task("21", &["status:done"]));
3499 mock.tasks_by_id
3500 .insert("22".to_string(), make_task("22", &["status:in_progress"]));
3501
3502 let status_updates = Arc::clone(&mock.status_updates);
3503 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3504 let task_manager = make_task_manager(Arc::clone(&backend));
3505 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3506
3507 tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3508 .await
3509 .unwrap();
3510
3511 let updates = status_updates.lock().unwrap();
3512 assert!(
3513 updates.is_empty(),
3514 "should not unblock when a child is still running"
3515 );
3516 }
3517
3518 #[tokio::test]
3519 async fn unblock_parents_skips_task_with_no_children() {
3520 let mut mock = MockBackend::new();
3521
3522 let parent = make_task("30", &["status:blocked"]);
3524 mock.blocked_tasks.push(parent.clone());
3525 let status_updates = Arc::clone(&mock.status_updates);
3528 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3529 let task_manager = make_task_manager(Arc::clone(&backend));
3530 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3531
3532 tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3533 .await
3534 .unwrap();
3535
3536 let updates = status_updates.lock().unwrap();
3537 assert!(
3538 updates.is_empty(),
3539 "should not unblock tasks with no sub-issues"
3540 );
3541 }
3542
3543 #[tokio::test]
3544 async fn unblock_parents_skips_task_with_block_reason_in_store() {
3545 let mut mock = MockBackend::new();
3548
3549 let parent = make_task("60", &["status:blocked"]);
3551 mock.blocked_tasks.push(parent.clone());
3552 mock.sub_issues
3555 .insert("60".to_string(), vec![ExternalId("61".to_string())]);
3556 mock.tasks_by_id
3557 .insert("61".to_string(), make_task("61", &["status:done"]));
3558
3559 let status_updates = Arc::clone(&mock.status_updates);
3560 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3561 let task_manager = make_task_manager(Arc::clone(&backend));
3562
3563 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3565 let task_id = store
3566 .upsert_external(&crate::store::UpsertExternal {
3567 repo: "test/repo",
3568 ext_id: "60",
3569 title: "Task 60",
3570 body: "",
3571 author: "bot",
3572 url: "https://github.com/test/test/issues/60",
3573 labels: &[],
3574 origin: "github",
3575 })
3576 .await
3577 .unwrap();
3578 store
3579 .update_status(task_id, TaskStatus::Blocked)
3580 .await
3581 .unwrap();
3582 store
3583 .set_block_reason(task_id, Some("CI failure limit reached"))
3584 .await
3585 .unwrap();
3586
3587 tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3588 .await
3589 .unwrap();
3590
3591 let updates = status_updates.lock().unwrap();
3592 assert!(
3593 updates.is_empty(),
3594 "task with block_reason should be skipped — no API call, no unblock"
3595 );
3596 }
3597
3598 #[tokio::test]
3599 async fn unblock_parents_unblocks_internal_parent_from_store_children() {
3600 let mock = MockBackend::new();
3601 let get_sub_issues_calls = Arc::clone(&mock.get_sub_issues_calls);
3602 let status_updates = Arc::clone(&mock.status_updates);
3603 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3604 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3605 let task_manager = Arc::new(TaskManager::with_store(
3606 Arc::clone(&backend),
3607 Arc::clone(&store),
3608 "test/repo".to_string(),
3609 ));
3610
3611 let parent_id = store
3612 .create_internal("test/repo", "Parent", "", "job", "daily", None)
3613 .await
3614 .unwrap();
3615 store
3616 .update_status(parent_id, TaskStatus::Blocked)
3617 .await
3618 .unwrap();
3619 let child_id = store
3620 .create(&NewTask {
3621 external_id: Some("91".to_string()),
3622 repo: "test/repo".to_string(),
3623 origin: "github".to_string(),
3624 title: "Child".to_string(),
3625 body: String::new(),
3626 source: String::new(),
3627 source_id: String::new(),
3628 author: String::new(),
3629 url: String::new(),
3630 labels: vec![],
3631 parent_id: Some(parent_id),
3632 })
3633 .await
3634 .unwrap();
3635 store
3636 .update_status(child_id, TaskStatus::Done)
3637 .await
3638 .unwrap();
3639
3640 tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3641 .await
3642 .unwrap();
3643
3644 let parent = store.get(parent_id).await.unwrap();
3645 assert_eq!(parent.status, TaskStatus::New);
3646 assert!(
3647 status_updates.lock().unwrap().is_empty(),
3648 "internal parent should be updated in store without backend mirroring"
3649 );
3650 assert!(
3651 get_sub_issues_calls.lock().unwrap().is_empty(),
3652 "internal parent should not query backend sub-issues"
3653 );
3654 }
3655
3656 #[cfg(test)]
3660 async fn set_task_updated_at_past(store: &TaskStore, task_id: i64) {
3661 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
3662 .bind(task_id)
3663 .execute(store.pool())
3664 .await
3665 .unwrap();
3666 }
3667
3668 async fn set_in_review_entered_at_past(store: &TaskStore, task_id: i64) {
3670 sqlx::query(
3671 "UPDATE task_activity SET timestamp = '2020-01-01T00:00:00Z'
3672 WHERE task_id = ? AND to_status = 'in_review'",
3673 )
3674 .bind(task_id)
3675 .execute(store.pool())
3676 .await
3677 .unwrap();
3678 }
3679
3680 #[tokio::test]
3681 async fn recover_stuck_tasks_resets_external_in_review_to_needs_review() {
3682 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3683 let mock = MockBackend::new();
3684 let status_updates = Arc::clone(&mock.status_updates);
3685 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3686 let task_manager = Arc::new(TaskManager::with_store(
3687 Arc::clone(&backend),
3688 Arc::clone(&store),
3689 "owner/repo".to_string(),
3690 ));
3691 let tmux = Arc::new(TmuxManager::new());
3692 let config = EngineConfig {
3693 no_session_stuck_timeout: 600,
3694 stuck_timeout: 1800,
3695 ..EngineConfig::default()
3696 };
3697
3698 let id = store
3700 .upsert_external(&crate::store::UpsertExternal {
3701 repo: "owner/repo",
3702 ext_id: "99",
3703 title: "Stuck InReview",
3704 body: "",
3705 author: "",
3706 url: "",
3707 labels: &[],
3708 origin: "github",
3709 })
3710 .await
3711 .unwrap();
3712 store.update_status(id, TaskStatus::InReview).await.unwrap();
3713 set_in_review_entered_at_past(&store, id).await;
3714 set_task_updated_at_past(&store, id).await;
3715
3716 tick_recover_stuck_tasks(
3717 &backend,
3718 &tmux,
3719 "owner/repo",
3720 &task_manager,
3721 &config,
3722 &store,
3723 &HashMap::new(),
3724 &Arc::new(DashMap::new()),
3725 )
3726 .await
3727 .unwrap();
3728
3729 let updates = status_updates.lock().unwrap();
3730 assert_eq!(updates.len(), 1, "stuck InReview task should be recovered");
3731 assert_eq!(updates[0].0, "99");
3732 assert_eq!(updates[0].1, Status::NeedsReview);
3733 }
3734
3735 #[tokio::test]
3736 async fn recover_stuck_tasks_skips_recent_in_review() {
3737 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3738 let mock = MockBackend::new();
3739 let status_updates = Arc::clone(&mock.status_updates);
3740 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3741 let task_manager = Arc::new(TaskManager::with_store(
3742 Arc::clone(&backend),
3743 Arc::clone(&store),
3744 "owner/repo".to_string(),
3745 ));
3746 let tmux = Arc::new(TmuxManager::new());
3747 let config = EngineConfig {
3748 no_session_stuck_timeout: 600, stuck_timeout: 1800,
3750 ..EngineConfig::default()
3751 };
3752
3753 let id = store
3755 .upsert_external(&crate::store::UpsertExternal {
3756 repo: "owner/repo",
3757 ext_id: "100",
3758 title: "Recent InReview",
3759 body: "",
3760 author: "",
3761 url: "",
3762 labels: &[],
3763 origin: "github",
3764 })
3765 .await
3766 .unwrap();
3767 store.update_status(id, TaskStatus::InReview).await.unwrap();
3768 set_task_updated_at_past(&store, id).await;
3769
3770 tick_recover_stuck_tasks(
3771 &backend,
3772 &tmux,
3773 "owner/repo",
3774 &task_manager,
3775 &config,
3776 &store,
3777 &HashMap::new(),
3778 &Arc::new(DashMap::new()),
3779 )
3780 .await
3781 .unwrap();
3782
3783 let updates = status_updates.lock().unwrap();
3784 assert!(
3785 updates.is_empty(),
3786 "recent InReview task should not be recovered"
3787 );
3788 }
3789
3790 #[tokio::test]
3791 async fn recover_stuck_tasks_resets_internal_in_review_to_needs_review() {
3792 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3793 let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
3794 let task_manager = Arc::new(TaskManager::with_store(
3795 Arc::clone(&backend),
3796 Arc::clone(&store),
3797 "owner/repo".to_string(),
3798 ));
3799 let tmux = Arc::new(TmuxManager::new());
3800 let config = EngineConfig {
3801 no_session_stuck_timeout: 600,
3802 stuck_timeout: 1800,
3803 ..EngineConfig::default()
3804 };
3805
3806 let id = store
3808 .create_internal("owner/repo", "Internal InReview", "", "cron", "1", None)
3809 .await
3810 .unwrap();
3811 store.update_status(id, TaskStatus::InReview).await.unwrap();
3812 set_in_review_entered_at_past(&store, id).await;
3813 set_task_updated_at_past(&store, id).await;
3814
3815 tick_recover_stuck_tasks(
3816 &backend,
3817 &tmux,
3818 "owner/repo",
3819 &task_manager,
3820 &config,
3821 &store,
3822 &HashMap::new(),
3823 &Arc::new(DashMap::new()),
3824 )
3825 .await
3826 .unwrap();
3827
3828 let task = store.get(id).await.unwrap();
3829 assert_eq!(
3830 task.status,
3831 TaskStatus::NeedsReview,
3832 "stuck internal InReview task should be reset to NeedsReview"
3833 );
3834 }
3835
3836 #[tokio::test]
3837 async fn recover_stuck_tasks_ages_in_review_from_status_entry_not_updated_at() {
3838 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3839 let mock = MockBackend::new();
3840 let status_updates = Arc::clone(&mock.status_updates);
3841 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3842 let task_manager = Arc::new(TaskManager::with_store(
3843 Arc::clone(&backend),
3844 Arc::clone(&store),
3845 "owner/repo".to_string(),
3846 ));
3847 let tmux = Arc::new(TmuxManager::new());
3848 let config = EngineConfig {
3849 no_session_stuck_timeout: 600,
3850 stuck_timeout: 1800,
3851 ..EngineConfig::default()
3852 };
3853
3854 let id = store
3855 .upsert_external(&crate::store::UpsertExternal {
3856 repo: "owner/repo",
3857 ext_id: "102",
3858 title: "Fresh updated_at InReview",
3859 body: "",
3860 author: "",
3861 url: "",
3862 labels: &[],
3863 origin: "github",
3864 })
3865 .await
3866 .unwrap();
3867 store.update_status(id, TaskStatus::InReview).await.unwrap();
3868 set_in_review_entered_at_past(&store, id).await;
3869 store
3870 .set_fields(id, &[("pr_number", serde_json::json!(42))])
3871 .await
3872 .unwrap();
3873
3874 tick_recover_stuck_tasks(
3875 &backend,
3876 &tmux,
3877 "owner/repo",
3878 &task_manager,
3879 &config,
3880 &store,
3881 &HashMap::new(),
3882 &Arc::new(DashMap::new()),
3883 )
3884 .await
3885 .unwrap();
3886
3887 let updates = status_updates.lock().unwrap();
3888 assert_eq!(
3889 updates.len(),
3890 1,
3891 "in_review stuck recovery must age from the status entry even when updated_at is fresh"
3892 );
3893 assert_eq!(updates[0].0, "102");
3894 assert_eq!(updates[0].1, Status::NeedsReview);
3895 }
3896
3897 async fn orphaned_review_status_after_recovery(with_decision: bool) -> TaskStatus {
3900 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3901 let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
3902 let task_manager = Arc::new(TaskManager::with_store(
3903 Arc::clone(&backend),
3904 Arc::clone(&store),
3905 "owner/repo".to_string(),
3906 ));
3907 let tmux = Arc::new(TmuxManager::new());
3908
3909 let id = store
3910 .upsert_external(&crate::store::UpsertExternal {
3911 repo: "owner/repo",
3912 ext_id: "55",
3913 title: "Orphaned review",
3914 body: "",
3915 author: "",
3916 url: "",
3917 labels: &[],
3918 origin: "github",
3919 })
3920 .await
3921 .unwrap();
3922 store.update_status(id, TaskStatus::InReview).await.unwrap();
3923 set_in_review_entered_at_past(&store, id).await;
3925 if with_decision {
3926 store::store_log_activity(
3927 &Some(Arc::clone(&store)),
3928 "owner/repo",
3929 "55",
3930 "review_decision",
3931 None,
3932 None,
3933 None::<&str>,
3934 None::<&str>,
3935 Some(&serde_json::json!({"decision": "approve"})),
3936 )
3937 .await;
3938 }
3939
3940 tick_recover_stuck_tasks(
3941 &backend,
3942 &tmux,
3943 "owner/repo",
3944 &task_manager,
3945 &EngineConfig::default(),
3946 &store,
3947 &HashMap::new(),
3948 &Arc::new(DashMap::new()),
3949 )
3950 .await
3951 .unwrap();
3952
3953 store.get(id).await.unwrap().status
3954 }
3955
3956 #[tokio::test]
3957 async fn stale_in_review_recovery_resets_orphaned_task_without_session() {
3958 assert_eq!(
3959 orphaned_review_status_after_recovery(false).await,
3960 TaskStatus::NeedsReview
3961 );
3962 }
3963
3964 #[tokio::test]
3965 async fn stale_in_review_recovery_leaves_decided_task_waiting_on_merge() {
3966 assert_eq!(
3967 orphaned_review_status_after_recovery(true).await,
3968 TaskStatus::InReview
3969 );
3970 }
3971
3972 #[tokio::test]
3973 async fn recover_stuck_tasks_resets_review_agent_failures() {
3974 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3975 let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
3976 let task_manager = Arc::new(TaskManager::with_store(
3977 Arc::clone(&backend),
3978 Arc::clone(&store),
3979 "owner/repo".to_string(),
3980 ));
3981 let tmux = Arc::new(TmuxManager::new());
3982
3983 let id = store
3984 .upsert_external(&crate::store::UpsertExternal {
3985 repo: "owner/repo",
3986 ext_id: "104",
3987 title: "Orphaned review with failures",
3988 body: "",
3989 author: "",
3990 url: "",
3991 labels: &[],
3992 origin: "github",
3993 })
3994 .await
3995 .unwrap();
3996 store.update_status(id, TaskStatus::InReview).await.unwrap();
3997 store
3998 .set_fields(id, &[("review_agent_failures", serde_json::json!(2))])
3999 .await
4000 .unwrap();
4001 set_in_review_entered_at_past(&store, id).await;
4002
4003 tick_recover_stuck_tasks(
4004 &backend,
4005 &tmux,
4006 "owner/repo",
4007 &task_manager,
4008 &EngineConfig::default(),
4009 &store,
4010 &HashMap::new(),
4011 &Arc::new(DashMap::new()),
4012 )
4013 .await
4014 .unwrap();
4015
4016 let task = store.get(id).await.unwrap();
4017 assert_eq!(task.status, TaskStatus::NeedsReview);
4018 assert_eq!(
4019 task.review_agent_failures, 0,
4020 "a dead review session is an infrastructure event, not an agent failure"
4021 );
4022 }
4023
4024 #[tokio::test]
4025 async fn recover_stuck_tasks_reclaims_hung_in_review_session() {
4026 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4027 let mock = MockBackend::new();
4028 let status_updates = Arc::clone(&mock.status_updates);
4029 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4030 let task_manager = Arc::new(TaskManager::with_store(
4031 Arc::clone(&backend),
4032 Arc::clone(&store),
4033 "owner/repo".to_string(),
4034 ));
4035 let tmux = Arc::new(TmuxManager::new());
4036 let config = EngineConfig {
4037 no_session_stuck_timeout: 600,
4038 stuck_timeout: 1800,
4039 ..EngineConfig::default()
4040 };
4041
4042 let id = store
4043 .upsert_external(&crate::store::UpsertExternal {
4044 repo: "owner/repo",
4045 ext_id: "103",
4046 title: "Hung review session",
4047 body: "",
4048 author: "",
4049 url: "",
4050 labels: &[],
4051 origin: "github",
4052 })
4053 .await
4054 .unwrap();
4055 store.update_status(id, TaskStatus::InReview).await.unwrap();
4056 set_in_review_entered_at_past(&store, id).await;
4057 store
4058 .set_fields(id, &[("pr_number", serde_json::json!(43))])
4059 .await
4060 .unwrap();
4061
4062 let session_name = tmux.session_name("owner/repo", "103-review");
4063 let mut session_map = HashMap::new();
4064 session_map.insert(session_name, true);
4065
4066 tick_recover_stuck_tasks(
4067 &backend,
4068 &tmux,
4069 "owner/repo",
4070 &task_manager,
4071 &config,
4072 &store,
4073 &session_map,
4074 &Arc::new(DashMap::new()),
4075 )
4076 .await
4077 .unwrap();
4078
4079 let updates = status_updates.lock().unwrap();
4080 assert_eq!(
4081 updates.len(),
4082 1,
4083 "a live hung review session must be reclaimed once the in_review entry ages past stuck_timeout"
4084 );
4085 assert_eq!(updates[0].0, "103");
4086 assert_eq!(updates[0].1, Status::NeedsReview);
4087 }
4088
4089 #[tokio::test]
4090 async fn untracked_repo_recovery_resets_stale_in_progress_to_new() {
4091 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4092 let tmux = Arc::new(TmuxManager::new());
4093 let config = EngineConfig {
4094 no_session_stuck_timeout: 600,
4095 stuck_timeout: 1800,
4096 ..EngineConfig::default()
4097 };
4098
4099 let id = store
4101 .upsert_external(&crate::store::UpsertExternal {
4102 repo: "owner/other",
4103 ext_id: "777",
4104 title: "Cross-repo stuck task",
4105 body: "",
4106 author: "",
4107 url: "",
4108 labels: &[],
4109 origin: "github",
4110 })
4111 .await
4112 .unwrap();
4113 store
4114 .update_status(id, TaskStatus::InProgress)
4115 .await
4116 .unwrap();
4117 set_task_updated_at_past(&store, id).await;
4118
4119 let active = HashSet::from(["owner/repo".to_string()]);
4120 recover_untracked_repo_stuck_tasks(
4121 &tmux,
4122 &config,
4123 &store,
4124 &HashMap::new(),
4125 &Arc::new(DashMap::new()),
4126 &active,
4127 )
4128 .await;
4129
4130 let task = store.get(id).await.unwrap();
4131 assert_eq!(
4132 task.status,
4133 TaskStatus::New,
4134 "cross-repo stale in_progress task should be reset to New"
4135 );
4136 assert_eq!(task.route_attempts, 0, "route attempts should be cleared");
4137 assert!(task.agent.is_none(), "agent should be cleared");
4138 assert!(task.model.is_none(), "model should be cleared");
4139 }
4140
4141 #[tokio::test]
4142 async fn untracked_repo_recovery_leaves_active_repos_to_their_own_pass() {
4143 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4144 let tmux = Arc::new(TmuxManager::new());
4145 let config = EngineConfig {
4146 no_session_stuck_timeout: 600,
4147 stuck_timeout: 1800,
4148 ..EngineConfig::default()
4149 };
4150 let id = store
4151 .upsert_external(&crate::store::UpsertExternal {
4152 repo: "owner/other",
4153 ext_id: "778",
4154 title: "Stuck task in an active repo",
4155 body: "",
4156 author: "",
4157 url: "",
4158 labels: &[],
4159 origin: "github",
4160 })
4161 .await
4162 .unwrap();
4163 store
4164 .update_status(id, TaskStatus::InProgress)
4165 .await
4166 .unwrap();
4167 set_task_updated_at_past(&store, id).await;
4168
4169 let active = HashSet::from(["owner/repo".to_string(), "owner/other".to_string()]);
4171 recover_untracked_repo_stuck_tasks(
4172 &tmux,
4173 &config,
4174 &store,
4175 &HashMap::new(),
4176 &Arc::new(DashMap::new()),
4177 &active,
4178 )
4179 .await;
4180
4181 assert_eq!(store.get(id).await.unwrap().status, TaskStatus::InProgress);
4182 }
4183
4184 #[tokio::test]
4185 async fn recover_stuck_tasks_finalizes_open_task_runs_rows() {
4186 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4187 let mock = MockBackend::new();
4188 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4189 let task_manager = Arc::new(TaskManager::with_store(
4190 Arc::clone(&backend),
4191 Arc::clone(&store),
4192 "owner/repo".to_string(),
4193 ));
4194 let tmux = Arc::new(TmuxManager::new());
4195 let config = EngineConfig {
4196 no_session_stuck_timeout: 600,
4197 stuck_timeout: 1800,
4198 ..EngineConfig::default()
4199 };
4200
4201 let id = store
4202 .create_internal("owner/repo", "Stuck with open run", "", "cron", "1", None)
4203 .await
4204 .unwrap();
4205 store
4206 .set_fields(
4207 id,
4208 &[
4209 ("agent", serde_json::json!("claude")),
4210 ("model", serde_json::json!("sonnet")),
4211 ("route_attempts", serde_json::json!(1)),
4212 ],
4213 )
4214 .await
4215 .unwrap();
4216 store
4217 .update_status(id, TaskStatus::InProgress)
4218 .await
4219 .unwrap();
4220 set_task_updated_at_past(&store, id).await;
4221
4222 store
4224 .start_run(&crate::store::StartRun {
4225 task_id: id,
4226 attempt: 1,
4227 run_type: "agent",
4228 agent: "claude",
4229 model: "sonnet",
4230 command: "claude --model sonnet",
4231 prompt: "fix it",
4232 })
4233 .await
4234 .unwrap();
4235
4236 tick_recover_stuck_tasks(
4237 &backend,
4238 &tmux,
4239 "owner/repo",
4240 &task_manager,
4241 &config,
4242 &store,
4243 &HashMap::new(),
4244 &Arc::new(DashMap::new()),
4245 )
4246 .await
4247 .unwrap();
4248
4249 let task = store.get(id).await.unwrap();
4250 assert_eq!(task.status, TaskStatus::New);
4251
4252 let runs = store.get_runs(id).await.unwrap();
4253 assert_eq!(runs.len(), 1);
4254 assert_eq!(runs[0].outcome, "aborted");
4255 assert!(runs[0].completed_at.is_some());
4256 assert_eq!(
4257 runs[0].error,
4258 "stuck-task recovery: internal no session found"
4259 );
4260 }
4261
4262 #[tokio::test]
4279 async fn recover_stuck_tasks_skips_reclaim_while_dispatch_guard_held() {
4280 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4281 let mock = MockBackend::new();
4282 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4283 let task_manager = Arc::new(TaskManager::with_store(
4284 Arc::clone(&backend),
4285 Arc::clone(&store),
4286 "owner/repo".to_string(),
4287 ));
4288 let tmux = Arc::new(TmuxManager::new());
4289 let config = EngineConfig {
4290 no_session_stuck_timeout: 600,
4291 stuck_timeout: 1800,
4292 ..EngineConfig::default()
4293 };
4294
4295 let id = store
4296 .create_internal(
4297 "owner/repo",
4298 "Long-running daily job",
4299 "",
4300 "cron",
4301 "1",
4302 None,
4303 )
4304 .await
4305 .unwrap();
4306 store
4307 .set_fields(
4308 id,
4309 &[
4310 ("agent", serde_json::json!("claude")),
4311 ("model", serde_json::json!("sonnet")),
4312 ("route_attempts", serde_json::json!(1)),
4313 ],
4314 )
4315 .await
4316 .unwrap();
4317 store
4318 .update_status(id, TaskStatus::InProgress)
4319 .await
4320 .unwrap();
4321 set_task_updated_at_past(&store, id).await;
4324
4325 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4328 dispatching.insert(
4329 format!("owner/repo/internal:{id}"),
4330 format!("internal:{id}"),
4331 );
4332
4333 tick_recover_stuck_tasks(
4334 &backend,
4335 &tmux,
4336 "owner/repo",
4337 &task_manager,
4338 &config,
4339 &store,
4340 &HashMap::new(),
4341 &dispatching,
4342 )
4343 .await
4344 .unwrap();
4345
4346 let task = store.get(id).await.unwrap();
4347 assert_eq!(
4348 task.status,
4349 TaskStatus::InProgress,
4350 "task should not be reclaimed while its dispatch guard is still held"
4351 );
4352
4353 let runs = store.get_runs(id).await.unwrap();
4355 assert!(runs.is_empty());
4356
4357 dispatching.clear();
4360 tick_recover_stuck_tasks(
4361 &backend,
4362 &tmux,
4363 "owner/repo",
4364 &task_manager,
4365 &config,
4366 &store,
4367 &HashMap::new(),
4368 &dispatching,
4369 )
4370 .await
4371 .unwrap();
4372 let task = store.get(id).await.unwrap();
4373 assert_eq!(
4374 task.status,
4375 TaskStatus::New,
4376 "task should still be reclaimed once the dispatch guard is released"
4377 );
4378 }
4379
4380 #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
4396 async fn recover_stuck_tasks_never_reclaims_while_guard_held_under_real_concurrency() {
4397 const HAMMER_ITERATIONS: u32 = 300;
4398
4399 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4400 let mock = MockBackend::new();
4401 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4402 let task_manager = Arc::new(TaskManager::with_store(
4403 Arc::clone(&backend),
4404 Arc::clone(&store),
4405 "owner/repo".to_string(),
4406 ));
4407 let tmux = Arc::new(TmuxManager::new());
4408 let config = EngineConfig {
4409 no_session_stuck_timeout: 600,
4410 stuck_timeout: 1800,
4411 ..EngineConfig::default()
4412 };
4413
4414 let id = store
4415 .create_internal(
4416 "owner/repo",
4417 "Long-running daily job",
4418 "",
4419 "cron",
4420 "1",
4421 None,
4422 )
4423 .await
4424 .unwrap();
4425 store
4426 .set_fields(
4427 id,
4428 &[
4429 ("agent", serde_json::json!("claude")),
4430 ("model", serde_json::json!("sonnet")),
4431 ("route_attempts", serde_json::json!(1)),
4432 ],
4433 )
4434 .await
4435 .unwrap();
4436 store
4437 .update_status(id, TaskStatus::InProgress)
4438 .await
4439 .unwrap();
4440 set_task_updated_at_past(&store, id).await;
4441
4442 let dispatch_key = format!("owner/repo/internal:{id}");
4443 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4444
4445 let claim_task_id = format!("internal:{id}");
4452 {
4453 match dispatching.entry(dispatch_key.clone()) {
4454 Entry::Vacant(slot) => {
4455 slot.insert(claim_task_id);
4456 }
4457 Entry::Occupied(_) => panic!("dispatch key already claimed"),
4458 }
4459 }
4460 let guard = DispatchGuard::new(Arc::clone(&dispatching), dispatch_key.clone());
4461
4462 let (release_tx, release_rx) = oneshot::channel::<()>();
4473 let holder = tokio::spawn(async move {
4474 let _ = release_rx.await;
4475 drop(guard);
4476 });
4477
4478 for i in 0..HAMMER_ITERATIONS {
4486 tick_recover_stuck_tasks(
4487 &backend,
4488 &tmux,
4489 "owner/repo",
4490 &task_manager,
4491 &config,
4492 &store,
4493 &HashMap::new(),
4494 &dispatching,
4495 )
4496 .await
4497 .unwrap();
4498
4499 let task = store.get(id).await.unwrap();
4500 assert_eq!(
4501 task.status,
4502 TaskStatus::InProgress,
4503 "task must never be reclaimed while the dispatch guard is deterministically \
4504 still held, even under real cross-thread concurrency (dispatch_key={dispatch_key}, \
4505 iteration={i}, contains_key_now={}, map_len_now={})",
4506 dispatching.contains_key(&dispatch_key),
4507 dispatching.len(),
4508 );
4509 task::yield_now().await;
4510 }
4511
4512 let _ = release_tx.send(());
4514 holder.await.unwrap();
4515
4516 tick_recover_stuck_tasks(
4518 &backend,
4519 &tmux,
4520 "owner/repo",
4521 &task_manager,
4522 &config,
4523 &store,
4524 &HashMap::new(),
4525 &dispatching,
4526 )
4527 .await
4528 .unwrap();
4529 let task = store.get(id).await.unwrap();
4530 assert_eq!(
4531 task.status,
4532 TaskStatus::New,
4533 "task should be reclaimed once the dispatch guard is released"
4534 );
4535 }
4536
4537 #[tokio::test]
4538 async fn recover_stuck_tasks_skips_external_in_review_waiting_for_human_review() {
4539 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4540 let mock = MockBackend::new();
4541 let status_updates = Arc::clone(&mock.status_updates);
4542 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4543 let task_manager = Arc::new(TaskManager::with_store(
4544 Arc::clone(&backend),
4545 Arc::clone(&store),
4546 "owner/repo".to_string(),
4547 ));
4548 let tmux = Arc::new(TmuxManager::new());
4549 let config = EngineConfig {
4550 no_session_stuck_timeout: 600,
4551 stuck_timeout: 1800,
4552 ..EngineConfig::default()
4553 };
4554
4555 let id = store
4556 .upsert_external(&crate::store::UpsertExternal {
4557 repo: "owner/repo",
4558 ext_id: "101",
4559 title: "Waiting For Human Review",
4560 body: "",
4561 author: "",
4562 url: "",
4563 labels: &[],
4564 origin: "github",
4565 })
4566 .await
4567 .unwrap();
4568 store.update_status(id, TaskStatus::InReview).await.unwrap();
4569 set_task_updated_at_past(&store, id).await;
4570
4571 tick_recover_stuck_tasks(
4572 &backend,
4573 &tmux,
4574 "owner/repo",
4575 &task_manager,
4576 &config,
4577 &store,
4578 &HashMap::new(),
4579 &Arc::new(DashMap::new()),
4580 )
4581 .await
4582 .unwrap();
4583
4584 let updates = status_updates.lock().unwrap();
4585 assert!(
4586 updates.is_empty(),
4587 "InReview task waiting for human review should not be reset"
4588 );
4589 }
4590
4591 #[tokio::test]
4592 async fn unblock_parents_no_blocked_tasks() {
4593 let mock = MockBackend::new(); let status_updates = Arc::clone(&mock.status_updates);
4595 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4596 let task_manager = make_task_manager(Arc::clone(&backend));
4597 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4598
4599 tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
4600 .await
4601 .unwrap();
4602
4603 assert!(status_updates.lock().unwrap().is_empty());
4604 }
4605
4606 #[tokio::test]
4607 async fn unblock_parents_handles_multiple_blocked_tasks() {
4608 let mut mock = MockBackend::new();
4609
4610 let t40 = make_task("40", &["status:blocked"]);
4612 mock.blocked_tasks.push(t40);
4613 mock.sub_issues
4614 .insert("40".to_string(), vec![ExternalId("41".to_string())]);
4615 mock.tasks_by_id
4616 .insert("41".to_string(), make_task("41", &["status:done"]));
4617
4618 let t50 = make_task("50", &["status:blocked"]);
4620 mock.blocked_tasks.push(t50);
4621 mock.sub_issues
4622 .insert("50".to_string(), vec![ExternalId("51".to_string())]);
4623 mock.tasks_by_id
4624 .insert("51".to_string(), make_task("51", &["status:in_progress"]));
4625
4626 let status_updates = Arc::clone(&mock.status_updates);
4627 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4628 let task_manager = make_task_manager(Arc::clone(&backend));
4629 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4630
4631 tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
4632 .await
4633 .unwrap();
4634
4635 let updates = status_updates.lock().unwrap();
4636 assert_eq!(updates.len(), 1, "only one parent should be unblocked");
4637 assert_eq!(updates[0].0, "40", "task 40 should be unblocked");
4638 assert_eq!(updates[0].1, Status::New);
4639 }
4640
4641 #[tokio::test]
4644 async fn tick_all_is_a_no_op_without_projects() {
4645 let transport = Arc::new(Transport::new());
4646 let capture = Arc::new(CaptureService::new(Arc::clone(&transport)));
4647 let tmux = Arc::new(TmuxManager::new());
4648 let semaphore = Arc::new(Semaphore::new(1));
4649 let router = Arc::new(RwLock::new(cooled_review_router_for_test()));
4650 let (weight_tx, mut weight_rx) = mpsc::channel(8);
4651 let dispatching = Arc::new(DashMap::new());
4652 let auto_merge_in_flight = Arc::new(DashSet::new());
4653
4654 tick_all(
4655 &[],
4656 &tmux,
4657 &capture,
4658 &semaphore,
4659 &EngineConfig::default(),
4660 &router,
4661 &weight_tx,
4662 &dispatching,
4663 Some(&transport),
4664 &HashSet::new(),
4665 &auto_merge_in_flight,
4666 )
4667 .await;
4668
4669 assert!(dispatching.is_empty(), "no project, so no dispatch claimed");
4670 assert!(
4671 weight_rx.try_recv().is_err(),
4672 "no project, so no weight signal"
4673 );
4674 }
4675
4676 #[serial_test::serial(cooldown_state)]
4681 #[tokio::test]
4682 async fn tick_all_runs_the_global_needs_review_sweep_once_for_two_projects() {
4683 cooldown::reset_global_state().await;
4684
4685 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4686 let id = store
4687 .upsert_external(&UpsertExternal {
4688 repo: "owner/one",
4689 ext_id: "7",
4690 title: "Stale needs_review task",
4691 body: "",
4692 author: "user",
4693 url: "https://github.com/owner/one/issues/7",
4694 labels: &[],
4695 origin: "github",
4696 })
4697 .await
4698 .unwrap();
4699 store
4700 .update_status(id, TaskStatus::NeedsReview)
4701 .await
4702 .unwrap();
4703 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4704 .bind(id)
4705 .execute(store.pool())
4706 .await
4707 .unwrap();
4708
4709 let engines: Vec<ProjectEngine> = ["owner/one", "owner/two"]
4710 .iter()
4711 .map(|repo| {
4712 let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
4713 ProjectEngine {
4714 repo: repo.to_string(),
4715 project_dir: PathBuf::from("/nonexistent"),
4716 backend: Arc::clone(&backend),
4717 task_manager: Arc::new(TaskManager::with_store(
4718 backend,
4719 Arc::clone(&store),
4720 repo.to_string(),
4721 )),
4722 runner: Arc::new(TaskRunner::new(repo.to_string())),
4723 store: Arc::clone(&store),
4724 }
4725 })
4726 .collect();
4727
4728 let mut config = RouterConfig::default();
4730 config
4731 .model_map
4732 .entry("review".to_string())
4733 .or_default()
4734 .insert("claude".to_string(), vec!["sonnet".to_string()]);
4735 let router = Arc::new(RwLock::new(Router::new_for_test(
4736 config,
4737 vec!["claude".to_string()],
4738 )));
4739
4740 let transport = Arc::new(Transport::new());
4741 let capture = Arc::new(CaptureService::new(Arc::clone(&transport)));
4742 let (weight_tx, _weight_rx) = mpsc::channel(8);
4743
4744 tick_all(
4745 &engines,
4746 &Arc::new(TmuxManager::new()),
4747 &capture,
4748 &Arc::new(Semaphore::new(1)),
4749 &EngineConfig::default(),
4750 &router,
4751 &weight_tx,
4752 &Arc::new(DashMap::new()),
4753 Some(&transport),
4754 &engines.iter().map(|e| e.repo.clone()).collect(),
4755 &Arc::new(DashSet::new()),
4756 )
4757 .await;
4758
4759 assert_eq!(
4760 store.get(id).await.unwrap().needs_review_refires,
4761 1,
4762 "two projects must share one pass of the global NeedsReview sweep"
4763 );
4764 }
4765
4766 #[tokio::test]
4769 async fn tick_all_does_not_kill_sessions_it_did_not_create() {
4770 let tmux = Arc::new(TmuxManager::new());
4771 let Ok(session_name) = tmux
4772 .create_session("other/foreign", "3889", "/tmp", "sleep 30", &[])
4773 .await
4774 else {
4775 eprintln!("Skipping test: tmux not available or failed to create test session");
4776 return;
4777 };
4778 let _guard = SessionGuard(session_name.clone());
4779
4780 time::sleep(Duration::from_millis(1100)).await;
4783
4784 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4785 let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
4786 let engines = vec![ProjectEngine {
4787 repo: "owner/one".to_string(),
4788 project_dir: PathBuf::from("/nonexistent"),
4789 backend: Arc::clone(&backend),
4790 task_manager: Arc::new(TaskManager::with_store(
4791 backend,
4792 Arc::clone(&store),
4793 "owner/one".to_string(),
4794 )),
4795 runner: Arc::new(TaskRunner::new("owner/one".to_string())),
4796 store,
4797 }];
4798
4799 let transport = Arc::new(Transport::new());
4800 let capture = Arc::new(CaptureService::new(Arc::clone(&transport)));
4801 let (weight_tx, _weight_rx) = mpsc::channel(8);
4802 let router = Arc::new(RwLock::new(cooled_review_router_for_test()));
4803
4804 tick_all(
4805 &engines,
4806 &tmux,
4807 &capture,
4808 &Arc::new(Semaphore::new(1)),
4809 &EngineConfig::default(),
4810 &router,
4811 &weight_tx,
4812 &Arc::new(DashMap::new()),
4813 Some(&transport),
4814 &engines.iter().map(|e| e.repo.clone()).collect(),
4815 &Arc::new(DashSet::new()),
4816 )
4817 .await;
4818
4819 assert!(
4820 tmux.session_exists(&session_name).await,
4821 "tick_all must not kill a tmux session it did not start"
4822 );
4823 }
4824
4825 #[tokio::test]
4826 async fn detect_silent_agents_skips_backend_for_internal_tasks() {
4827 let transport = Arc::new(Transport::new());
4828 let capture = Arc::new(CaptureService::new(transport));
4829 let mock = Arc::new(MockBackend::new());
4830 let backend: Arc<dyn ExternalBackend> = Arc::<MockBackend>::clone(&mock);
4831 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4832 let task_manager = Arc::new(TaskManager::with_store(
4833 Arc::clone(&backend),
4834 Arc::clone(&store),
4835 "owner/repo".to_string(),
4836 ));
4837 let tmux = Arc::new(TmuxManager::new());
4838 let config = EngineConfig {
4839 silence_grace_period: 0,
4840 ..EngineConfig::default()
4841 };
4842
4843 let internal_id = store
4844 .create_internal("owner/repo", "Silent internal", "", "cron", "1", None)
4845 .await
4846 .unwrap();
4847 store
4848 .update_status(internal_id, TaskStatus::InProgress)
4849 .await
4850 .unwrap();
4851
4852 let task_id = format!("internal:{internal_id}");
4853 capture
4854 .register_session("owner/repo", &task_id, "orch-test-internal")
4855 .await;
4856 time::sleep(Duration::from_millis(1100)).await;
4857
4858 tick_detect_silent_agents(
4859 &tmux,
4860 "owner/repo",
4861 &capture,
4862 &backend,
4863 &task_manager,
4864 &config,
4865 &store,
4866 )
4867 .await
4868 .unwrap();
4869
4870 assert!(
4871 mock.removed_labels.lock().unwrap().is_empty(),
4872 "internal tasks should skip backend label removal"
4873 );
4874 assert!(
4875 mock.posted_comments.lock().unwrap().is_empty(),
4876 "internal tasks should skip backend comments"
4877 );
4878
4879 let task = store.get(internal_id).await.unwrap();
4880 assert!(
4884 matches!(task.status, TaskStatus::Routed | TaskStatus::New),
4885 "internal task should be routed to fallback or reset to new, got {:?}",
4886 task.status
4887 );
4888 }
4889
4890 #[tokio::test]
4891 async fn detect_silent_agents_finalizes_open_task_runs_rows() {
4892 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4893 let mock = MockBackend::new();
4894 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4895 let task_manager = Arc::new(TaskManager::with_store(
4896 Arc::clone(&backend),
4897 Arc::clone(&store),
4898 "owner/repo".to_string(),
4899 ));
4900 let transport = Arc::new(Transport::new());
4901 let capture = Arc::new(CaptureService::new(transport));
4902 let tmux = Arc::new(TmuxManager::new());
4903 let config = EngineConfig {
4904 silence_grace_period: 60,
4905 silence_cooldown: 300,
4906 ..EngineConfig::default()
4907 };
4908
4909 let internal_id = store
4910 .create_internal("owner/repo", "Silent with open run", "", "cron", "1", None)
4911 .await
4912 .unwrap();
4913 store
4914 .set_fields(
4915 internal_id,
4916 &[
4917 ("agent", serde_json::json!("claude")),
4918 ("model", serde_json::json!("sonnet")),
4919 ],
4920 )
4921 .await
4922 .unwrap();
4923 store
4924 .update_status(internal_id, TaskStatus::InProgress)
4925 .await
4926 .unwrap();
4927
4928 let task_id = format!("internal:{internal_id}");
4929 capture
4930 .register_session("owner/repo", &task_id, "orch-test-silent-run")
4931 .await;
4932 capture
4933 .set_buffer_state_for_test(
4934 "owner/repo",
4935 &task_id,
4936 false,
4937 false,
4938 chrono::Utc::now() - chrono::Duration::seconds(120),
4939 )
4940 .await;
4941
4942 store
4944 .start_run(&crate::store::StartRun {
4945 task_id: internal_id,
4946 attempt: 1,
4947 run_type: "agent",
4948 agent: "claude",
4949 model: "sonnet",
4950 command: "claude --model sonnet",
4951 prompt: "fix it",
4952 })
4953 .await
4954 .unwrap();
4955
4956 tick_detect_silent_agents(
4957 &tmux,
4958 "owner/repo",
4959 &capture,
4960 &backend,
4961 &task_manager,
4962 &config,
4963 &store,
4964 )
4965 .await
4966 .unwrap();
4967
4968 let runs = store.get_runs(internal_id).await.unwrap();
4969 assert_eq!(runs.len(), 1);
4970 assert_eq!(runs[0].outcome, "aborted");
4971 assert!(runs[0].completed_at.is_some());
4972 assert_eq!(runs[0].error, "silence detection: session killed");
4973 }
4974
4975 #[tokio::test]
4986 async fn phase1_session_completion_touch_prevents_phase2_reclaim() {
4987 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4988 let mock = MockBackend::new();
4989 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4990 let task_manager = Arc::new(TaskManager::with_store(
4991 Arc::clone(&backend),
4992 Arc::clone(&store),
4993 "owner/repo".to_string(),
4994 ));
4995 let tmux = Arc::new(TmuxManager::new());
4996 let config = EngineConfig {
4997 no_session_stuck_timeout: 600,
4998 stuck_timeout: 1800,
4999 ..EngineConfig::default()
5000 };
5001
5002 let id = store
5006 .create_internal("owner/repo", "Completed task", "", "cron", "1", None)
5007 .await
5008 .unwrap();
5009 store
5010 .update_status(id, TaskStatus::InProgress)
5011 .await
5012 .unwrap();
5013 set_task_updated_at_past(&store, id).await;
5014
5015 let task_id = format!("internal:{id}");
5018 store_touch_updated_at(&Some(Arc::clone(&store)), "owner/repo", &task_id).await;
5019
5020 tick_recover_stuck_tasks(
5022 &backend,
5023 &tmux,
5024 "owner/repo",
5025 &task_manager,
5026 &config,
5027 &store,
5028 &HashMap::new(),
5029 &Arc::new(DashMap::new()),
5030 )
5031 .await
5032 .unwrap();
5033
5034 let task = store.get(id).await.unwrap();
5035 assert_eq!(
5036 task.status,
5037 TaskStatus::InProgress,
5038 "Phase 2 must not reclaim a task whose updated_at was just touched by Phase 1"
5039 );
5040 }
5041
5042 #[tokio::test]
5043 async fn stuck_task_timing_treats_dead_session_as_missing() {
5044 let tmux = Arc::new(TmuxManager::new());
5045 let repo = "owner/repo";
5046 let task_id = "internal:33498";
5047 let session_name = tmux.session_name(repo, task_id);
5048
5049 let create_result = tmux
5050 .command()
5051 .args(["new-session", "-d", "-s", &session_name, "-c", "/tmp"])
5052 .output()
5053 .await;
5054
5055 match create_result {
5056 Ok(o) if o.status.success() => {}
5057 _ => {
5058 eprintln!("Skipping test: tmux not available or failed to create test session");
5059 return;
5060 }
5061 }
5062
5063 let _guard = SessionGuard(session_name.clone());
5064
5065 let set_option_result = tmux
5066 .command()
5067 .args(["set-option", "-t", &session_name, "remain-on-exit", "on"])
5068 .output()
5069 .await;
5070 if !matches!(set_option_result, Ok(ref o) if o.status.success()) {
5071 eprintln!("Skipping test: unable to set tmux remain-on-exit option");
5072 return;
5073 }
5074
5075 let send_exit_result = tmux
5076 .command()
5077 .args(["send-keys", "-t", &session_name, "exit", "Enter"])
5078 .output()
5079 .await;
5080 if !matches!(send_exit_result, Ok(ref o) if o.status.success()) {
5081 eprintln!("Skipping test: unable to exit tmux pane");
5082 return;
5083 }
5084
5085 for _ in 0..20 {
5087 time::sleep(Duration::from_millis(100)).await;
5088 if !tmux.session_is_running(&session_name).await {
5089 break;
5090 }
5091 }
5092
5093 let mut session_map = tmux.batch_session_active().await;
5098 for _ in 0..20 {
5099 if !session_map.get(&session_name).copied().unwrap_or(false) {
5100 break;
5101 }
5102 time::sleep(Duration::from_millis(100)).await;
5103 session_map = tmux.batch_session_active().await;
5104 }
5105 if session_map.get(&session_name).copied().unwrap_or(false) {
5106 eprintln!(
5107 "Skipping test: session still alive in batch_session_active after retries (CI tmux lag)"
5108 );
5109 return;
5110 }
5111
5112 let config = EngineConfig {
5113 no_session_stuck_timeout: 600,
5114 stuck_timeout: 1800,
5115 ..EngineConfig::default()
5116 };
5117 let updated_at = (chrono::Utc::now()
5118 - chrono::Duration::seconds(
5119 i64::try_from(config.no_session_stuck_timeout).expect("timeout fits in i64") + 5,
5120 ))
5121 .to_rfc3339();
5122
5123 let timing = stuck_task_timing_from_map(
5124 &tmux,
5125 repo,
5126 task_id,
5127 &updated_at,
5128 &config,
5129 "parse failure",
5130 &session_map,
5131 )
5132 .expect("dead session should be treated as missing");
5133
5134 assert!(!timing.has_session);
5135 assert_eq!(timing.threshold, config.no_session_stuck_timeout);
5136 }
5137
5138 #[tokio::test]
5139 #[serial_test::serial(suspend_state)]
5140 async fn stuck_task_timing_discounts_host_suspend_gap() {
5141 super::super::suspend::clear_for_test();
5142
5143 let tmux = Arc::new(TmuxManager::new());
5144 let repo = "owner/repo";
5145 let task_id = "internal:99001";
5146 let session_map = HashMap::new();
5147
5148 let config = EngineConfig {
5149 no_session_stuck_timeout: 1800,
5150 stuck_timeout: 1800,
5151 ..EngineConfig::default()
5152 };
5153
5154 let updated_at = (chrono::Utc::now() - chrono::Duration::minutes(33)).to_rfc3339();
5158 let suspend_gap_detected_at = chrono::Utc::now() - chrono::Duration::minutes(1);
5159 super::super::suspend::inject_gap_for_test(
5160 suspend_gap_detected_at,
5161 chrono::Duration::minutes(30),
5162 );
5163
5164 let timing = stuck_task_timing_from_map(
5165 &tmux,
5166 repo,
5167 task_id,
5168 &updated_at,
5169 &config,
5170 "parse failure",
5171 &session_map,
5172 );
5173
5174 assert!(
5175 timing.is_none(),
5176 "task age should be discounted below the stuck threshold once suspend time is subtracted, got {timing:?}"
5177 );
5178
5179 super::super::suspend::clear_for_test();
5180 }
5181
5182 #[tokio::test]
5183 async fn recover_stuck_tasks_reclaims_routed_task_blocked_by_stale_session() {
5184 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5185 let mock = MockBackend::new();
5186 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
5187 let task_manager = Arc::new(TaskManager::with_store(
5188 Arc::clone(&backend),
5189 Arc::clone(&store),
5190 "owner/repo".to_string(),
5191 ));
5192 let tmux = Arc::new(TmuxManager::new());
5193 let config = EngineConfig {
5194 no_session_stuck_timeout: 600,
5195 stuck_timeout: 1,
5196 ..EngineConfig::default()
5197 };
5198
5199 let id = store
5200 .create_internal("owner/repo", "Blocked routed task", "", "cron", "1", None)
5201 .await
5202 .unwrap();
5203 store.update_status(id, TaskStatus::Routed).await.unwrap();
5204 set_task_updated_at_past(&store, id).await;
5205
5206 let task_id = format!("internal:{id}");
5207 let session_name = tmux.session_name("owner/repo", &task_id);
5208 let create_result = tmux
5209 .command()
5210 .args([
5211 "new-session",
5212 "-d",
5213 "-s",
5214 &session_name,
5215 "-c",
5216 "/tmp",
5217 "sleep 60",
5218 ])
5219 .output()
5220 .await;
5221
5222 match create_result {
5223 Ok(o) if o.status.success() => {}
5224 _ => {
5225 eprintln!("Skipping test: tmux not available or failed to create test session");
5226 return;
5227 }
5228 }
5229
5230 let session_map = tmux.batch_session_active().await;
5231 assert_eq!(session_map.get(&session_name).copied(), Some(true));
5232
5233 tick_recover_stuck_tasks(
5234 &backend,
5235 &tmux,
5236 "owner/repo",
5237 &task_manager,
5238 &config,
5239 &store,
5240 &session_map,
5241 &Arc::new(DashMap::new()),
5242 )
5243 .await
5244 .unwrap();
5245
5246 let task = store.get(id).await.unwrap();
5247 assert_eq!(
5248 task.status,
5249 TaskStatus::New,
5250 "routed task blocked by a stale session should be reset for re-routing"
5251 );
5252 assert!(
5253 !tmux.session_exists(&session_name).await,
5254 "recovery should kill the stale tmux session so dispatch can proceed"
5255 );
5256 }
5257
5258 async fn live_complex_run(
5261 store: &TaskStore,
5262 tmux: &TmuxManager,
5263 minutes: i64,
5264 updated_minutes_ago: i64,
5265 ) -> Option<(i64, String)> {
5266 let id = store
5267 .create_internal("owner/repo", "Long complex run", "", "cron", "1", None)
5268 .await
5269 .unwrap();
5270 store
5271 .set_fields(id, &[("complexity", serde_json::json!("complex"))])
5272 .await
5273 .unwrap();
5274 store
5275 .update_status(id, TaskStatus::InProgress)
5276 .await
5277 .unwrap();
5278 let ago = |m: i64| {
5279 (chrono::Utc::now() - chrono::Duration::minutes(m))
5280 .format("%Y-%m-%dT%H:%M:%SZ")
5281 .to_string()
5282 };
5283 let entered = ago(minutes);
5284 sqlx::query(
5285 "UPDATE task_activity SET timestamp = ? WHERE task_id = ? AND to_status = 'in_progress'",
5286 )
5287 .bind(&entered)
5288 .bind(id)
5289 .execute(store.pool())
5290 .await
5291 .unwrap();
5292 sqlx::query("UPDATE tasks SET updated_at = ? WHERE id = ?")
5293 .bind(ago(updated_minutes_ago))
5294 .bind(id)
5295 .execute(store.pool())
5296 .await
5297 .unwrap();
5298
5299 let session_name = tmux.session_name("owner/repo", &format!("internal:{id}"));
5300 let created = tmux
5301 .command()
5302 .args([
5303 "new-session",
5304 "-d",
5305 "-s",
5306 &session_name,
5307 "-c",
5308 "/tmp",
5309 "sleep 60",
5310 ])
5311 .output()
5312 .await;
5313 match created {
5314 Ok(o) if o.status.success() => Some((id, session_name)),
5315 _ => None,
5316 }
5317 }
5318
5319 fn long_run_config() -> EngineConfig {
5320 EngineConfig {
5321 stuck_timeout: 1800,
5322 run_timeouts: RunTimeouts {
5323 base: 1800,
5324 medium: 2700,
5325 complex: 3600,
5326 },
5327 ..EngineConfig::default()
5328 }
5329 }
5330
5331 #[test]
5332 fn tick_router_loads_the_config_once_for_many_live_runs() {
5333 let cell = OnceLock::new();
5334 let loads = Cell::new(0);
5335 for _ in 0..4 {
5336 tick_router(&cell, || {
5337 loads.set(loads.get() + 1);
5338 RouterConfig::default()
5339 });
5340 }
5341 assert_eq!(loads.get(), 1);
5342 }
5343
5344 #[tokio::test]
5345 async fn run_tier_follows_the_tier_of_the_stored_model() {
5346 let mut router = RouterConfig::default();
5347 router.model_map.clear();
5348 for (tier, model) in [("simple", "rt-s"), ("complex", "rt-c")] {
5349 router
5350 .model_map
5351 .entry(tier.to_string())
5352 .or_default()
5353 .insert("rtagent".to_string(), vec![model.to_string()]);
5354 }
5355 let store = TaskStore::open_memory().await.unwrap();
5356 let id = store
5357 .create_internal(
5358 "owner/repo",
5359 "Simple on a complex model",
5360 "",
5361 "cron",
5362 "1",
5363 None,
5364 )
5365 .await
5366 .unwrap();
5367 store
5368 .set_fields(
5369 id,
5370 &[
5371 ("complexity", serde_json::json!("simple")),
5372 ("agent", serde_json::json!("rtagent")),
5373 ("model", serde_json::json!("rt-c")),
5374 ],
5375 )
5376 .await
5377 .unwrap();
5378 let task = store.get(id).await.unwrap();
5379 assert_eq!(run_tier(&router, &task), "complex");
5380
5381 store
5382 .set_fields(id, &[("model", serde_json::json!("rt-s"))])
5383 .await
5384 .unwrap();
5385 let task = store.get(id).await.unwrap();
5386 assert_eq!(run_tier(&router, &task), "simple");
5387 }
5388
5389 #[tokio::test]
5390 async fn recover_stuck_tasks_keeps_a_live_complex_run_inside_its_timeout() {
5391 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5392 let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
5393 let task_manager = Arc::new(TaskManager::with_store(
5394 Arc::clone(&backend),
5395 Arc::clone(&store),
5396 "owner/repo".to_string(),
5397 ));
5398 let tmux = Arc::new(TmuxManager::new());
5399 let Some((id, session_name)) = live_complex_run(&store, &tmux, 31, 31).await else {
5400 eprintln!("Skipping test: tmux not available or failed to create test session");
5401 return;
5402 };
5403 let session_map = tmux.batch_session_active().await;
5404
5405 tick_recover_stuck_tasks(
5406 &backend,
5407 &tmux,
5408 "owner/repo",
5409 &task_manager,
5410 &long_run_config(),
5411 &store,
5412 &session_map,
5413 &Arc::new(DashMap::new()),
5414 )
5415 .await
5416 .unwrap();
5417
5418 let alive = tmux.session_exists(&session_name).await;
5419 let _ = tmux.kill_session(&session_name).await;
5420 assert_eq!(
5421 store.get(id).await.unwrap().status,
5422 TaskStatus::InProgress,
5423 "a complex run at 31 minutes is inside its 3600s timeout"
5424 );
5425 assert!(alive, "recovery must not touch a run inside its timeout");
5426 }
5427
5428 #[tokio::test]
5429 async fn recover_stuck_tasks_kills_a_live_complex_run_past_its_timeout() {
5430 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5431 let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
5432 let task_manager = Arc::new(TaskManager::with_store(
5433 Arc::clone(&backend),
5434 Arc::clone(&store),
5435 "owner/repo".to_string(),
5436 ));
5437 let tmux = Arc::new(TmuxManager::new());
5438 let Some((id, session_name)) = live_complex_run(&store, &tmux, 66, 0).await else {
5439 eprintln!("Skipping test: tmux not available or failed to create test session");
5440 return;
5441 };
5442 let session_map = tmux.batch_session_active().await;
5443
5444 tick_recover_stuck_tasks(
5445 &backend,
5446 &tmux,
5447 "owner/repo",
5448 &task_manager,
5449 &long_run_config(),
5450 &store,
5451 &session_map,
5452 &Arc::new(DashMap::new()),
5453 )
5454 .await
5455 .unwrap();
5456
5457 let alive = tmux.session_exists(&session_name).await;
5458 let _ = tmux.kill_session(&session_name).await;
5459 assert_eq!(
5460 store.get(id).await.unwrap().status,
5461 TaskStatus::New,
5462 "a complex run past its timeout plus grace is stuck"
5463 );
5464 assert!(
5465 !alive,
5466 "recovery must kill the session it reports as killed"
5467 );
5468 }
5469
5470 #[tokio::test]
5484 async fn dispatch_does_not_deadlock_under_write_lock() {
5485 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5486 let mock = MockBackend::new();
5487 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
5488 let task_manager = Arc::new(TaskManager::with_store(
5489 Arc::clone(&backend),
5490 Arc::clone(&store),
5491 "owner/repo".to_string(),
5492 ));
5493 let tmux = Arc::new(TmuxManager::new());
5494 let semaphore = Arc::new(Semaphore::new(4));
5495 let (weight_tx, _weight_rx) = mpsc::channel(16);
5496 let dispatching = Arc::new(DashMap::new());
5497 let transport = Arc::new(Transport::new());
5498 let capture = Arc::new(CaptureService::new(transport));
5499
5500 let router_config = RouterConfig::default();
5501 let router = Router::new_for_test(router_config, vec!["claude".to_string()]);
5502 let router_arc = Arc::new(RwLock::new(router));
5503
5504 let write_guard = router_arc.write().await;
5508 let dispatch_mode = dispatch_mode_from_router(&write_guard);
5509
5510 let result = time::timeout(
5511 Duration::from_secs(2),
5512 tick_dispatch_tasks(
5513 &backend,
5514 &tmux,
5515 "owner/repo",
5516 &Arc::new(TaskRunner::new("owner/repo".to_string())),
5517 &capture,
5518 &semaphore,
5519 &task_manager,
5520 &weight_tx,
5521 dispatch_mode,
5522 &dispatching,
5523 &store,
5524 &HashMap::new(),
5525 ),
5526 )
5527 .await;
5528
5529 assert!(
5530 result.is_ok(),
5531 "tick_dispatch_tasks deadlocked! It tried to acquire a read lock \
5532 on router_arc while a write lock was already held (issue #1361)"
5533 );
5534 }
5535
5536 #[tokio::test]
5544 async fn dispatch_skips_task_whose_routed_model_entered_cooldown() {
5545 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5546 let mock = MockBackend::new();
5547 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
5548 let task_manager = Arc::new(TaskManager::with_store(
5549 Arc::clone(&backend),
5550 Arc::clone(&store),
5551 "owner/repo".to_string(),
5552 ));
5553 let tmux = Arc::new(TmuxManager::new());
5554 let runner = Arc::new(TaskRunner::new("owner/repo".to_string()));
5555 let semaphore = Arc::new(Semaphore::new(4));
5556 let (weight_tx, _weight_rx) = mpsc::channel(16);
5557 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
5558 let transport = Arc::new(Transport::new());
5559 let capture = Arc::new(CaptureService::new(transport));
5560
5561 let id = store
5562 .upsert_external(&UpsertExternal {
5563 repo: "owner/repo",
5564 ext_id: "999",
5565 title: "Stale routed task",
5566 body: "",
5567 author: "user",
5568 url: "https://github.com/owner/repo/issues/999",
5569 labels: &[],
5570 origin: "github",
5571 })
5572 .await
5573 .unwrap();
5574 store
5575 .store_route(&StoreRoute {
5576 id,
5577 agent: "minimax",
5578 model: Some("opus"),
5579 complexity: "medium",
5580 estimate: 0,
5581 reason: "test",
5582 profile: "{}",
5583 skills: "[]",
5584 })
5585 .await
5586 .unwrap();
5587 store.update_status(id, DbStatus::Routed).await.unwrap();
5588
5589 cooldown::set_model_cooldown("minimax", "opus", 300, "rate_limit_usage_window").await;
5592
5593 let dispatch_mode = DispatchMode {
5594 is_degraded: false,
5595 healthy_agents: 1,
5596 threshold: 1,
5597 };
5598
5599 tick_dispatch_tasks(
5600 &backend,
5601 &tmux,
5602 "owner/repo",
5603 &runner,
5604 &capture,
5605 &semaphore,
5606 &task_manager,
5607 &weight_tx,
5608 dispatch_mode,
5609 &dispatching,
5610 &store,
5611 &HashMap::new(),
5612 )
5613 .await
5614 .unwrap();
5615
5616 let task = store.get(id).await.unwrap();
5617 assert_eq!(
5618 task.status,
5619 DbStatus::Routed,
5620 "task whose routed model is now cooled down must stay Routed, not be dispatched"
5621 );
5622 assert!(
5623 dispatching.is_empty(),
5624 "dispatch guard must be released when skipping a stale-cooldown task"
5625 );
5626 }
5627
5628 #[tokio::test]
5629 async fn dispatch_skips_a_task_with_an_open_dependency() {
5630 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5631 let mock = MockBackend::new();
5632 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
5633 let task_manager = Arc::new(TaskManager::with_store(
5634 Arc::clone(&backend),
5635 Arc::clone(&store),
5636 "owner/repo".to_string(),
5637 ));
5638 let tmux = Arc::new(TmuxManager::new());
5639 let runner = Arc::new(TaskRunner::new("owner/repo".to_string()));
5640 let semaphore = Arc::new(Semaphore::new(4));
5641 let (weight_tx, _weight_rx) = mpsc::channel(16);
5642 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
5643 let transport = Arc::new(Transport::new());
5644 let capture = Arc::new(CaptureService::new(transport));
5645
5646 let id = store
5647 .upsert_external(&UpsertExternal {
5648 repo: "owner/repo",
5649 ext_id: "42",
5650 title: "Waiting routed task",
5651 body: "",
5652 author: "user",
5653 url: "https://github.com/owner/repo/issues/42",
5654 labels: &[],
5655 origin: "github",
5656 })
5657 .await
5658 .unwrap();
5659 store
5660 .replace_dependencies(
5661 id,
5662 &[NewDependency {
5663 repo: "owner/repo".into(),
5664 number: 7,
5665 source: DepSource::Github,
5666 }],
5667 )
5668 .await
5669 .unwrap();
5670 store
5671 .store_route(&StoreRoute {
5672 id,
5673 agent: "claude",
5674 model: Some("haiku"),
5675 complexity: "simple",
5676 estimate: 0,
5677 reason: "test",
5678 profile: "{}",
5679 skills: "[]",
5680 })
5681 .await
5682 .unwrap();
5683 store.update_status(id, DbStatus::Routed).await.unwrap();
5684
5685 let dispatch_mode = DispatchMode {
5686 is_degraded: false,
5687 healthy_agents: 1,
5688 threshold: 1,
5689 };
5690 tick_dispatch_tasks(
5691 &backend,
5692 &tmux,
5693 "owner/repo",
5694 &runner,
5695 &capture,
5696 &semaphore,
5697 &task_manager,
5698 &weight_tx,
5699 dispatch_mode,
5700 &dispatching,
5701 &store,
5702 &HashMap::new(),
5703 )
5704 .await
5705 .unwrap();
5706
5707 let task = store.get(id).await.unwrap();
5708 assert_eq!(task.status, DbStatus::Waiting);
5709 assert!(dispatching.is_empty());
5710 }
5711
5712 #[tokio::test]
5725 async fn completed_session_does_not_cause_stuck_reclaim_race() {
5726 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5727 let mock = MockBackend::new();
5728 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
5729 let task_manager = Arc::new(TaskManager::with_store(
5730 Arc::clone(&backend),
5731 Arc::clone(&store),
5732 "owner/repo".to_string(),
5733 ));
5734 let tmux = Arc::new(TmuxManager::new());
5735 let transport = Arc::new(Transport::new());
5736 let capture = Arc::new(CaptureService::new(transport));
5737
5738 let id = store
5740 .upsert_external(&crate::store::UpsertExternal {
5741 repo: "owner/repo",
5742 ext_id: "200",
5743 title: "Race Task",
5744 body: "",
5745 author: "",
5746 url: "",
5747 labels: &[],
5748 origin: "github",
5749 })
5750 .await
5751 .unwrap();
5752 store
5753 .update_status(id, TaskStatus::InProgress)
5754 .await
5755 .unwrap();
5756
5757 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
5759 .bind(id)
5760 .execute(store.pool())
5761 .await
5762 .unwrap();
5763
5764 let task_id = "200";
5766 let session_name = tmux.session_name("owner/repo", task_id);
5767 let create_result = tmux
5768 .command()
5769 .args(["new-session", "-d", "-s", &session_name, "-c", "/tmp"])
5770 .output()
5771 .await;
5772 match create_result {
5773 Ok(o) if o.status.success() => {}
5774 _ => {
5775 eprintln!("Skipping test: tmux not available or failed to create test session");
5776 return;
5777 }
5778 }
5779 let _guard = SessionGuard(session_name.clone());
5780
5781 let _ = tmux
5783 .command()
5784 .args(["set-option", "-t", &session_name, "remain-on-exit", "on"])
5785 .output()
5786 .await;
5787 let _ = tmux
5788 .command()
5789 .args(["send-keys", "-t", &session_name, "exit", "Enter"])
5790 .output()
5791 .await;
5792
5793 for _ in 0..20 {
5795 time::sleep(Duration::from_millis(100)).await;
5796 if !tmux.session_is_running(&session_name).await {
5797 break;
5798 }
5799 }
5800
5801 let task_key = task_id.to_string();
5803 capture
5804 .register_session("owner/repo", &task_key, &session_name)
5805 .await;
5806
5807 let config = EngineConfig {
5809 no_session_stuck_timeout: 600,
5810 stuck_timeout: 1800,
5811 ..EngineConfig::default()
5812 };
5813
5814 tick_check_session_completions(&tmux, "owner/repo", &capture, &store)
5815 .await
5816 .unwrap();
5817
5818 {
5824 let task_state = store.get(id).await.unwrap();
5825 if task_state.updated_at.starts_with("2020") {
5826 eprintln!(
5827 "Skipping test: tick_check_session_completions did not update stored \
5828 updated_at — session was not in snapshot (CI tmux environment issue)"
5829 );
5830 return;
5831 }
5832 }
5833
5834 tick_recover_stuck_tasks(
5836 &backend,
5837 &tmux,
5838 "owner/repo",
5839 &task_manager,
5840 &config,
5841 &store,
5842 &HashMap::new(),
5843 &Arc::new(DashMap::new()),
5844 )
5845 .await
5846 .unwrap();
5847
5848 let task = store.get(id).await.unwrap();
5850 assert_eq!(
5851 task.status,
5852 TaskStatus::InProgress,
5853 "task should not be reclaimed to New"
5854 );
5855 }
5856
5857 #[tokio::test]
5867 async fn session_completion_matches_repo_configured_with_git_suffix() {
5868 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5869 let tmux = Arc::new(TmuxManager::new());
5870 let transport = Arc::new(Transport::new());
5871 let capture = Arc::new(CaptureService::new(transport));
5872
5873 let repo = "owner/repo.git";
5874 let id = store
5875 .upsert_external(&crate::store::UpsertExternal {
5876 repo,
5877 ext_id: "201",
5878 title: "Git-suffix repo task",
5879 body: "",
5880 author: "",
5881 url: "",
5882 labels: &[],
5883 origin: "github",
5884 })
5885 .await
5886 .unwrap();
5887 store
5888 .update_status(id, TaskStatus::InProgress)
5889 .await
5890 .unwrap();
5891 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
5892 .bind(id)
5893 .execute(store.pool())
5894 .await
5895 .unwrap();
5896
5897 let task_id = "201";
5900 let session_name = tmux.session_name(repo, task_id);
5901 let create_result = tmux
5902 .command()
5903 .args(["new-session", "-d", "-s", &session_name, "-c", "/tmp"])
5904 .output()
5905 .await;
5906 match create_result {
5907 Ok(o) if o.status.success() => {}
5908 _ => {
5909 eprintln!("Skipping test: tmux not available or failed to create test session");
5910 return;
5911 }
5912 }
5913 let _guard = SessionGuard(session_name.clone());
5914
5915 let _ = tmux
5916 .command()
5917 .args(["set-option", "-t", &session_name, "remain-on-exit", "on"])
5918 .output()
5919 .await;
5920 let _ = tmux
5921 .command()
5922 .args(["send-keys", "-t", &session_name, "exit", "Enter"])
5923 .output()
5924 .await;
5925
5926 for _ in 0..20 {
5927 time::sleep(Duration::from_millis(100)).await;
5928 if !tmux.session_is_running(&session_name).await {
5929 break;
5930 }
5931 }
5932
5933 capture.register_session(repo, task_id, &session_name).await;
5934
5935 tick_check_session_completions(&tmux, repo, &capture, &store)
5936 .await
5937 .unwrap();
5938
5939 let task_state = store.get(id).await.unwrap();
5940 if task_state.updated_at.starts_with("2020") {
5941 eprintln!(
5942 "Skipping test: tick_check_session_completions did not update stored \
5943 updated_at — session was not in snapshot (CI tmux environment issue)"
5944 );
5945 return;
5946 }
5947 assert!(
5948 !task_state.updated_at.starts_with("2020"),
5949 "tick_check_session_completions must match its own repo's live sessions \
5950 even when the repo is configured with a \".git\" suffix"
5951 );
5952 }
5953
5954 #[tokio::test]
5960 async fn job_scheduler_returns_ok_on_duplicate_job_config() {
5961 let tmp = tempfile::tempdir().unwrap();
5962 let cfg = tmp.path().join(".orch.yml");
5963 fs::write(
5964 &cfg,
5965 "jobs:\n - id: trading-scan\n schedule: \"0 9 * * *\"\n task:\n title: Scan\n body: do it\n",
5966 )
5967 .unwrap();
5968 let jobs_dir = tmp.path().join("prompts").join("jobs");
5969 fs::create_dir_all(&jobs_dir).unwrap();
5970 fs::write(
5971 jobs_dir.join("trading-scan.md"),
5972 "---\nid: trading-scan\nschedule: '0 9 * * *'\ntitle: Trading Scan\n---\n\nDo trading scan.\n",
5973 )
5974 .unwrap();
5975
5976 let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
5977 let result = tick_job_scheduler(&cfg, &backend, None, "test/repo", None).await;
5978 assert!(
5979 result.is_ok(),
5980 "duplicate job id must not propagate as Err: {result:?}"
5981 );
5982 }
5983
5984 #[tokio::test]
5988 async fn job_scheduler_processes_valid_project_when_other_has_duplicate() {
5989 let bad_tmp = tempfile::tempdir().unwrap();
5991 let bad_cfg = bad_tmp.path().join(".orch.yml");
5992 fs::write(
5993 &bad_cfg,
5994 "jobs:\n - id: dup\n schedule: \"0 9 * * *\"\n task:\n title: Dup\n body: x\n",
5995 )
5996 .unwrap();
5997 let bad_jobs_dir = bad_tmp.path().join("prompts").join("jobs");
5998 fs::create_dir_all(&bad_jobs_dir).unwrap();
5999 fs::write(
6000 bad_jobs_dir.join("dup.md"),
6001 "---\nid: dup\nschedule: '0 9 * * *'\ntitle: Dup\n---\n\nbody\n",
6002 )
6003 .unwrap();
6004
6005 let good_tmp = tempfile::tempdir().unwrap();
6007 let good_cfg = good_tmp.path().join(".orch.yml");
6008 fs::write(
6009 &good_cfg,
6010 "jobs:\n - id: valid-job\n schedule: \"0 9 * * *\"\n task:\n title: Valid\n body: ok\n",
6011 )
6012 .unwrap();
6013
6014 let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
6015
6016 let bad_result = tick_job_scheduler(&bad_cfg, &backend, None, "test/bad-repo", None).await;
6017 assert!(
6018 bad_result.is_ok(),
6019 "bad project must not abort scheduler: {bad_result:?}"
6020 );
6021
6022 let good_result =
6023 tick_job_scheduler(&good_cfg, &backend, None, "test/good-repo", None).await;
6024 assert!(
6025 good_result.is_ok(),
6026 "valid project must still run after bad project: {good_result:?}"
6027 );
6028 }
6029
6030 #[serial_test::serial(cooldown_state)]
6038 #[tokio::test]
6039 async fn global_sweep_routes_inactive_repo_tasks() {
6040 cooldown::reset_global_state().await;
6041
6042 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6043
6044 let active_id = store
6046 .upsert_external(&UpsertExternal {
6047 repo: "active/repo",
6048 ext_id: "1",
6049 title: "Active task",
6050 body: "",
6051 author: "user",
6052 url: "https://github.com/active/repo/issues/1",
6053 labels: &[],
6054 origin: "github",
6055 })
6056 .await
6057 .unwrap();
6058
6059 let inactive_id = store
6061 .upsert_external(&UpsertExternal {
6062 repo: "inactive/repo",
6063 ext_id: "42",
6064 title: "Orphaned task",
6065 body: "",
6066 author: "user",
6067 url: "https://github.com/inactive/repo/issues/42",
6068 labels: &[],
6069 origin: "github",
6070 })
6071 .await
6072 .unwrap();
6073
6074 assert_eq!(store.get(active_id).await.unwrap().status, TaskStatus::New);
6075 assert_eq!(
6076 store.get(inactive_id).await.unwrap().status,
6077 TaskStatus::New
6078 );
6079
6080 let mut model_map: HashMap<String, HashMap<String, Vec<String>>> = HashMap::new();
6081 let mut tier = HashMap::new();
6082 tier.insert("claude".to_string(), vec!["haiku".to_string()]);
6083 model_map.insert("medium".to_string(), tier);
6084 let config = RouterConfig {
6085 mode: "round_robin".to_string(),
6086 agents: vec!["claude".to_string()],
6087 model_map,
6088 ..Default::default()
6089 };
6090 let mut router = Router::new_for_test(config, vec!["claude".to_string()]);
6091
6092 let active_repos: HashSet<String> = ["active/repo".to_string()].into_iter().collect();
6093 let mut routing_quota = RoutingTickQuota::new(1);
6094 route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
6095 .await
6096 .unwrap();
6097
6098 assert_eq!(
6100 store.get(active_id).await.unwrap().status,
6101 TaskStatus::New,
6102 "global sweep must not touch tasks for the active repo"
6103 );
6104
6105 let after = store.get(inactive_id).await.unwrap();
6107 assert_eq!(
6108 after.status,
6109 TaskStatus::Routed,
6110 "global sweep must route tasks from inactive repos"
6111 );
6112 assert!(
6113 matches!(&after.agent, Some(a) if !a.is_empty()),
6114 "agent must be set after routing"
6115 );
6116 }
6117
6118 #[serial_test::serial(cooldown_state)]
6126 #[tokio::test]
6127 async fn global_sweep_does_not_touch_other_active_repo_tasks() {
6128 cooldown::reset_global_state().await;
6129
6130 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6131
6132 let other_active_id = store
6135 .upsert_external(&UpsertExternal {
6136 repo: "other-active/repo",
6137 ext_id: "7",
6138 title: "Other active project's task",
6139 body: "",
6140 author: "user",
6141 url: "https://github.com/other-active/repo/issues/7",
6142 labels: &[],
6143 origin: "github",
6144 })
6145 .await
6146 .unwrap();
6147
6148 let inactive_id = store
6150 .upsert_external(&UpsertExternal {
6151 repo: "inactive/repo",
6152 ext_id: "42",
6153 title: "Orphaned task",
6154 body: "",
6155 author: "user",
6156 url: "https://github.com/inactive/repo/issues/42",
6157 labels: &[],
6158 origin: "github",
6159 })
6160 .await
6161 .unwrap();
6162
6163 let mut model_map: HashMap<String, HashMap<String, Vec<String>>> = HashMap::new();
6164 let mut tier = HashMap::new();
6165 tier.insert("claude".to_string(), vec!["haiku".to_string()]);
6166 model_map.insert("medium".to_string(), tier);
6167 let config = RouterConfig {
6168 mode: "round_robin".to_string(),
6169 agents: vec!["claude".to_string()],
6170 model_map,
6171 ..Default::default()
6172 };
6173 let mut router = Router::new_for_test(config, vec!["claude".to_string()]);
6174
6175 let active_repos: HashSet<String> =
6179 ["active/repo".to_string(), "other-active/repo".to_string()]
6180 .into_iter()
6181 .collect();
6182 let mut routing_quota = RoutingTickQuota::new(1);
6183 route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
6184 .await
6185 .unwrap();
6186
6187 assert_eq!(
6188 store.get(other_active_id).await.unwrap().status,
6189 TaskStatus::New,
6190 "global sweep run from a different active repo's tick must not route \
6191 another active repo's new tasks — that repo's own tick will route them"
6192 );
6193 assert_eq!(
6194 store.get(inactive_id).await.unwrap().status,
6195 TaskStatus::Routed,
6196 "global sweep must still route tasks from genuinely orphaned repos"
6197 );
6198 }
6199
6200 #[serial_test::serial(cooldown_state)]
6202 #[tokio::test]
6203 async fn global_sweep_skips_no_agent_tasks() {
6204 cooldown::reset_global_state().await;
6205
6206 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6207
6208 let id = store
6209 .upsert_external(&UpsertExternal {
6210 repo: "inactive/repo",
6211 ext_id: "99",
6212 title: "No-agent task",
6213 body: "",
6214 author: "user",
6215 url: "https://github.com/inactive/repo/issues/99",
6216 labels: &["no-agent".to_string()],
6217 origin: "github",
6218 })
6219 .await
6220 .unwrap();
6221
6222 let mut model_map: HashMap<String, HashMap<String, Vec<String>>> = HashMap::new();
6223 let mut tier = HashMap::new();
6224 tier.insert("claude".to_string(), vec!["haiku".to_string()]);
6225 model_map.insert("medium".to_string(), tier);
6226 let config = RouterConfig {
6227 mode: "round_robin".to_string(),
6228 agents: vec!["claude".to_string()],
6229 model_map,
6230 ..Default::default()
6231 };
6232 let mut router = Router::new_for_test(config, vec!["claude".to_string()]);
6233
6234 let active_repos: HashSet<String> = ["active/repo".to_string()].into_iter().collect();
6235 let mut routing_quota = RoutingTickQuota::new(1);
6236 route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
6237 .await
6238 .unwrap();
6239
6240 assert_eq!(
6241 store.get(id).await.unwrap().status,
6242 TaskStatus::New,
6243 "no-agent tasks must not be routed"
6244 );
6245 }
6246
6247 #[serial_test::serial(cooldown_state)]
6252 #[tokio::test]
6253 async fn global_sweep_honors_shared_routing_quota() {
6254 cooldown::reset_global_state().await;
6255
6256 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6257 for ext_id in ["1", "2"] {
6258 store
6259 .upsert_external(&UpsertExternal {
6260 repo: "inactive/repo",
6261 ext_id,
6262 title: "Orphaned task",
6263 body: "",
6264 author: "user",
6265 url: &format!("https://github.com/inactive/repo/issues/{ext_id}"),
6266 labels: &[],
6267 origin: "github",
6268 })
6269 .await
6270 .unwrap();
6271 }
6272
6273 let mut model_map: HashMap<String, HashMap<String, Vec<String>>> = HashMap::new();
6274 let mut tier = HashMap::new();
6275 tier.insert("claude".to_string(), vec!["haiku".to_string()]);
6276 model_map.insert("medium".to_string(), tier);
6277 let config = RouterConfig {
6278 mode: "round_robin".to_string(),
6279 agents: vec!["claude".to_string()],
6280 model_map,
6281 ..Default::default()
6282 };
6283 let mut router = Router::new_for_test(config, vec!["claude".to_string()]);
6284
6285 let active_repos: HashSet<String> = ["active/repo".to_string()].into_iter().collect();
6286 let mut routing_quota = RoutingTickQuota::new(1);
6287 route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
6288 .await
6289 .unwrap();
6290
6291 let all_tasks = store.list_all_global().await.unwrap();
6292 let routed = all_tasks
6293 .iter()
6294 .filter(|task| task.status == TaskStatus::Routed)
6295 .count();
6296 let still_new = all_tasks
6297 .iter()
6298 .filter(|task| task.status == TaskStatus::New)
6299 .count();
6300
6301 assert_eq!(routed, 1, "only one task may consume the shared quota");
6302 assert_eq!(still_new, 1, "remaining tasks wait for the next tick");
6303
6304 route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
6305 .await
6306 .unwrap();
6307 let after_exhausted = store.list_all_global().await.unwrap();
6308 assert_eq!(
6309 after_exhausted
6310 .iter()
6311 .filter(|task| task.status == TaskStatus::Routed)
6312 .count(),
6313 1,
6314 "an exhausted shared quota must not reset within the same tick"
6315 );
6316 }
6317
6318 #[tokio::test]
6321 async fn block_task_without_project_blocks_with_the_cause() {
6322 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6323 let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
6324 let task_manager = Arc::new(TaskManager::with_store(
6325 backend,
6326 Arc::clone(&store),
6327 "removed/repo".to_string(),
6328 ));
6329 let id = store
6330 .create_internal("removed/repo", "Orphan", "", "cron", "1", None)
6331 .await
6332 .unwrap();
6333 store.update_status(id, TaskStatus::Routed).await.unwrap();
6334
6335 block_task_without_project(
6336 &task_manager,
6337 &store,
6338 "removed/repo",
6339 &format!("internal:{id}"),
6340 )
6341 .await;
6342
6343 let task = store.get(id).await.unwrap();
6344 assert_eq!(task.status, TaskStatus::Blocked);
6345 assert_eq!(
6346 task.block_reason.as_deref(),
6347 Some(NoProjectDir::BLOCK_REASON)
6348 );
6349 }
6350
6351 #[tokio::test]
6368 async fn group_orphaned_routed_tasks_selects_inactive_repo_tasks() {
6369 async fn make_routed(store: &TaskStore, repo: &str, ext_id: &str) -> i64 {
6370 let id = store
6371 .upsert_external(&UpsertExternal {
6372 repo,
6373 ext_id,
6374 title: "Routed task",
6375 body: "",
6376 author: "user",
6377 url: &format!("https://github.com/{repo}/issues/{ext_id}"),
6378 labels: &[],
6379 origin: "github",
6380 })
6381 .await
6382 .unwrap();
6383 store
6384 .store_route(&StoreRoute {
6385 id,
6386 agent: "claude",
6387 model: Some("sonnet"),
6388 complexity: "medium",
6389 estimate: 0,
6390 reason: "test",
6391 profile: "{}",
6392 skills: "[]",
6393 })
6394 .await
6395 .unwrap();
6396 store.update_status(id, TaskStatus::Routed).await.unwrap();
6397 id
6398 }
6399
6400 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6401
6402 let active_id = make_routed(&store, "active/repo", "1").await;
6403 let inactive_id_a = make_routed(&store, "inactive/repo", "2").await;
6404 let other_inactive_id = make_routed(&store, "other-inactive/repo", "3").await;
6405 let inactive_id_b = make_routed(&store, "inactive/repo", "4").await;
6406
6407 let all_routed = store
6408 .list_all_by_status_global(TaskStatus::Routed)
6409 .await
6410 .unwrap();
6411 let active_repos: HashSet<String> = ["active/repo".to_string()].into_iter().collect();
6412 let by_repo = group_orphaned_routed_tasks(all_routed, &active_repos);
6413
6414 assert!(
6415 !by_repo.contains_key("active/repo"),
6416 "global sweep must not select tasks for the active repo"
6417 );
6418 let inactive_ids: HashSet<i64> = by_repo
6419 .get("inactive/repo")
6420 .expect("inactive/repo must be selected")
6421 .iter()
6422 .map(|t| t.id)
6423 .collect();
6424 assert_eq!(
6425 inactive_ids,
6426 [inactive_id_a, inactive_id_b].into_iter().collect(),
6427 "global sweep must group all orphaned tasks for a repo together"
6428 );
6429 assert_eq!(
6430 by_repo
6431 .get("other-inactive/repo")
6432 .map(|tasks| tasks.iter().map(|t| t.id).collect::<Vec<_>>()),
6433 Some(vec![other_inactive_id])
6434 );
6435
6436 assert_eq!(
6439 store.get(active_id).await.unwrap().status,
6440 TaskStatus::Routed
6441 );
6442 }
6443
6444 #[tokio::test]
6449 async fn group_orphaned_routed_tasks_excludes_all_active_repos_not_just_current() {
6450 async fn make_routed(store: &TaskStore, repo: &str, ext_id: &str) -> i64 {
6451 let id = store
6452 .upsert_external(&UpsertExternal {
6453 repo,
6454 ext_id,
6455 title: "Routed task",
6456 body: "",
6457 author: "user",
6458 url: &format!("https://github.com/{repo}/issues/{ext_id}"),
6459 labels: &[],
6460 origin: "github",
6461 })
6462 .await
6463 .unwrap();
6464 store
6465 .store_route(&StoreRoute {
6466 id,
6467 agent: "claude",
6468 model: Some("sonnet"),
6469 complexity: "medium",
6470 estimate: 0,
6471 reason: "test",
6472 profile: "{}",
6473 skills: "[]",
6474 })
6475 .await
6476 .unwrap();
6477 store.update_status(id, TaskStatus::Routed).await.unwrap();
6478 id
6479 }
6480
6481 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6482
6483 let repo_a_id = make_routed(&store, "active/repo-a", "1").await;
6484 let repo_b_id = make_routed(&store, "active/repo-b", "2").await;
6485 let orphaned_id = make_routed(&store, "inactive/repo", "3").await;
6486
6487 let all_routed = store
6488 .list_all_by_status_global(TaskStatus::Routed)
6489 .await
6490 .unwrap();
6491 let active_repos: HashSet<String> =
6494 ["active/repo-a".to_string(), "active/repo-b".to_string()]
6495 .into_iter()
6496 .collect();
6497 let by_repo = group_orphaned_routed_tasks(all_routed, &active_repos);
6498
6499 assert!(
6500 !by_repo.contains_key("active/repo-a"),
6501 "the current repo's own tasks must not be swept"
6502 );
6503 assert!(
6504 !by_repo.contains_key("active/repo-b"),
6505 "another active repo's tasks must not be swept during repo A's tick"
6506 );
6507 assert_eq!(
6508 by_repo
6509 .get("inactive/repo")
6510 .map(|tasks| tasks.iter().map(|t| t.id).collect::<Vec<_>>()),
6511 Some(vec![orphaned_id]),
6512 "a genuinely inactive repo's tasks must still be swept"
6513 );
6514
6515 assert_eq!(
6517 store.get(repo_a_id).await.unwrap().status,
6518 TaskStatus::Routed
6519 );
6520 assert_eq!(
6521 store.get(repo_b_id).await.unwrap().status,
6522 TaskStatus::Routed
6523 );
6524 }
6525
6526 #[tokio::test]
6528 async fn global_dispatch_sweep_skips_no_agent_tasks() {
6529 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6530
6531 let id = store
6532 .upsert_external(&UpsertExternal {
6533 repo: "inactive/repo",
6534 ext_id: "99",
6535 title: "No-agent routed task",
6536 body: "",
6537 author: "user",
6538 url: "https://github.com/inactive/repo/issues/99",
6539 labels: &["no-agent".to_string()],
6540 origin: "github",
6541 })
6542 .await
6543 .unwrap();
6544 store
6545 .store_route(&StoreRoute {
6546 id,
6547 agent: "claude",
6548 model: Some("sonnet"),
6549 complexity: "medium",
6550 estimate: 0,
6551 reason: "test",
6552 profile: "{}",
6553 skills: "[]",
6554 })
6555 .await
6556 .unwrap();
6557 store.update_status(id, TaskStatus::Routed).await.unwrap();
6558
6559 let tmux = Arc::new(TmuxManager::new());
6564 let transport = Arc::new(Transport::new());
6565 let capture = Arc::new(CaptureService::new(transport));
6566 let semaphore = Arc::new(Semaphore::new(4));
6567 let (weight_tx, _weight_rx) = mpsc::channel(16);
6568 let dispatching = Arc::new(DashMap::new());
6569 let dispatch_mode = DispatchMode {
6570 is_degraded: false,
6571 healthy_agents: 1,
6572 threshold: 1,
6573 };
6574
6575 let active_repos: HashSet<String> = ["active/repo".to_string()].into_iter().collect();
6576 dispatch_routed_tasks_global(
6577 &tmux,
6578 &capture,
6579 &semaphore,
6580 &weight_tx,
6581 dispatch_mode,
6582 &dispatching,
6583 &store,
6584 &HashMap::new(),
6585 &active_repos,
6586 )
6587 .await
6588 .unwrap();
6589
6590 assert_eq!(
6591 store.get(id).await.unwrap().status,
6592 TaskStatus::Routed,
6593 "no-agent tasks must not be dispatched"
6594 );
6595 }
6596}