1use super::{EngineConfig, ProjectEngine};
17use crate::backends::github::GitHubBackend;
18use crate::backends::{ExternalBackend, ExternalId, ExternalTask, Status};
19use crate::channels::capture::CaptureService;
20use crate::channels::transport::Transport;
21use crate::cmd_cache;
22use crate::config;
23use crate::engine;
24use crate::engine::cooldown::{
25 SILENCE_AGENT_COOLDOWN_SECS, SILENCE_EXTENDED_COOLDOWN_SECS, github_circuit_remaining_secs,
26 is_agent_in_cooldown, is_github_circuit_open, is_model_in_cooldown,
27 record_agent_failure_with_message, record_silence_detection, set_agent_cooldown,
28 set_model_cooldown,
29};
30use crate::engine::dependencies;
31use crate::engine::dispatch_guard::DispatchGuard;
32use crate::engine::jobs;
33use crate::engine::priority;
34use crate::engine::priority::Priority;
35use crate::engine::router::config as router_config;
36use crate::engine::router::{
37 AllAgentsCooledError, RouteResultError, Router, RouterConfig, get_route_result,
38};
39use crate::engine::runner::response;
40use crate::engine::runner::{NoProjectDir, TaskRunner, WeightSignal};
41use crate::engine::sync;
42use crate::engine::tasks;
43use crate::engine::tasks::{TaskManager, is_internal_id};
44use crate::github::http;
45use crate::repo_context::REPO_CONTEXT;
46use crate::store;
47use crate::store::StoreRoute;
48use crate::store::Task;
49use crate::store::TaskStatus;
50use crate::store::TaskStatus as DbStatus;
51use crate::store::TaskStore;
52use crate::store::WaitingOnDependency;
53use crate::store::{store_set_by_id, store_touch_updated_at};
54use crate::tmux::TmuxManager;
55use dashmap::mapref::entry::Entry;
56use dashmap::{DashMap, DashSet};
57use futures::future;
58use std::collections::HashMap;
59use std::collections::HashSet;
60use std::path::Path;
61use std::sync::Arc;
62use std::sync::OnceLock;
63use std::time::Duration;
64use std::time::{Instant, SystemTime};
65use tokio::sync::RwLock;
66use tokio::sync::{Semaphore, mpsc};
67use tokio::time;
68
69pub(crate) struct RoutingTickQuota {
75 remaining: usize,
76}
77
78impl RoutingTickQuota {
79 pub(crate) fn new(max: usize) -> Self {
80 Self { remaining: max }
81 }
82
83 fn is_exhausted(&self) -> bool {
84 self.remaining == 0
85 }
86
87 fn remaining(&self) -> usize {
88 self.remaining
89 }
90
91 fn consume_one(&mut self) -> bool {
92 if self.remaining == 0 {
93 return false;
94 }
95 self.remaining -= 1;
96 true
97 }
98}
99
100pub(crate) async fn startup_cleanup(tmux: &TmuxManager) {
103 let start_time = SystemTime::now()
104 .duration_since(SystemTime::UNIX_EPOCH)
105 .unwrap_or_default()
106 .as_secs();
107 tracing::info!(
108 start_time,
109 "running startup cleanup for stale tmux sessions"
110 );
111
112 match tmux.kill_stale_sessions(start_time).await {
113 Ok(killed) => {
114 if killed > 0 {
115 tracing::info!(killed, "startup cleanup killed stale sessions");
116 } else {
117 tracing::debug!("no stale tmux sessions found on startup");
118 }
119 }
120 Err(e) => {
121 tracing::warn!(error = %e, "failed to run startup cleanup");
122 }
123 }
124}
125
126async fn recover_routed_blocked_dispatch(
127 backend: &Arc<dyn ExternalBackend>,
128 store: &Arc<TaskStore>,
129 task_manager: &Arc<TaskManager>,
130 repo: &str,
131 task: &ExternalTask,
132 timing: &StuckTaskTiming,
133) {
134 tracing::warn!(
135 task_id = task.id.0,
136 age_mins = timing.age.num_minutes(),
137 threshold_mins = timing.threshold / 60,
138 "recovering routed task: dispatch blocked by stale tmux session → new"
139 );
140
141 let resolved_store_id = match store.resolve_task_id(repo, &task.id.0).await {
142 Ok(Some(id)) => Some(id),
143 Ok(None) => {
144 tracing::warn!(
145 task_id = %task.id.0,
146 repo,
147 "resolve_task_id returned None during routed-task recovery"
148 );
149 None
150 }
151 Err(e) => {
152 tracing::error!(
153 task_id = %task.id.0,
154 repo,
155 error = %e,
156 "resolve_task_id failed during routed-task recovery"
157 );
158 None
159 }
160 };
161
162 if let Some(store_id) = resolved_store_id {
163 if let Err(e) = store_set_by_id(
164 Some(store),
165 store_id,
166 &[
167 ("agent", serde_json::Value::Null),
168 ("model", serde_json::Value::Null),
169 ("route_attempts", serde_json::json!(0)),
170 ],
171 )
172 .await
173 {
174 tracing::warn!(
175 task_id = task.id.0,
176 ?e,
177 "failed to clear routing fields for routed stuck task — will retry next tick"
178 );
179 return;
180 }
181 finalize_open_runs_for_recovery(
182 store,
183 store_id,
184 "stuck-task recovery: stale session blocking dispatch killed",
185 )
186 .await;
187 }
188
189 if let Err(e) = task_manager.update_task_status(&task.id, Status::New).await {
190 tracing::warn!(
191 task_id = task.id.0,
192 ?e,
193 "failed to reset routed stuck task status"
194 );
195 return;
196 }
197
198 store::store_log_activity(
199 &Some(Arc::clone(store)),
200 repo,
201 &task.id.0,
202 "rerouted",
203 Some("routed"),
204 Some("new"),
205 None::<&str>,
206 None::<&str>,
207 Some(&serde_json::json!({
208 "failure_reason": "routed_blocked_by_existing_session",
209 "age_minutes": timing.age.num_minutes(),
210 "had_session": true,
211 })),
212 )
213 .await;
214
215 if !is_internal_id(&task.id.0) {
216 let backend_clone = Arc::clone(backend);
217 let task_id_clone = task.id.clone();
218 let body = format!(
219 "[{}] recovered: stuck routed task — existing tmux session blocked dispatch for {}m; cleared route for retry{}",
220 chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
221 timing.age.num_minutes(),
222 engine::orch_footer()
223 );
224 tokio::spawn(async move {
225 if let Err(e) = backend_clone.post_comment(&task_id_clone, &body).await {
226 tracing::warn!(
227 task_id = task_id_clone.0,
228 ?e,
229 "failed to post routed-task recovery comment"
230 );
231 }
232 });
233 }
234}
235
236pub(crate) async fn tick_check_session_completions(
238 tmux: &Arc<TmuxManager>,
239 repo: &str,
240 capture: &Arc<CaptureService>,
241 store: &Arc<TaskStore>,
242) -> anyhow::Result<()> {
243 let _span = tracing::info_span!("engine.tick.phase1.sessions").entered();
244 let session_snapshot = tmux.snapshot().await;
245 let repo_name = repo
251 .rsplit('/')
252 .next()
253 .unwrap_or(repo)
254 .trim_end_matches(".git");
255 for (session, active) in session_snapshot {
256 if session.project != repo_name {
258 continue;
259 }
260
261 if !active {
262 tracing::info!(
263 session = %session.name,
264 task_id = %session.task_id,
265 "session completed, collecting results"
266 );
267 let store_task_id = {
278 let base = session
279 .task_id
280 .strip_suffix("-review")
281 .unwrap_or(&session.task_id);
282 if let Some(n) = base.strip_prefix("internal-") {
283 format!("internal:{n}")
284 } else {
285 base.to_string()
286 }
287 };
288 store_touch_updated_at(&Some(Arc::clone(store)), repo, &store_task_id).await;
289 capture.unregister_session(repo, &session.task_id).await;
292 if let Err(e) = tmux.kill_session(&session.name).await {
294 tracing::debug!(
295 session = %session.name,
296 ?e,
297 "kill_session failed (session may already be gone)"
298 );
299 }
300 }
301 }
302 Ok(())
303}
304
305#[derive(Debug)]
306struct StuckTaskTiming {
307 has_session: bool,
308 age: chrono::Duration,
309 threshold: u64,
310}
311
312fn stuck_task_timing_from_map(
313 tmux: &Arc<TmuxManager>,
314 repo: &str,
315 session_task_id: &str,
316 updated_at: &str,
317 config: &EngineConfig,
318 parse_error_message: &'static str,
319 session_map: &HashMap<String, bool>,
320) -> Option<StuckTaskTiming> {
321 let has_session = session_alive(tmux, repo, session_task_id, session_map);
322 let threshold = if has_session {
323 config.stuck_timeout
324 } else {
325 config.no_session_stuck_timeout
326 };
327
328 let updated = match chrono::DateTime::parse_from_rfc3339(updated_at) {
329 Ok(dt) => dt.with_timezone(&chrono::Utc),
330 Err(e) => {
331 tracing::warn!(
332 task_id = session_task_id,
333 updated_at,
334 ?e,
335 error_message = parse_error_message,
336 "cannot parse updated_at"
337 );
338 return None;
339 }
340 };
341
342 let raw_age = chrono::Utc::now() - updated;
343 let suspended = super::suspend::suspended_duration_since(updated);
347 let age = (raw_age - suspended).max(chrono::Duration::zero());
348 if age.num_seconds() <= i64::try_from(threshold).unwrap_or(i64::MAX) {
349 return None;
350 }
351
352 Some(StuckTaskTiming {
353 has_session,
354 age,
355 threshold,
356 })
357}
358
359fn session_alive(
360 tmux: &TmuxManager,
361 repo: &str,
362 session_task_id: &str,
363 session_map: &HashMap<String, bool>,
364) -> bool {
365 session_map
366 .get(&tmux.session_name(repo, session_task_id))
367 .copied()
368 .unwrap_or(false)
369}
370
371fn run_tier(router: &RouterConfig, task: &Task) -> String {
374 match (task.agent.as_deref(), task.model.as_deref()) {
375 (Some(a), Some(m)) if !m.is_empty() => router.tier_of_model(a, m, &task.complexity),
376 _ => task.complexity.clone(),
377 }
378}
379
380fn tick_router(
383 cell: &OnceLock<RouterConfig>,
384 load: impl FnOnce() -> RouterConfig,
385) -> &RouterConfig {
386 cell.get_or_init(load)
387}
388
389async fn in_progress_stuck_clock(
394 store: &TaskStore,
395 repo: &str,
396 task_id: &str,
397 updated_at: &str,
398 has_session: bool,
399 config: &EngineConfig,
400 router: &OnceLock<RouterConfig>,
401) -> (String, EngineConfig) {
402 if !has_session {
403 return (updated_at.to_string(), config.clone());
404 }
405
406 let task = match store.resolve_task_id(repo, task_id).await {
407 Ok(Some(store_id)) => store.get(store_id).await.ok(),
408 _ => None,
409 };
410 let Some(task) = task else {
411 return (updated_at.to_string(), config.clone());
412 };
413
414 let entered_at = match store
415 .status_entered_at(task.id, TaskStatus::InProgress)
416 .await
417 {
418 Ok(Some(ts)) => ts,
419 Ok(None) => updated_at.to_string(),
420 Err(e) => {
421 tracing::warn!(
422 task_id,
423 error = %e,
424 "failed to read in_progress entry, falling back to updated_at"
425 );
426 updated_at.to_string()
427 }
428 };
429 let tier = run_tier(tick_router(router, RouterConfig::from_config), &task);
430 let task_config = EngineConfig {
431 stuck_timeout: config.live_session_stuck_limit(Some(&tier)),
432 ..config.clone()
433 };
434 (entered_at, task_config)
435}
436
437async fn in_review_stuck_entry(store: &TaskStore, store_id: i64, fallback: &str) -> (String, bool) {
440 match store.in_review_entry(store_id).await {
441 Ok(Some(entry)) => entry,
442 Ok(None) => (fallback.to_string(), false),
443 Err(e) => {
444 tracing::warn!(
445 store_id,
446 error = %e,
447 "failed to read in_review entry, falling back to updated_at"
448 );
449 (fallback.to_string(), false)
450 }
451 }
452}
453
454async fn finalize_open_runs_for_recovery(store: &TaskStore, store_id: i64, reason: &str) {
457 if let Err(e) = store
458 .finalize_incomplete_runs(store_id, "aborted", reason)
459 .await
460 {
461 tracing::warn!(
462 store_id,
463 error = %e,
464 reason,
465 "failed to finalize incomplete runs during recovery"
466 );
467 }
468}
469
470pub(crate) async fn tick_detect_silent_agents(
479 tmux: &Arc<TmuxManager>,
480 repo: &str,
481 capture: &Arc<CaptureService>,
482 backend: &Arc<dyn ExternalBackend>,
483 task_manager: &Arc<TaskManager>,
484 config: &EngineConfig,
485 store: &Arc<TaskStore>,
486) -> anyhow::Result<()> {
487 let _span = tracing::info_span!("engine.tick.phase1b.silence").entered();
488 let grace = Duration::from_secs(config.silence_grace_period);
489 let silent_sessions = capture.get_silent_sessions_for_repo(repo, grace).await;
490
491 for (task_id, session_name, silence_age_secs) in silent_sessions {
492 let use_backend = should_use_backend(&task_id);
493 let store_task = match store.resolve_task_id(repo, &task_id).await {
495 Ok(Some(store_id)) => match store.get(store_id).await {
496 Ok(task) => Some(task),
497 Err(e) => {
498 tracing::warn!(
499 task_id,
500 repo,
501 store_id,
502 operation = "store.get",
503 error = %e,
504 "silence detection: db error fetching task — agent/model cooldown will be skipped"
505 );
506 None
507 }
508 },
509 Ok(None) => {
510 tracing::debug!(
511 task_id,
512 repo,
513 operation = "resolve_task_id",
514 "silence detection: task not found in store (may have been cleaned up)"
515 );
516 None
517 }
518 Err(e) => {
519 tracing::warn!(
520 task_id,
521 repo,
522 operation = "resolve_task_id",
523 error = %e,
524 "silence detection: db error resolving task id — agent/model cooldown will be skipped"
525 );
526 None
527 }
528 };
529 let agent_name = store_task
530 .as_ref()
531 .and_then(|t| t.agent.clone())
532 .unwrap_or_default();
533 let model_name = store_task
534 .as_ref()
535 .and_then(|t| t.model.clone())
536 .unwrap_or_default();
537
538 tracing::warn!(
539 task_id,
540 agent = %agent_name,
541 model = %model_name,
542 silence_age_secs,
543 grace_secs = config.silence_grace_period,
544 cooldown_secs = config.silence_cooldown,
545 "agent silent since session start — killing session, cooling down model + agent, failing over"
546 );
547
548 if let Err(e) = tmux.kill_session(&session_name).await {
550 tracing::debug!(
551 task_id,
552 ?e,
553 "kill_session failed for silent agent (may already be gone)"
554 );
555 }
556
557 capture.unregister_session(repo, &task_id).await;
559
560 if let Some(store_id) = store_task.as_ref().map(|t| t.id) {
562 finalize_open_runs_for_recovery(store, store_id, "silence detection: session killed")
563 .await;
564 }
565
566 let mut extended_note = String::new();
567
568 if !agent_name.is_empty() && !model_name.is_empty() {
570 set_model_cooldown(
571 &agent_name,
572 &model_name,
573 config.silence_cooldown,
574 "silence_detected",
575 )
576 .await;
577 if let Some(result) = record_silence_detection(&agent_name, &model_name).await
578 && result.extended_cooldown_applied
579 {
580 extended_note = format!(
581 " ({} silences in 24h -> extended cooldown {}s)",
582 result.count, SILENCE_EXTENDED_COOLDOWN_SECS
583 );
584 }
585 }
586
587 if !agent_name.is_empty() {
591 set_agent_cooldown(&agent_name, SILENCE_AGENT_COOLDOWN_SECS).await;
592 }
593
594 let task_eid = ExternalId(task_id.clone());
599
600 let available: Vec<String> = engine::configured_agents()
602 .into_iter()
603 .filter(|a| cmd_cache::command_exists(a))
604 .collect();
605 let chain = response::get_reroute_chain(&task_id, &Some(Arc::clone(store)), repo).await;
606 let chain = response::update_reroute_chain(
607 &task_id,
608 &agent_name,
609 &chain,
610 &Some(Arc::clone(store)),
611 repo,
612 )
613 .await;
614
615 let (next_status, next_agent) = if let Some(fallback) =
616 response::pick_fallback_agent(&agent_name, &chain, &available)
617 {
618 tracing::info!(
619 task_id,
620 from = %agent_name,
621 to = %fallback,
622 "silence detection: failover to different agent, setting routed"
623 );
624 (Status::Routed, Some(fallback))
625 } else {
626 tracing::warn!(
627 task_id,
628 "silence detection: no fallback agents available, resetting to new for re-routing"
629 );
630 (Status::New, None)
631 };
632
633 if use_backend && let Some(ref st) = store_task {
634 let stale_labels: Vec<String> = st
636 .labels
637 .iter()
638 .filter(|l| {
639 l.starts_with("agent:")
640 || l.starts_with("complexity:")
641 || l.starts_with("model:")
642 })
643 .cloned()
644 .collect();
645 if !stale_labels.is_empty() {
646 let backend_clone = Arc::clone(backend);
647 let task_eid_clone = task_eid.clone();
648 tokio::spawn(async move {
649 for label in &stale_labels {
650 if let Err(e) = backend_clone.remove_label(&task_eid_clone, label).await {
651 tracing::warn!(task_id = task_eid_clone.0, label, error = %e, "failed to remove label during silence detection re-route");
652 }
653 }
654 });
655 }
656 }
657
658 if let Some(store_id) = store_task.as_ref().map(|t| t.id) {
659 if let Some(ref fallback) = next_agent {
660 let _ = store_set_by_id(
664 Some(store),
665 store_id,
666 &[
667 ("agent", serde_json::json!(fallback)),
668 ("model", serde_json::json!("")),
669 (
670 "last_error",
671 serde_json::json!(format!(
672 "silence detected after {}s, failing over to {}",
673 config.silence_grace_period, fallback
674 )),
675 ),
676 ],
677 )
678 .await;
679 } else {
680 let _ = store_set_by_id(
681 Some(store),
682 store_id,
683 &[
684 ("agent", serde_json::Value::Null),
685 ("model", serde_json::Value::Null),
686 (
687 "last_error",
688 serde_json::json!(format!(
689 "silence detected after {}s, no fallback agents available",
690 config.silence_grace_period
691 )),
692 ),
693 ],
694 )
695 .await;
696 }
697 }
698
699 if let Err(e) = task_manager
700 .update_task_status(&task_eid, next_status)
701 .await
702 {
703 tracing::warn!(task_id, ?e, "failed to update silent task status");
704 continue;
705 }
706
707 let details = serde_json::json!({
709 "failure_reason": "agent_silence",
710 "silence_duration_secs": config.silence_grace_period,
711 "cooldown_applied": !agent_name.is_empty() && !model_name.is_empty(),
712 "fallback_available": next_agent.is_some(),
713 });
714 store::store_log_activity(
715 &Some(Arc::clone(store)),
716 repo,
717 &task_id,
718 "rerouted",
719 Some("in_progress"),
720 Some(next_status.as_str()),
721 next_agent.as_deref(),
722 None,
723 Some(&details),
724 )
725 .await;
726
727 let action = if let Some(ref fallback) = next_agent {
729 format!("failing over to {fallback}")
730 } else {
731 "marking needs_review (no fallback agents)".to_string()
732 };
733 let comment = format!(
734 "[{}] agent silent for {}s since session start — killed session, cooled down model `{}:{}` for {}s{}, agent `{}` for {}s, {}{}",
735 chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
736 config.silence_grace_period,
737 agent_name,
738 model_name,
739 config.silence_cooldown,
740 extended_note,
741 agent_name,
742 SILENCE_AGENT_COOLDOWN_SECS,
743 action,
744 engine::orch_footer(),
745 );
746 if use_backend {
747 let backend_clone = Arc::clone(backend);
749 let task_eid_clone = task_eid.clone();
750 tokio::spawn(async move {
751 if let Err(e) = backend_clone.post_comment(&task_eid_clone, &comment).await {
752 tracing::warn!(
753 task_id = task_eid_clone.0,
754 ?e,
755 "failed to post silence detection comment"
756 );
757 }
758 });
759 }
760 }
761
762 Ok(())
763}
764
765fn should_use_backend(task_id: &str) -> bool {
766 !is_internal_id(task_id)
767}
768
769#[expect(
774 clippy::too_many_arguments,
775 reason = "each argument is engine state the tick passes down"
776)]
777pub(crate) async fn tick_recover_stuck_tasks(
778 backend: &Arc<dyn ExternalBackend>,
779 tmux: &Arc<TmuxManager>,
780 repo: &str,
781 task_manager: &Arc<TaskManager>,
782 config: &EngineConfig,
783 store: &Arc<TaskStore>,
784 session_map: &HashMap<String, bool>,
785 dispatching: &Arc<DashMap<String, String>>,
786) -> anyhow::Result<()> {
787 let _span = tracing::info_span!("engine.tick.phase2.stuck_tasks").entered();
788 let tick_router_cell = OnceLock::new();
789
790 let in_progress = match task_manager
791 .list_external_by_status(Status::InProgress)
792 .await
793 {
794 Ok(tasks) => tasks,
795 Err(e) => {
796 tracing::warn!(?e, "failed to list in_progress tasks for stuck recovery");
797 vec![]
798 }
799 };
800 for task in &in_progress {
801 let (since, task_config) = in_progress_stuck_clock(
802 store,
803 repo,
804 &task.id.0,
805 &task.updated_at,
806 session_alive(tmux, repo, &task.id.0, session_map),
807 config,
808 &tick_router_cell,
809 )
810 .await;
811 let Some(timing) = stuck_task_timing_from_map(
812 tmux,
813 repo,
814 &task.id.0,
815 &since,
816 &task_config,
817 "cannot parse updated_at, skipping stuck-task check",
818 session_map,
819 ) else {
820 continue;
821 };
822
823 if !timing.has_session {
830 let dispatch_key = format!("{repo}/{}", task.id.0);
831 if dispatching.contains_key(&dispatch_key) {
832 tracing::info!(
840 task_id = task.id.0,
841 dispatch_key,
842 age_mins = timing.age.num_minutes(),
843 "skipping stuck-task reclaim: runner still actively processing this task"
844 );
845 continue;
846 }
847 }
848
849 if timing.has_session {
850 tracing::warn!(
851 task_id = task.id.0,
852 age_mins = timing.age.num_minutes(),
853 threshold_mins = timing.threshold / 60,
854 "recovering stuck task: timed out with active session → new"
855 );
856 let session_name = tmux.session_name(repo, &task.id.0);
859 if let Err(e) = tmux.kill_session(&session_name).await {
860 tracing::warn!(
861 task_id = task.id.0,
862 session = %session_name,
863 error = %e,
864 "failed to kill stuck tmux session, will retry next tick"
865 );
866 continue;
867 }
868 } else {
869 tracing::warn!(
870 task_id = task.id.0,
871 dispatch_key = format!("{repo}/{}", task.id.0),
872 age_mins = timing.age.num_minutes(),
873 threshold_mins = timing.threshold / 60,
874 "recovering stuck task: no session found — reclaiming early → new"
875 );
876 }
877 let mut cached_store_id: Option<i64> = None;
882 if timing.has_session {
883 let store_task = match store.resolve_task_id(repo, &task.id.0).await {
884 Ok(Some(store_id)) => {
885 cached_store_id = Some(store_id);
886 match store.get(store_id).await {
887 Ok(t) => Some(t),
888 Err(e) => {
889 tracing::warn!(task_id = task.id.0, error = %e, "failed to fetch task from store for stuck-task cooldown");
890 None
891 }
892 }
893 }
894 Ok(None) => None,
895 Err(e) => {
896 tracing::warn!(task_id = task.id.0, error = %e, "failed to resolve task id for stuck-task cooldown");
897 None
898 }
899 };
900 let agent_name = store_task
901 .as_ref()
902 .and_then(|t| t.agent.clone())
903 .unwrap_or_default();
904 let model_name = store_task
905 .as_ref()
906 .and_then(|t| t.model.clone())
907 .unwrap_or_default();
908
909 if !agent_name.is_empty() && !model_name.is_empty() {
910 set_model_cooldown(
911 &agent_name,
912 &model_name,
913 config.silence_cooldown,
914 "silence_detected",
915 )
916 .await;
917 record_agent_failure_with_message(
918 &agent_name,
919 &format!(
920 "stuck with active session after {}m",
921 timing.age.num_minutes()
922 ),
923 )
924 .await;
925 }
926 if !agent_name.is_empty() {
927 set_agent_cooldown(&agent_name, SILENCE_AGENT_COOLDOWN_SECS).await;
928 }
929 tracing::warn!(
930 task_id = task.id.0,
931 agent = %agent_name,
932 model = %model_name,
933 cooldown_secs = config.silence_cooldown,
934 "stuck-task cooldown applied — router will avoid this agent/model on next attempt"
935 );
936 }
937
938 {
941 let stale_labels: Vec<String> = task
942 .labels
943 .iter()
944 .filter(|l| l.starts_with("agent:") || l.starts_with("model:"))
945 .cloned()
946 .collect();
947 if !stale_labels.is_empty() {
948 let backend_clone = Arc::clone(backend);
949 let task_id_clone = task.id.clone();
950 tokio::spawn(async move {
951 for label in &stale_labels {
952 if let Err(e) = backend_clone.remove_label(&task_id_clone, label).await {
953 tracing::warn!(task_id = task_id_clone.0, label, error = %e, "failed to remove stale routing label during stuck-task recovery");
954 }
955 }
956 });
957 }
958 }
959 let resolved_store_id = match cached_store_id {
960 Some(id) => Some(id),
961 None => match store.resolve_task_id(repo, &task.id.0).await {
962 Ok(Some(id)) => Some(id),
963 Ok(None) => {
964 tracing::warn!(
965 task_id = %task.id.0,
966 repo,
967 "resolve_task_id returned None during stuck-task recovery"
968 );
969 None
970 }
971 Err(e) => {
972 tracing::error!(
973 task_id = %task.id.0,
974 repo,
975 error = %e,
976 "resolve_task_id failed during stuck-task recovery"
977 );
978 None
979 }
980 },
981 };
982 if let Some(store_id) = resolved_store_id {
983 if let Err(e) = store_set_by_id(
984 Some(store),
985 store_id,
986 &[
987 ("agent", serde_json::Value::Null),
988 ("model", serde_json::Value::Null),
989 ("route_attempts", serde_json::json!(0)),
990 ],
991 )
992 .await
993 {
994 tracing::warn!(
995 task_id = task.id.0,
996 ?e,
997 "failed to clear routing fields for stuck task — will retry next tick"
998 );
999 continue;
1000 }
1001 finalize_open_runs_for_recovery(
1002 store,
1003 store_id,
1004 if timing.has_session {
1005 "stuck-task recovery: active session killed"
1006 } else {
1007 "stuck-task recovery: no session found"
1008 },
1009 )
1010 .await;
1011 }
1012 if let Err(e) = task_manager.update_task_status(&task.id, Status::New).await {
1013 tracing::warn!(task_id = task.id.0, ?e, "failed to reset stuck task status");
1014 continue;
1015 }
1016
1017 let details = serde_json::json!({
1019 "failure_reason": if timing.has_session { "stuck_with_session" } else { "stuck_no_session" },
1020 "age_minutes": timing.age.num_minutes(),
1021 "had_session": timing.has_session,
1022 });
1023 store::store_log_activity(
1024 &Some(Arc::clone(store)),
1025 repo,
1026 &task.id.0,
1027 "rerouted",
1028 Some("in_progress"),
1029 Some("new"),
1030 None::<&str>,
1031 None::<&str>,
1032 Some(&details),
1033 )
1034 .await;
1035
1036 let reason = if timing.has_session {
1037 format!(
1038 "timed out after {}m with active session (cleared agent for re-routing)",
1039 timing.age.num_minutes()
1040 )
1041 } else {
1042 format!(
1043 "no session found — reclaiming early after {}m (cleared agent for re-routing)",
1044 timing.age.num_minutes()
1045 )
1046 };
1047 {
1049 let backend_clone = Arc::clone(backend);
1050 let task_id_clone = task.id.clone();
1051 let body = format!(
1052 "[{}] recovered: stuck in_progress — {}{}",
1053 chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
1054 reason,
1055 engine::orch_footer()
1056 );
1057 tokio::spawn(async move {
1058 if let Err(e) = backend_clone.post_comment(&task_id_clone, &body).await {
1059 tracing::warn!(
1060 task_id = task_id_clone.0,
1061 ?e,
1062 "failed to post stuck-task recovery comment"
1063 );
1064 }
1065 });
1066 }
1067 }
1068
1069 let mut routed = match task_manager.list_external_by_status(Status::Routed).await {
1070 Ok(tasks) => tasks,
1071 Err(e) => {
1072 tracing::warn!(?e, "failed to list routed tasks for stuck recovery");
1073 vec![]
1074 }
1075 };
1076 match task_manager
1077 .list_internal_by_status(TaskStatus::Routed)
1078 .await
1079 {
1080 Ok(tasks) => routed.extend(tasks),
1081 Err(e) => tracing::warn!(
1082 ?e,
1083 "failed to list internal routed tasks for stuck recovery"
1084 ),
1085 }
1086 for task in &routed {
1087 let Some(timing) = stuck_task_timing_from_map(
1088 tmux,
1089 repo,
1090 &task.id.0,
1091 &task.updated_at,
1092 config,
1093 "cannot parse updated_at, skipping routed stuck-task check",
1094 session_map,
1095 ) else {
1096 continue;
1097 };
1098
1099 if !timing.has_session {
1100 continue;
1101 }
1102
1103 let session_name = tmux.session_name(repo, &task.id.0);
1104 if let Err(e) = tmux.kill_session(&session_name).await {
1105 tracing::warn!(
1106 task_id = task.id.0,
1107 session = %session_name,
1108 error = %e,
1109 "failed to kill stale tmux session blocking routed task"
1110 );
1111 continue;
1112 }
1113
1114 recover_routed_blocked_dispatch(backend, store, task_manager, repo, task, &timing).await;
1115 }
1116
1117 let internal_in_progress = match task_manager
1120 .list_internal_by_status(DbStatus::InProgress)
1121 .await
1122 {
1123 Ok(tasks) => tasks,
1124 Err(e) => {
1125 tracing::warn!(
1126 ?e,
1127 "failed to list internal in_progress tasks for stuck recovery"
1128 );
1129 vec![]
1130 }
1131 };
1132 for task in &internal_in_progress {
1133 let task_id = task.id.0.clone();
1134 let (since, task_config) = in_progress_stuck_clock(
1135 store,
1136 repo,
1137 &task_id,
1138 &task.updated_at,
1139 session_alive(tmux, repo, &task_id, session_map),
1140 config,
1141 &tick_router_cell,
1142 )
1143 .await;
1144 let Some(timing) = stuck_task_timing_from_map(
1145 tmux,
1146 repo,
1147 &task_id,
1148 &since,
1149 &task_config,
1150 "cannot parse updated_at, skipping stuck internal-task check",
1151 session_map,
1152 ) else {
1153 continue;
1154 };
1155
1156 if !timing.has_session {
1162 let dispatch_key = format!("{repo}/{task_id}");
1163 if dispatching.contains_key(&dispatch_key) {
1164 tracing::info!(
1174 task_id,
1175 dispatch_key,
1176 age_mins = timing.age.num_minutes(),
1177 "skipping stuck-task reclaim: runner still actively processing this task"
1178 );
1179 continue;
1180 }
1181 }
1182
1183 if timing.has_session {
1184 tracing::warn!(
1185 task_id,
1186 age_mins = timing.age.num_minutes(),
1187 threshold_mins = timing.threshold / 60,
1188 "recovering stuck task: timed out with active session → new"
1189 );
1190 let session_name = tmux.session_name(repo, &task_id);
1191 if let Err(e) = tmux.kill_session(&session_name).await {
1192 tracing::warn!(
1193 task_id,
1194 session = %session_name,
1195 error = %e,
1196 "failed to kill stuck tmux session, will retry next tick"
1197 );
1198 continue;
1199 }
1200 } else {
1201 tracing::warn!(
1202 task_id,
1203 dispatch_key = format!("{repo}/{task_id}"),
1204 age_mins = timing.age.num_minutes(),
1205 threshold_mins = timing.threshold / 60,
1206 "recovering stuck task: no session found — reclaiming early → new"
1207 );
1208 }
1209 let mut cached_store_id: Option<i64> = None;
1213 if timing.has_session {
1214 let store_task = match store.resolve_task_id(repo, &task_id).await {
1215 Ok(Some(store_id)) => {
1216 cached_store_id = Some(store_id);
1217 match store.get(store_id).await {
1218 Ok(t) => Some(t),
1219 Err(e) => {
1220 tracing::warn!(task_id, error = %e, "failed to fetch task from store for stuck internal-task cooldown");
1221 None
1222 }
1223 }
1224 }
1225 Ok(None) => None,
1226 Err(e) => {
1227 tracing::warn!(task_id, error = %e, "failed to resolve task id for stuck internal-task cooldown");
1228 None
1229 }
1230 };
1231 let agent_name = store_task
1232 .as_ref()
1233 .and_then(|t| t.agent.clone())
1234 .unwrap_or_default();
1235 let model_name = store_task
1236 .as_ref()
1237 .and_then(|t| t.model.clone())
1238 .unwrap_or_default();
1239
1240 if !agent_name.is_empty() && !model_name.is_empty() {
1241 set_model_cooldown(
1242 &agent_name,
1243 &model_name,
1244 config.silence_cooldown,
1245 "silence_detected",
1246 )
1247 .await;
1248 record_agent_failure_with_message(
1249 &agent_name,
1250 &format!(
1251 "internal task stuck with active session after {}m",
1252 timing.age.num_minutes()
1253 ),
1254 )
1255 .await;
1256 }
1257 if !agent_name.is_empty() {
1258 set_agent_cooldown(&agent_name, SILENCE_AGENT_COOLDOWN_SECS).await;
1259 }
1260 tracing::warn!(
1261 task_id,
1262 agent = %agent_name,
1263 model = %model_name,
1264 cooldown_secs = config.silence_cooldown,
1265 "internal stuck-task cooldown applied — router will avoid this agent/model on next attempt"
1266 );
1267 }
1268 let resolved_store_id = match cached_store_id {
1271 Some(id) => Some(id),
1272 None => match store.resolve_task_id(repo, &task_id).await {
1273 Ok(Some(id)) => Some(id),
1274 Ok(None) => {
1275 tracing::warn!(
1276 task_id = %task_id,
1277 repo,
1278 "resolve_task_id returned None during stuck internal-task recovery"
1279 );
1280 None
1281 }
1282 Err(e) => {
1283 tracing::error!(
1284 task_id = %task_id,
1285 repo,
1286 error = %e,
1287 "resolve_task_id failed during stuck internal-task recovery"
1288 );
1289 None
1290 }
1291 },
1292 };
1293 if let Some(store_id) = resolved_store_id {
1294 if let Err(e) = store_set_by_id(
1295 Some(store),
1296 store_id,
1297 &[
1298 ("agent", serde_json::Value::Null),
1299 ("model", serde_json::Value::Null),
1300 ("route_attempts", serde_json::json!(0)),
1301 ],
1302 )
1303 .await
1304 {
1305 tracing::warn!(
1306 task_id,
1307 ?e,
1308 "failed to clear routing fields for stuck internal task — will retry next tick"
1309 );
1310 continue;
1311 }
1312 finalize_open_runs_for_recovery(
1313 store,
1314 store_id,
1315 if timing.has_session {
1316 "stuck-task recovery: internal active session killed"
1317 } else {
1318 "stuck-task recovery: internal no session found"
1319 },
1320 )
1321 .await;
1322 }
1323 if let Err(e) = task_manager
1324 .update_task_status(&ExternalId(task_id.clone()), Status::New)
1325 .await
1326 {
1327 tracing::warn!(task_id, ?e, "failed to reset stuck internal task status");
1328 }
1329 }
1330
1331 let in_review = match store.list_by_status(repo, TaskStatus::InReview).await {
1338 Ok(v) => v,
1339 Err(e) => {
1340 tracing::warn!(?e, "failed to list in_review tasks for stuck recovery");
1341 vec![]
1342 }
1343 };
1344 let in_review_config = engine::EngineConfig {
1348 no_session_stuck_timeout: config.in_review_no_session_stuck_timeout,
1349 ..config.clone()
1350 };
1351 for task in &in_review {
1352 let Some(task_id) = task.external_id.clone() else {
1353 tracing::warn!(
1354 store_id = task.id,
1355 "in_review task has no external id, skipping stuck recovery"
1356 );
1357 continue;
1358 };
1359
1360 if dispatching.contains_key(&format!("{repo}/{task_id}")) {
1362 tracing::debug!(
1363 task_id,
1364 "task locked by dispatch flow, skipping stuck check"
1365 );
1366 continue;
1367 }
1368
1369 let (entered_at, decided) = in_review_stuck_entry(store, task.id, &task.updated_at).await;
1372 if decided {
1373 tracing::debug!(
1374 task_id,
1375 "in_review task already has a review decision, waiting on merge"
1376 );
1377 continue;
1378 }
1379
1380 let review_task_id = format!("{task_id}-review");
1381 let Some(timing) = stuck_task_timing_from_map(
1382 tmux,
1383 repo,
1384 &review_task_id,
1385 &entered_at,
1386 &in_review_config,
1387 "cannot parse in_review timestamp, skipping stuck in_review check",
1388 session_map,
1389 ) else {
1390 continue;
1391 };
1392
1393 tracing::warn!(
1394 task_id,
1395 age_mins = timing.age.num_minutes(),
1396 threshold_mins = timing.threshold / 60,
1397 has_session = timing.has_session,
1398 "recovering stuck in_review task — resetting to needs_review"
1399 );
1400
1401 match task_manager
1402 .update_task_status_if(
1403 &ExternalId(task_id.clone()),
1404 Status::NeedsReview,
1405 Status::InReview,
1406 )
1407 .await
1408 {
1409 Err(e) => {
1410 tracing::warn!(task_id, ?e, "failed to reset stuck in_review task status");
1411 continue;
1412 }
1413 Ok(false) => {
1414 tracing::debug!(
1415 task_id,
1416 "stuck in_review reset skipped — task already transitioned"
1417 );
1418 continue;
1419 }
1420 Ok(true) => {}
1421 }
1422
1423 if let Err(e) = store
1426 .set_fields(task.id, &[("review_agent_failures", serde_json::json!(0))])
1427 .await
1428 {
1429 tracing::warn!(task_id, ?e, "failed to reset review_agent_failures");
1430 }
1431
1432 finalize_open_runs_for_recovery(
1433 store,
1434 task.id,
1435 if timing.has_session {
1436 "stuck-task recovery: in_review session killed"
1437 } else {
1438 "stuck-task recovery: in_review no session found"
1439 },
1440 )
1441 .await;
1442
1443 let details = serde_json::json!({
1444 "failure_reason": "stuck_in_review",
1445 "age_minutes": timing.age.num_minutes(),
1446 "had_session": timing.has_session,
1447 });
1448 store::store_log_activity(
1449 &Some(Arc::clone(store)),
1450 repo,
1451 &task_id,
1452 "rerouted",
1453 Some("in_review"),
1454 Some("needs_review"),
1455 None::<&str>,
1456 None::<&str>,
1457 Some(&details),
1458 )
1459 .await;
1460
1461 if task.origin == "internal" {
1463 continue;
1464 }
1465
1466 if let Err(e) = backend
1467 .post_comment(
1468 &ExternalId(task_id.clone()),
1469 &format!(
1470 "[{}] recovered: stuck in_review — {} after {}m, resetting to needs_review{}",
1471 chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
1472 if timing.has_session {
1473 "review session hung"
1474 } else {
1475 "no session found"
1476 },
1477 timing.age.num_minutes(),
1478 engine::orch_footer()
1479 ),
1480 )
1481 .await
1482 {
1483 tracing::warn!(
1484 task_id,
1485 ?e,
1486 "failed to post stuck in_review recovery comment"
1487 );
1488 }
1489 }
1490
1491 Ok(())
1492}
1493
1494pub(crate) async fn recover_untracked_repo_stuck_tasks(
1497 tmux: &Arc<TmuxManager>,
1498 config: &EngineConfig,
1499 store: &Arc<TaskStore>,
1500 session_map: &HashMap<String, bool>,
1501 dispatching: &Arc<DashMap<String, String>>,
1502 active_repos: &HashSet<String>,
1503) {
1504 let all_external_in_progress = match store
1505 .list_all_by_status_global(TaskStatus::InProgress)
1506 .await
1507 {
1508 Ok(tasks) => tasks,
1509 Err(e) => {
1510 tracing::warn!(
1511 ?e,
1512 "failed to list global in_progress tasks for cross-repo stuck recovery"
1513 );
1514 vec![]
1515 }
1516 };
1517 for task in all_external_in_progress
1518 .iter()
1519 .filter(|t| t.origin != "internal" && !active_repos.contains(&t.repo))
1520 {
1521 let external_id = task
1522 .external_id
1523 .clone()
1524 .unwrap_or_else(|| task.id.to_string());
1525 let Some(timing) = stuck_task_timing_from_map(
1526 tmux,
1527 &task.repo,
1528 &external_id,
1529 &task.updated_at,
1530 config,
1531 "cannot parse updated_at, skipping cross-repo stuck-task check",
1532 session_map,
1533 ) else {
1534 continue;
1535 };
1536
1537 if !timing.has_session {
1541 let dispatch_key = format!("{}/{external_id}", task.repo);
1542 if dispatching.contains_key(&dispatch_key) {
1543 tracing::info!(
1547 task_id = task.id,
1548 repo = %task.repo,
1549 dispatch_key,
1550 age_mins = timing.age.num_minutes(),
1551 "skipping cross-repo stuck-task reclaim: runner still actively processing this task"
1552 );
1553 continue;
1554 }
1555 }
1556
1557 if timing.has_session {
1558 tracing::warn!(
1559 task_id = task.id,
1560 repo = %task.repo,
1561 age_mins = timing.age.num_minutes(),
1562 threshold_mins = timing.threshold / 60,
1563 "recovering cross-repo stuck task: timed out with active session → new"
1564 );
1565 } else {
1566 tracing::warn!(
1567 task_id = task.id,
1568 repo = %task.repo,
1569 dispatch_key = format!("{}/{external_id}", task.repo),
1570 age_mins = timing.age.num_minutes(),
1571 threshold_mins = timing.threshold / 60,
1572 "recovering cross-repo stuck task: no session found — reclaiming early → new"
1573 );
1574 }
1575
1576 if let Err(e) = store_set_by_id(
1577 Some(store),
1578 task.id,
1579 &[
1580 ("agent", serde_json::Value::Null),
1581 ("model", serde_json::Value::Null),
1582 ("route_attempts", serde_json::json!(0)),
1583 ],
1584 )
1585 .await
1586 {
1587 tracing::warn!(
1588 task_id = task.id,
1589 repo = %task.repo,
1590 ?e,
1591 "failed to clear routing fields for cross-repo stuck task — will retry next tick"
1592 );
1593 continue;
1594 }
1595
1596 finalize_open_runs_for_recovery(
1597 store,
1598 task.id,
1599 if timing.has_session {
1600 "stuck-task recovery: cross-repo active session killed"
1601 } else {
1602 "stuck-task recovery: cross-repo no session found"
1603 },
1604 )
1605 .await;
1606
1607 if let Err(e) = store.update_status(task.id, TaskStatus::New).await {
1608 tracing::warn!(
1609 task_id = task.id,
1610 repo = %task.repo,
1611 ?e,
1612 "failed to reset cross-repo stuck task status"
1613 );
1614 continue;
1615 }
1616
1617 let details = serde_json::json!({
1618 "failure_reason": if timing.has_session { "stuck_with_session" } else { "stuck_no_session" },
1619 "age_minutes": timing.age.num_minutes(),
1620 "had_session": timing.has_session,
1621 "scope": "cross_repo_global_sweep",
1622 });
1623 store::store_log_activity(
1624 &Some(Arc::clone(store)),
1625 &task.repo,
1626 &external_id,
1627 "rerouted",
1628 Some("in_progress"),
1629 Some("new"),
1630 None::<&str>,
1631 None::<&str>,
1632 Some(&details),
1633 )
1634 .await;
1635 }
1636}
1637
1638pub(crate) async fn tick_global_sweeps(
1647 task_manager: &Arc<TaskManager>,
1648 store: &Arc<TaskStore>,
1649 router: &Router,
1650 dispatching: &Arc<DashMap<String, String>>,
1651 auto_merge_in_flight: &Arc<DashSet<String>>,
1652 active_repos: &HashSet<String>,
1653) {
1654 if let Err(e) = sync::auto_unblock_ci_failure_blocked_tasks_global(
1659 task_manager,
1660 store,
1661 auto_merge_in_flight,
1662 )
1663 .await
1664 {
1665 tracing::warn!(err = %e, "global CI-failure blocked sweep failed");
1666 }
1667
1668 if let Err(e) = sync::refire_and_escalate_stale_needs_review_global(
1671 task_manager,
1672 store,
1673 router,
1674 dispatching,
1675 active_repos,
1676 )
1677 .await
1678 {
1679 tracing::warn!(err = %e, "global NeedsReview refire/escalation sweep failed");
1680 }
1681
1682 if let Err(e) = sync::auto_recover_rebroadcast_blocked_tasks_global(
1685 router,
1686 task_manager,
1687 store,
1688 active_repos,
1689 )
1690 .await
1691 {
1692 tracing::warn!(err = %e, "global rebroadcast-blocked recovery sweep failed");
1693 }
1694}
1695
1696pub(crate) async fn tick_route_tasks(
1698 backend: &Arc<dyn ExternalBackend>,
1699 task_manager: &Arc<TaskManager>,
1700 router: &mut Router,
1701 store: &Arc<TaskStore>,
1702 repo: &str,
1703 routing_quota: &mut RoutingTickQuota,
1704) -> anyhow::Result<()> {
1705 let _span = tracing::info_span!("engine.tick.phase3a.route").entered();
1706 if let Err(err) = dependencies::apply_claim_gates(store).await {
1707 tracing::warn!(err = %err, "dependency claim gate failed");
1708 }
1709 if routing_quota.is_exhausted() {
1710 tracing::debug!("routing quota exhausted for this tick — skipping repo routing phase");
1711 return Ok(());
1712 }
1713
1714 if is_github_circuit_open() {
1718 let remaining = github_circuit_remaining_secs();
1719 tracing::info!(
1720 remaining_secs = remaining,
1721 "GitHub 5xx circuit breaker open — skipping routing phase"
1722 );
1723 return Ok(());
1724 }
1725 let new_tasks = task_manager.list_routable().await?;
1726 let routable: Vec<&ExternalTask> = new_tasks
1727 .iter()
1728 .filter(|t| !skips_agent(&t.labels))
1729 .collect();
1730
1731 if !routable.is_empty() {
1735 let start = Instant::now();
1736 router.refresh_health(store).await;
1737 tracing::debug!(
1738 duration_ms = start.elapsed().as_millis(),
1739 "router.refresh_health completed"
1740 );
1741 }
1742
1743 for task in routable.into_iter().take(routing_quota.remaining()) {
1745 if !routing_quota.consume_one() {
1746 break;
1747 }
1748 let _task_span = tracing::info_span!("engine.route", task_id = %task.id.0).entered();
1749
1750 let task_start = Instant::now();
1751 match router.route(task, store, repo).await {
1752 Ok(result) => {
1753 tracing::debug!(task_id = %task.id.0, duration_ms = task_start.elapsed().as_millis(), "route completed");
1754 if let Err(e) = router
1756 .store_route_result(&task.id.0, &result, store, repo)
1757 .await
1758 {
1759 tracing::warn!(
1760 task_id = task.id.0,
1761 ?e,
1762 "failed to store route result; skipping Routed transition"
1763 );
1764 continue;
1765 }
1766
1767 if is_internal_id(&task.id.0) {
1768 if let Err(e) = task_manager
1770 .update_task_status(&task.id, Status::Routed)
1771 .await
1772 {
1773 tracing::warn!(
1774 task_id = task.id.0,
1775 ?e,
1776 "failed to set internal status:routed"
1777 );
1778 continue;
1779 }
1780 } else {
1781 let backend_clone = Arc::clone(backend);
1785 let task_id_clone = task.id.clone();
1786 let task_labels: Vec<String> = task.labels.clone();
1787 let new_labels = {
1788 let mut labels = vec![
1789 format!("agent:{}", result.agent),
1790 format!("complexity:{}", result.complexity),
1791 ];
1792 if let Some(ref model) = result.model {
1793 labels.push(format!("model:{model}"));
1794 }
1795 labels
1796 };
1797 tokio::spawn(async move {
1798 for label in &task_labels {
1800 if (label.starts_with("agent:")
1801 || label.starts_with("complexity:")
1802 || label.starts_with("model:"))
1803 && let Err(e) =
1804 backend_clone.remove_label(&task_id_clone, label).await
1805 {
1806 tracing::warn!(task_id = task_id_clone.0, label, error = %e, "failed to remove stale routing label during re-route");
1807 }
1808 }
1809 if let Err(e) = backend_clone.set_labels(&task_id_clone, &new_labels).await
1811 {
1812 tracing::warn!(
1813 task_id = task_id_clone.0,
1814 ?e,
1815 "failed to set routing labels"
1816 );
1817 }
1818 });
1819
1820 if let Err(e) = task_manager
1822 .update_task_status(&task.id, Status::Routed)
1823 .await
1824 {
1825 tracing::warn!(task_id = task.id.0, ?e, "failed to set status:routed");
1826 continue;
1827 }
1828 }
1829
1830 match store.ensure_external_task(repo, task).await {
1832 Ok(store_id) => {
1833 let profile_json =
1834 serde_json::to_string(&result.profile).unwrap_or_default();
1835 if let Err(e) = store
1836 .store_route(&crate::store::StoreRoute {
1837 id: store_id,
1838 agent: &result.agent,
1839 model: result.model.as_deref(),
1840 complexity: &result.complexity,
1841 estimate: result.estimate,
1842 reason: &result.reason,
1843 profile: &profile_json,
1844 })
1845 .await
1846 {
1847 tracing::debug!(
1848 task_id = task.id.0,
1849 ?e,
1850 "dual-write: store_route failed"
1851 );
1852 }
1853 if let Err(e) = store.update_status(store_id, TaskStatus::Routed).await {
1854 tracing::debug!(
1855 task_id = task.id.0,
1856 ?e,
1857 "dual-write: update_status failed"
1858 );
1859 }
1860 if result.estimate > 0 {
1864 let backend_clone = Arc::clone(backend);
1865 let task_id = task.id.clone();
1866 let estimate = result.estimate;
1867 tokio::spawn(async move {
1868 if let Err(e) = backend_clone
1869 .sync_estimate_to_project(&task_id, estimate)
1870 .await
1871 {
1872 tracing::debug!(
1873 task_id = task_id.0,
1874 estimate,
1875 err = %e,
1876 "dual-write: estimate project sync failed"
1877 );
1878 }
1879 });
1880 }
1881 }
1882 Err(e) => {
1883 tracing::debug!(
1884 task_id = task.id.0,
1885 ?e,
1886 "dual-write: ensure_external_task failed"
1887 );
1888 }
1889 }
1890
1891 if let Some(ref warning) = result.warning {
1892 tracing::warn!(task_id = task.id.0, warning, "routing sanity warning");
1893 }
1894
1895 tracing::info!(
1896 task_id = task.id.0,
1897 agent = %result.agent,
1898 complexity = %result.complexity,
1899 reason = %result.reason,
1900 "task routed"
1901 );
1902 }
1903 Err(e) => {
1904 if let Some(cooled) = e.downcast_ref::<AllAgentsCooledError>() {
1905 tracing::error!(
1906 task_id = %task.id.0,
1907 scope = %cooled.scope(),
1908 remaining_secs = ?cooled.remaining_secs(),
1909 degraded = ?cooled.degraded(),
1910 "ALL AGENTS COOLED — no agent available to route this task. \
1911 Task stays in 'new' and will be retried on every tick. \
1912 Check `orch cooldown list` to see what is blocking dispatch."
1913 );
1914 continue;
1915 }
1916 if let Some(waiting) = e.downcast_ref::<WaitingOnDependency>() {
1917 tracing::info!(
1918 task_id = %task.id.0,
1919 blockers = %waiting.blockers.join(", "),
1920 "task is waiting on a dependency"
1921 );
1922 continue;
1923 }
1924 tracing::error!(task_id = task.id.0, ?e, "routing failed, skipping task");
1925 }
1926 }
1927 }
1928 Ok(())
1929}
1930
1931pub(crate) async fn route_new_tasks_global(
1947 router: &mut Router,
1948 store: &Arc<TaskStore>,
1949 active_repos: &HashSet<String>,
1950 routing_quota: &mut RoutingTickQuota,
1951) -> anyhow::Result<()> {
1952 if routing_quota.is_exhausted() {
1953 tracing::debug!("routing quota exhausted for this tick — skipping global routing sweep");
1954 return Ok(());
1955 }
1956
1957 let all_new = store.list_all_by_status_global(TaskStatus::New).await?;
1958 let orphaned: Vec<&Task> = all_new
1959 .iter()
1960 .filter(|t| !active_repos.contains(&t.repo))
1961 .filter(|t| !skips_agent(&t.labels))
1962 .collect();
1963
1964 if orphaned.is_empty() {
1965 return Ok(());
1966 }
1967
1968 tracing::info!(
1969 count = orphaned.len(),
1970 "global routing sweep: found new tasks from inactive repos"
1971 );
1972
1973 router.refresh_health(store).await;
1974
1975 for task in orphaned.into_iter().take(routing_quota.remaining()) {
1976 if !routing_quota.consume_one() {
1977 break;
1978 }
1979 let ext_task = tasks::store_task_to_external(task);
1980 let task_start = Instant::now();
1981 match router.route(&ext_task, store, &task.repo).await {
1982 Ok(result) => {
1983 tracing::debug!(
1984 task_id = %ext_task.id.0,
1985 repo = %task.repo,
1986 duration_ms = task_start.elapsed().as_millis(),
1987 "global sweep: route completed"
1988 );
1989 let profile_json = serde_json::to_string(&result.profile).unwrap_or_default();
1990 if let Err(e) = store
1991 .store_route(&StoreRoute {
1992 id: task.id,
1993 agent: &result.agent,
1994 model: result.model.as_deref(),
1995 complexity: &result.complexity,
1996 estimate: result.estimate,
1997 reason: &result.reason,
1998 profile: &profile_json,
1999 })
2000 .await
2001 {
2002 tracing::warn!(
2003 task_id = %ext_task.id.0,
2004 repo = %task.repo,
2005 err = %e,
2006 "global sweep: failed to store route result; skipping status update"
2007 );
2008 continue;
2009 }
2010 match store.update_status(task.id, TaskStatus::Routed).await {
2011 Err(e) => {
2012 tracing::warn!(
2013 task_id = %ext_task.id.0,
2014 repo = %task.repo,
2015 err = %e,
2016 "global sweep: failed to set status:routed"
2017 );
2018 }
2019 _ => {
2020 tracing::info!(
2021 task_id = %ext_task.id.0,
2022 repo = %task.repo,
2023 agent = %result.agent,
2024 complexity = %result.complexity,
2025 "global sweep: task routed"
2026 );
2027 }
2028 }
2029 if let Some(ref warning) = result.warning {
2030 tracing::warn!(task_id = %ext_task.id.0, warning, "global sweep: routing sanity warning");
2031 }
2032 }
2033 Err(e) => {
2034 if let Some(cooled) = e.downcast_ref::<AllAgentsCooledError>() {
2035 tracing::error!(
2036 task_id = %ext_task.id.0,
2037 repo = %task.repo,
2038 scope = %cooled.scope(),
2039 remaining_secs = ?cooled.remaining_secs(),
2040 degraded = ?cooled.degraded(),
2041 "global sweep: ALL AGENTS COOLED — task stays in 'new'"
2042 );
2043 continue;
2044 }
2045 if let Some(waiting) = e.downcast_ref::<WaitingOnDependency>() {
2046 tracing::info!(
2047 task_id = %ext_task.id.0,
2048 repo = %task.repo,
2049 blockers = %waiting.blockers.join(", "),
2050 "global sweep: task is waiting on a dependency"
2051 );
2052 continue;
2053 }
2054 tracing::error!(
2055 task_id = %ext_task.id.0,
2056 repo = %task.repo,
2057 err = ?e,
2058 "global sweep: routing failed, skipping task"
2059 );
2060 }
2061 }
2062 }
2063
2064 Ok(())
2065}
2066
2067#[derive(Debug, Clone, Copy)]
2069pub(crate) struct DispatchMode {
2070 pub(crate) is_degraded: bool,
2071 pub(crate) healthy_agents: usize,
2072 pub(crate) threshold: usize,
2073}
2074
2075pub(crate) fn dispatch_mode_from_router(router: &Router) -> DispatchMode {
2076 let threshold = router_config::min_healthy_agents_threshold();
2077 let healthy_agents = router.healthy_agent_count("simple");
2078 let is_degraded = router.is_degraded(threshold);
2079 DispatchMode {
2080 is_degraded,
2081 healthy_agents,
2082 threshold,
2083 }
2084}
2085
2086#[expect(
2088 clippy::too_many_arguments,
2089 reason = "each argument is engine state the tick passes down"
2090)]
2091#[tracing::instrument(skip_all, name = "engine.tick.phase3b.dispatch")]
2092pub(crate) async fn tick_dispatch_tasks(
2093 backend: &Arc<dyn ExternalBackend>,
2094 tmux: &Arc<TmuxManager>,
2095 repo: &str,
2096 runner: &Arc<TaskRunner>,
2097 capture: &Arc<CaptureService>,
2098 semaphore: &Arc<Semaphore>,
2099 task_manager: &Arc<TaskManager>,
2100 weight_tx: &mpsc::Sender<WeightSignal>,
2101 dispatch_mode: DispatchMode,
2102 dispatching: &Arc<DashMap<String, String>>,
2103 store: &Arc<TaskStore>,
2104 session_map: &HashMap<String, bool>,
2105) -> anyhow::Result<()> {
2106 let mut routed_tasks = task_manager.list_external_by_status(Status::Routed).await?;
2108
2109 let internal_routed = task_manager
2111 .list_internal_by_status(DbStatus::Routed)
2112 .await?;
2113 routed_tasks.extend(internal_routed);
2114
2115 let dispatchable: Vec<ExternalTask> = routed_tasks
2117 .into_iter()
2118 .filter(|t| !skips_agent(&t.labels))
2119 .collect();
2120
2121 dispatch_tasks_for_repo(
2122 backend,
2123 tmux,
2124 repo,
2125 runner,
2126 capture,
2127 semaphore,
2128 task_manager,
2129 weight_tx,
2130 dispatch_mode,
2131 dispatching,
2132 store,
2133 session_map,
2134 dispatchable,
2135 )
2136 .await
2137}
2138
2139#[expect(
2145 clippy::too_many_arguments,
2146 reason = "each argument is engine state the tick passes down"
2147)]
2148async fn dispatch_tasks_for_repo(
2149 backend: &Arc<dyn ExternalBackend>,
2150 tmux: &Arc<TmuxManager>,
2151 repo: &str,
2152 runner: &Arc<TaskRunner>,
2153 capture: &Arc<CaptureService>,
2154 semaphore: &Arc<Semaphore>,
2155 task_manager: &Arc<TaskManager>,
2156 weight_tx: &mpsc::Sender<WeightSignal>,
2157 dispatch_mode: DispatchMode,
2158 dispatching: &Arc<DashMap<String, String>>,
2159 store: &Arc<TaskStore>,
2160 session_map: &HashMap<String, bool>,
2161 dispatchable: Vec<ExternalTask>,
2162) -> anyhow::Result<()> {
2163 if dispatchable.is_empty() {
2164 tracing::debug!(count = 0, "dispatchable tasks found");
2165 return Ok(());
2166 }
2167
2168 tracing::info!(count = dispatchable.len(), "dispatchable tasks found");
2169 if dispatch_mode.is_degraded {
2170 tracing::warn!(
2171 healthy_agents = dispatch_mode.healthy_agents,
2172 threshold = dispatch_mode.threshold,
2173 "degraded mode: using sequential dispatch"
2174 );
2175 }
2176
2177 let sequential_delay = if dispatch_mode.is_degraded {
2178 router_config::sequential_dispatch_delay_ms()
2179 } else {
2180 0
2181 };
2182
2183 let mut ranked: Vec<(Priority, ExternalTask)> = Vec::with_capacity(dispatchable.len());
2184 for task in dispatchable {
2185 let has_pr = match store.resolve_id_by_external(repo, &task.id.0).await {
2186 Ok(Some(id)) => store.get(id).await.is_ok_and(|t| t.pr_number.is_some()),
2187 _ => false,
2188 };
2189 ranked.push((Priority::from_labels(&task.labels).with_pr(has_pr), task));
2190 }
2191 priority::sort_by_priority(&mut ranked, |t| t.0);
2192
2193 for (idx, (priority, task)) in ranked.into_iter().enumerate() {
2194 if dependencies::dispatch_is_waiting(store, repo, &task.id.0).await {
2195 continue;
2196 }
2197 if idx > 0 && dispatch_mode.is_degraded {
2199 let delay_ms = sequential_delay;
2200 tracing::debug!(
2201 task_id = task.id.0,
2202 delay_ms = delay_ms,
2203 "sequential dispatch: waiting before dispatch"
2204 );
2205 time::sleep(Duration::from_millis(delay_ms)).await;
2206 }
2207 let dispatch_key = format!("{}/{}", repo, task.id.0);
2214 {
2219 match dispatching.entry(dispatch_key.clone()) {
2220 Entry::Occupied(existing) => {
2221 let existing_id = existing.get().clone();
2222 drop(existing); if existing_id == task.id.0 {
2224 tracing::debug!(
2225 task_id = task.id.0,
2226 "task already dispatching, skipping duplicate"
2227 );
2228 } else {
2229 tracing::warn!(
2230 task_id = task.id.0,
2231 existing_task_id = existing_id,
2232 dispatch_key,
2233 "dispatch key collision: unexpected task already holds this key"
2234 );
2235 }
2236 continue;
2237 }
2238 Entry::Vacant(slot) => {
2239 slot.insert(task.id.0.clone());
2240 tracing::info!(task_id = task.id.0, dispatch_key, "dispatch guard claimed");
2247 }
2248 }
2249 }
2250 let dispatch_guard = DispatchGuard::new(Arc::clone(dispatching), dispatch_key.clone());
2252 let task_id = task.id.0.clone();
2253
2254 let route_result = match get_route_result(store, repo, &task_id).await {
2257 Ok(r) => Some(r),
2258 Err(RouteResultError::NoAgent { .. }) => {
2259 tracing::warn!(
2260 task_id,
2261 "routed task missing agent — resetting to new for re-routing (#1604)"
2262 );
2263 if let Err(e2) = task_manager.update_task_status(&task.id, Status::New).await {
2264 tracing::error!(task_id, error = %e2, "failed to reset task to new");
2265 }
2266 continue; }
2268 Err(e) => {
2269 tracing::warn!(task_id, error = %e, "get_route_result failed — dispatching without route info");
2270 None
2271 }
2272 };
2273
2274 if let Some(route) = route_result.as_ref() {
2280 let agent_cooled = is_agent_in_cooldown(&route.agent);
2281 let model_cooled = route
2282 .model
2283 .as_deref()
2284 .is_some_and(|model| is_model_in_cooldown(&route.agent, model));
2285 if agent_cooled || model_cooled {
2286 tracing::warn!(
2287 task_id,
2288 agent = route.agent,
2289 model = route.model.as_deref().unwrap_or(""),
2290 "routed agent/model entered cooldown after routing — deferring dispatch to next tick"
2291 );
2292 continue; }
2294 }
2295
2296 let session_name = tmux.session_name(repo, &task_id);
2298 if tmux
2299 .session_blocks_dispatch_from_map(&session_name, session_map)
2300 .await
2301 {
2302 tracing::warn!(
2303 task_id,
2304 session_name,
2305 "task has existing tmux session, skipping dispatch"
2306 );
2307 continue; }
2309
2310 let Some(permit) = priority::acquire_dispatch_slot(semaphore, priority) else {
2312 tracing::debug!("all parallel slots busy, skipping remaining tasks");
2313 break; };
2315
2316 let set_in_progress_result = task_manager
2318 .update_task_status(&task.id, Status::InProgress)
2319 .await;
2320 if let Err(e) = set_in_progress_result {
2321 tracing::error!(task_id, ?e, "failed to set in_progress, skipping dispatch");
2322 drop(permit);
2323 continue; }
2325 tracing::info!(task_id, "dispatching task");
2326
2327 capture
2329 .register_session(repo, &task_id, &session_name)
2330 .await;
2331
2332 let runner = Arc::clone(runner);
2334 let backend = Arc::clone(backend);
2335 let tmux = Arc::clone(tmux);
2336 let capture = Arc::clone(capture);
2337 let task_id_for_cleanup = task_id.clone();
2338 let task_owned = task;
2339 let weight_tx = weight_tx.clone();
2340 let repo_owned = repo.to_string();
2341 let task_manager_for_spawn = Arc::clone(task_manager);
2342 let store_for_spawn = Arc::clone(store);
2343
2344 let repo_ctx = repo_owned.clone();
2345 tokio::spawn(REPO_CONTEXT.scope(repo_ctx, async move {
2346 let _dispatch_guard = dispatch_guard; let dispatch_start = Instant::now();
2348 match runner
2349 .run_with_context(&task_owned, &backend, &tmux, route_result.as_ref())
2350 .await
2351 {
2352 Ok(signal) => {
2353 tracing::info!(task_id, "task runner completed");
2354
2355 let duration = dispatch_start.elapsed().as_secs_f64();
2356
2357 let enable_review = config::get("workflow.enable_review_agent")
2363 .map_or(true, |v| v != "false");
2364 let has_pr = if enable_review && matches!(signal, WeightSignal::Success { .. })
2366 {
2367 match store_for_spawn.resolve_task_id(&repo_owned, &task_id).await {
2368 Ok(Some(store_id)) => match store_for_spawn.get(store_id).await {
2369 Ok(t) => t.pr_number.is_some(),
2370 Err(e) => {
2371 tracing::warn!(task_id, err = %e, "store.get() failed — defaulting to needs_review");
2372 true
2373 }
2374 },
2375 Ok(None) => false,
2376 Err(e) => {
2377 tracing::warn!(task_id, err = %e, "failed to check PR status — defaulting to needs_review");
2378 true
2379 }
2380 }
2381 } else {
2382 false
2383 };
2384 let display_status = display_status_for(&signal, enable_review, has_pr);
2385
2386 if display_status == "done" {
2387 tracing::info!(
2388 task_id,
2389 "marking done: the run succeeded with no PR to review"
2390 );
2391 }
2392 tracing::info!(task_id, display_status, "task run finished, applying status");
2393
2394 let _ = weight_tx.send(signal).await;
2396
2397 if display_status == "needs_review" {
2401 if let Err(e) = task_manager_for_spawn
2404 .update_task_status_with_duration(
2405 &ExternalId(task_id.clone()),
2406 Status::NeedsReview,
2407 Some(duration),
2408 )
2409 .await
2410 {
2411 tracing::error!(task_id, err = %e, "update_task_status(NeedsReview) failed — task may be stuck");
2412 }
2413 } else {
2415 let final_status = match display_status {
2417 "done" => Status::Done,
2418 "blocked" => Status::Blocked,
2419 _ => Status::New,
2420 };
2421 if let Err(e) = task_manager_for_spawn
2422 .update_task_status_with_duration(
2423 &ExternalId(task_id.clone()),
2424 final_status,
2425 Some(duration),
2426 )
2427 .await
2428 {
2429 tracing::warn!(task_id, ?e, "failed to update task status after completion");
2430 }
2431 }
2432 }
2433 Err(e) => {
2434 tracing::error!(task_id, ?e, "task runner failed");
2435 let duration = dispatch_start.elapsed().as_secs_f64();
2436 if e.downcast_ref::<NoProjectDir>().is_some() {
2437 block_task_without_project(
2439 &task_manager_for_spawn,
2440 &store_for_spawn,
2441 &repo_owned,
2442 &task_id,
2443 )
2444 .await;
2445 } else if is_internal_id(&task_id) {
2446 if let Err(ue) = task_manager_for_spawn
2448 .update_task_status_with_duration(
2449 &ExternalId(task_id.clone()),
2450 Status::NeedsReview,
2451 Some(duration),
2452 )
2453 .await
2454 {
2455 tracing::error!(task_id, err = %ue, "update_task_status(NeedsReview) failed — task may be stuck");
2456 }
2457 } else {
2458 if let Err(comment_err) = backend
2460 .post_comment(
2461 &ExternalId(task_id.clone()),
2462 &format!(
2463 "[{}] error: task runner failed: {e}{}",
2464 chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
2465 engine::orch_footer()
2466 ),
2467 )
2468 .await
2469 {
2470 tracing::warn!(
2471 task_id,
2472 ?comment_err,
2473 "failed to post error comment to GitHub"
2474 );
2475 }
2476 if let Err(ue) = task_manager_for_spawn
2480 .update_task_status_with_duration(
2481 &ExternalId(task_id.clone()),
2482 Status::NeedsReview,
2483 Some(duration),
2484 )
2485 .await
2486 {
2487 tracing::error!(task_id, err = %ue, "update_task_status(NeedsReview) failed — task may be stuck");
2488 }
2489 }
2490 }
2491 }
2492
2493 capture.unregister_session(&repo_owned, &task_id_for_cleanup).await;
2495
2496 drop(permit);
2498 }));
2500 }
2501 Ok(())
2502}
2503
2504async fn block_task_without_project(
2507 task_manager: &Arc<TaskManager>,
2508 store: &Arc<TaskStore>,
2509 repo: &str,
2510 task_id: &str,
2511) {
2512 let store_id = match store.resolve_task_id(repo, task_id).await {
2513 Ok(Some(id)) => id,
2514 Ok(None) => {
2515 tracing::error!(
2516 task_id,
2517 repo,
2518 "cannot block task without project: not in the store"
2519 );
2520 return;
2521 }
2522 Err(e) => {
2523 tracing::error!(task_id, repo, err = %e, "cannot block task without project: store lookup failed");
2524 return;
2525 }
2526 };
2527 let fields = [
2528 (
2529 "block_reason",
2530 serde_json::json!(NoProjectDir::BLOCK_REASON),
2531 ),
2532 (
2533 "last_error",
2534 serde_json::json!(format!("no project directory for repo {repo}")),
2535 ),
2536 ];
2537 if let Err(e) = task_manager
2538 .update_task_status_and_result_by_store_id(
2539 store_id,
2540 repo,
2541 &ExternalId(task_id.to_string()),
2542 Status::Blocked,
2543 &fields,
2544 )
2545 .await
2546 {
2547 tracing::error!(task_id, repo, err = %e, "failed to block task without project");
2548 }
2549}
2550
2551fn skips_agent(labels: &[String]) -> bool {
2552 labels
2553 .iter()
2554 .any(|label| label == "no-agent" || label == "owner:human")
2555}
2556
2557fn group_orphaned_routed_tasks(
2564 all_routed: Vec<Task>,
2565 active_repos: &HashSet<String>,
2566) -> HashMap<String, Vec<Task>> {
2567 let mut by_repo: HashMap<String, Vec<Task>> = HashMap::new();
2568 for task in all_routed
2569 .into_iter()
2570 .filter(|t| !active_repos.contains(&t.repo))
2571 .filter(|t| !skips_agent(&t.labels))
2572 {
2573 by_repo.entry(task.repo.clone()).or_default().push(task);
2574 }
2575 by_repo
2576}
2577
2578#[expect(
2597 clippy::too_many_arguments,
2598 reason = "each argument is engine state the tick passes down"
2599)]
2600pub(crate) async fn dispatch_routed_tasks_global(
2601 tmux: &Arc<TmuxManager>,
2602 capture: &Arc<CaptureService>,
2603 semaphore: &Arc<Semaphore>,
2604 weight_tx: &mpsc::Sender<WeightSignal>,
2605 dispatch_mode: DispatchMode,
2606 dispatching: &Arc<DashMap<String, String>>,
2607 store: &Arc<TaskStore>,
2608 session_map: &HashMap<String, bool>,
2609 active_repos: &HashSet<String>,
2610) -> anyhow::Result<()> {
2611 let all_routed = store.list_all_by_status_global(TaskStatus::Routed).await?;
2612 let by_repo = group_orphaned_routed_tasks(all_routed, active_repos);
2613
2614 if by_repo.is_empty() {
2615 return Ok(());
2616 }
2617
2618 tracing::info!(
2619 repos = by_repo.len(),
2620 "global dispatch sweep: found routed tasks from inactive repos"
2621 );
2622
2623 for (repo, tasks) in by_repo {
2624 let backend: Arc<dyn ExternalBackend> = match GitHubBackend::new(repo.clone()) {
2625 Ok(b) => Arc::new(b),
2626 Err(e) => {
2627 tracing::warn!(repo = %repo, err = %e, "global sweep: failed to construct backend, skipping repo");
2628 continue;
2629 }
2630 };
2631 let task_manager = Arc::new(TaskManager::with_store(
2632 Arc::clone(&backend),
2633 Arc::clone(store),
2634 repo.clone(),
2635 ));
2636 let runner = Arc::new(TaskRunner::new(repo.clone()).with_store(Arc::clone(store)));
2637
2638 if let Err(e) = runner.resolve_project_dir().await
2641 && e.downcast_ref::<NoProjectDir>().is_some()
2642 {
2643 tracing::warn!(repo = %repo, count = tasks.len(), "global sweep: repo has no project directory, blocking its routed tasks");
2644 for task in &tasks {
2645 let ext = task
2646 .external_id
2647 .clone()
2648 .unwrap_or_else(|| format!("internal:{}", task.id));
2649 block_task_without_project(&task_manager, store, &repo, &ext).await;
2650 }
2651 continue;
2652 }
2653
2654 let dispatchable: Vec<ExternalTask> =
2655 tasks.iter().map(tasks::store_task_to_external).collect();
2656
2657 if let Err(e) = dispatch_tasks_for_repo(
2658 &backend,
2659 tmux,
2660 &repo,
2661 &runner,
2662 capture,
2663 semaphore,
2664 &task_manager,
2665 weight_tx,
2666 dispatch_mode,
2667 dispatching,
2668 store,
2669 session_map,
2670 dispatchable,
2671 )
2672 .await
2673 {
2674 tracing::warn!(repo = %repo, err = %e, "global sweep: dispatch failed for repo");
2675 }
2676 }
2677
2678 Ok(())
2679}
2680
2681pub(crate) async fn tick_unblock_parents(
2683 backend: &Arc<dyn ExternalBackend>,
2684 task_manager: &Arc<TaskManager>,
2685 store: &Arc<TaskStore>,
2686 repo: &str,
2687) -> anyhow::Result<()> {
2688 let blocked = task_manager.list_all_by_status(Status::Blocked).await?;
2689
2690 let store_blocked = store.list_by_status(repo, TaskStatus::Blocked).await?;
2694 let has_block_reason: HashSet<String> = store_blocked
2695 .iter()
2696 .filter(|t| t.block_reason.is_some())
2697 .filter_map(|t| t.external_id.clone())
2698 .collect();
2699
2700 let handles: Vec<_> = blocked
2704 .into_iter()
2705 .filter(|task| {
2706 if has_block_reason.contains(&task.id.0) {
2707 tracing::debug!(
2708 task_id = task.id.0,
2709 "skipping blocked task with known block_reason"
2710 );
2711 false
2712 } else {
2713 true
2714 }
2715 })
2716 .map(|task| {
2717 let backend_clone = Arc::clone(backend);
2718 let task_manager_clone = Arc::clone(task_manager);
2719 let store_clone = Arc::clone(store);
2720 let repo_owned = repo.to_string();
2721 tokio::spawn(async move {
2722 let task_id = &task.id;
2723
2724 let mut children: Vec<ExternalId> = Vec::new();
2725 let mut child_statuses: HashMap<
2726 String,
2727 TaskStatus,
2728 > = HashMap::new();
2729
2730 match store_clone.resolve_task_id(&repo_owned, &task_id.0).await {
2731 Ok(Some(parent_store_id)) => {
2732 match store_clone.list_children(&repo_owned, parent_store_id).await {
2733 Ok(store_children) => {
2734 for child in store_children {
2735 let Some(child_external_id) = child.external_id else {
2736 continue;
2737 };
2738 child_statuses.insert(child_external_id.clone(), child.status);
2739 children.push(ExternalId(child_external_id));
2740 }
2741 }
2742 Err(e) => {
2743 tracing::debug!(
2744 task_id = task_id.0,
2745 ?e,
2746 "failed to list store-linked children"
2747 );
2748 }
2749 }
2750 }
2751 Ok(None) => {}
2752 Err(e) => {
2753 tracing::debug!(
2754 task_id = task_id.0,
2755 ?e,
2756 "failed to resolve blocked task in store for child lookup"
2757 );
2758 }
2759 }
2760
2761 if !is_internal_id(&task_id.0) {
2762 match backend_clone.get_sub_issues(task_id).await {
2763 Ok(ids) => {
2764 let mut seen: HashSet<String> =
2765 children.iter().map(|child| child.0.clone()).collect();
2766 for id in ids {
2767 if seen.insert(id.0.clone()) {
2768 children.push(id);
2769 }
2770 }
2771 }
2772 Err(e) => {
2773 tracing::debug!(task_id = task_id.0, ?e, "failed to get sub-issues");
2774 }
2775 }
2776 }
2777
2778 if children.is_empty() {
2780 return;
2781 }
2782
2783 let mut all_done = true;
2786 let child_ext_strs: Vec<String> =
2788 children.iter().map(|c| c.0.clone()).collect();
2789 let child_exts: Vec<&str> =
2790 child_ext_strs.iter().map(String::as_str).collect();
2791 match store_clone
2793 .get_statuses_by_external_ids(&repo_owned, &child_exts)
2794 .await
2795 { Ok(map) => {
2796 for (k, v) in map {
2797 child_statuses.insert(k, v);
2798 }
2799 } _ => {
2800 tracing::debug!(
2801 task_id = task_id.0,
2802 "failed to batch-query store for child statuses; falling back to per-child lookups"
2803 );
2804 }}
2805
2806 for child_id in &children {
2807 if let Some(status) = child_statuses.get(&child_id.0) {
2809 if *status != TaskStatus::Done {
2810 all_done = false;
2811 break;
2812 }
2813 continue;
2814 }
2815
2816 match backend_clone.get_task(child_id).await {
2818 Ok(child) => {
2819 if !child.labels.iter().any(|l| l == Status::Done.as_label()) {
2820 all_done = false;
2821 break;
2822 }
2823 }
2824 Err(e) => {
2825 tracing::debug!(task_id = task_id.0, child = %child_id.0, ?e, "failed to fetch child task from backend — treating as not done");
2826 all_done = false;
2827 break;
2828 }
2829 }
2830 }
2831
2832 if all_done {
2833 tracing::info!(
2834 task_id = task_id.0,
2835 children = children.len(),
2836 "all children done, unblocking parent"
2837 );
2838 if let Err(e) =
2839 task_manager_clone.update_task_status(task_id, Status::New).await
2840 {
2841 tracing::warn!(task_id = task_id.0, ?e, "failed to unblock parent");
2842 }
2843 }
2844 })
2845 })
2846 .collect();
2847
2848 for handle in handles {
2849 if let Err(e) = handle.await {
2850 tracing::warn!(?e, "phase 4 sub-issue check panicked");
2851 }
2852 }
2853 Ok(())
2854}
2855
2856pub(crate) async fn tick_job_scheduler(
2858 jobs_path: &Path,
2859 backend: &Arc<dyn ExternalBackend>,
2860 store: Option<&Arc<TaskStore>>,
2861 repo: &str,
2862 transport: Option<&Arc<Transport>>,
2863) -> anyhow::Result<()> {
2864 jobs::tick(jobs_path, backend, store, repo, transport).await
2865}
2866
2867#[expect(
2877 clippy::too_many_arguments,
2878 reason = "each argument is engine state the tick passes down"
2879)]
2880pub(crate) async fn tick_before_routing(
2881 backend: &Arc<dyn ExternalBackend>,
2882 tmux: &Arc<TmuxManager>,
2883 repo: &str,
2884 capture: &Arc<CaptureService>,
2885 config: &EngineConfig,
2886 task_manager: &Arc<TaskManager>,
2887 dispatching: &Arc<DashMap<String, String>>,
2888 store: &Arc<TaskStore>,
2889 session_map: &HashMap<String, bool>,
2890) {
2891 let mut phases: Vec<(&str, u128)> = Vec::new();
2892 let mut at = Instant::now();
2893
2894 if let Err(e) = tick_check_session_completions(tmux, repo, capture, store).await {
2895 tracing::warn!(err = %e, "session completion phase failed");
2896 }
2897 phases.push(("sessions", at.elapsed().as_millis()));
2898
2899 if !store.is_leader() {
2902 super::log_phase_timings("tick.pre_routing", repo, &phases);
2903 return;
2904 }
2905
2906 at = Instant::now();
2907 if let Err(e) =
2908 tick_detect_silent_agents(tmux, repo, capture, backend, task_manager, config, store).await
2909 {
2910 tracing::warn!(err = %e, "silent-agent detection phase failed");
2911 }
2912 phases.push(("silent_agents", at.elapsed().as_millis()));
2913
2914 at = Instant::now();
2915 if let Err(e) = tick_recover_stuck_tasks(
2916 backend,
2917 tmux,
2918 repo,
2919 task_manager,
2920 config,
2921 store,
2922 session_map,
2923 dispatching,
2924 )
2925 .await
2926 {
2927 tracing::warn!(err = %e, "stuck-task recovery phase failed");
2928 }
2929 phases.push(("stuck_tasks", at.elapsed().as_millis()));
2930
2931 super::log_phase_timings("tick.pre_routing", repo, &phases);
2932}
2933
2934#[expect(
2942 clippy::too_many_arguments,
2943 reason = "each argument is engine state the tick passes down"
2944)]
2945pub(crate) async fn tick_after_routing(
2946 backend: &Arc<dyn ExternalBackend>,
2947 tmux: &Arc<TmuxManager>,
2948 repo: &str,
2949 runner: &Arc<TaskRunner>,
2950 capture: &Arc<CaptureService>,
2951 semaphore: &Arc<Semaphore>,
2952 project_dir: &Path,
2953 task_manager: &Arc<TaskManager>,
2954 weight_tx: &mpsc::Sender<WeightSignal>,
2955 dispatch_mode: DispatchMode,
2956 dispatching: &Arc<DashMap<String, String>>,
2957 store: &Arc<TaskStore>,
2958 transport: Option<&Arc<Transport>>,
2959 session_map: &HashMap<String, bool>,
2960) {
2961 let mut phases: Vec<(&str, u128)> = Vec::new();
2962 let mut at = Instant::now();
2963
2964 if let Err(e) = tick_dispatch_tasks(
2965 backend,
2966 tmux,
2967 repo,
2968 runner,
2969 capture,
2970 semaphore,
2971 task_manager,
2972 weight_tx,
2973 dispatch_mode,
2974 dispatching,
2975 store,
2976 session_map,
2977 )
2978 .await
2979 {
2980 tracing::warn!(err = %e, "dispatch phase failed");
2981 }
2982 phases.push(("dispatch", at.elapsed().as_millis()));
2983
2984 at = Instant::now();
2985 if let Err(e) = tick_unblock_parents(backend, task_manager, store, repo).await {
2986 tracing::warn!(err = %e, "parent-unblock phase failed");
2987 }
2988 phases.push(("unblock_parents", at.elapsed().as_millis()));
2989
2990 at = Instant::now();
2991 let jobs_path = project_dir.join(".orch.yml");
2992 if let Err(e) = tick_job_scheduler(&jobs_path, backend, Some(store), repo, transport).await {
2993 tracing::error!(?e, "job scheduler tick failed");
2994 }
2995 phases.push(("jobs", at.elapsed().as_millis()));
2996
2997 super::log_phase_timings("tick.post_routing", repo, &phases);
2998}
2999
3000#[expect(
3010 clippy::too_many_arguments,
3011 reason = "each argument is engine state the tick passes down"
3012)]
3013pub(crate) async fn tick_all(
3014 engines: &[ProjectEngine],
3015 tmux: &Arc<TmuxManager>,
3016 capture: &Arc<CaptureService>,
3017 semaphore: &Arc<Semaphore>,
3018 config: &EngineConfig,
3019 router: &Arc<RwLock<Router>>,
3020 weight_tx: &mpsc::Sender<WeightSignal>,
3021 dispatching: &Arc<DashMap<String, String>>,
3022 transport: Option<&Arc<Transport>>,
3023 active_repos: &HashSet<String>,
3024 auto_merge_in_flight: &Arc<DashSet<String>>,
3025) {
3026 let _tick_span = tracing::info_span!("engine.tick").entered();
3027
3028 let Some(first) = engines.first() else {
3029 return;
3030 };
3031 let store = &first.store;
3034
3035 for (project, call_type, calls) in http::drain_call_counts().into_iter().take(10) {
3037 tracing::debug!(
3038 project,
3039 call_type,
3040 calls,
3041 "github api calls since last tick"
3042 );
3043 }
3044
3045 let session_map = tmux.batch_session_active().await;
3047
3048 let mut passes = Vec::with_capacity(engines.len());
3049 for engine in engines {
3050 passes.push(REPO_CONTEXT.scope(
3051 engine.repo.clone(),
3052 tick_before_routing(
3053 &engine.backend,
3054 tmux,
3055 &engine.repo,
3056 capture,
3057 config,
3058 &engine.task_manager,
3059 dispatching,
3060 &engine.store,
3061 &session_map,
3062 ),
3063 ));
3064 }
3065 future::join_all(passes).await;
3066
3067 if !store.is_leader() {
3068 return;
3069 }
3070
3071 let at = Instant::now();
3075 {
3076 let router_guard = router.read().await;
3077 tick_global_sweeps(
3078 &first.task_manager,
3079 store,
3080 &router_guard,
3081 dispatching,
3082 auto_merge_in_flight,
3083 active_repos,
3084 )
3085 .await;
3086 }
3087 let sweeps_ms = at.elapsed().as_millis();
3088
3089 let at = Instant::now();
3090 recover_untracked_repo_stuck_tasks(
3091 tmux,
3092 config,
3093 store,
3094 &session_map,
3095 dispatching,
3096 active_repos,
3097 )
3098 .await;
3099 let untracked_ms = at.elapsed().as_millis();
3100
3101 let at = Instant::now();
3102 if let Err(err) = dependencies::reconcile_dependencies(
3103 first.backend.as_ref(),
3104 store,
3105 transport.map(AsRef::as_ref),
3106 )
3107 .await
3108 {
3109 tracing::warn!(err = %err, "dependency reconcile failed");
3110 }
3111 super::log_phase_timings(
3112 "tick.global",
3113 &first.repo,
3114 &[
3115 ("global_sweeps", sweeps_ms),
3116 ("untracked_stuck", untracked_ms),
3117 ("dependencies", at.elapsed().as_millis()),
3118 ],
3119 );
3120
3121 {
3122 let mut router_guard = router.write().await;
3123 let mut routing_quota = RoutingTickQuota::new(router_config::max_tasks_per_routing_tick());
3124 for engine in engines {
3125 let at = Instant::now();
3126 REPO_CONTEXT
3127 .scope(engine.repo.clone(), async {
3128 if let Err(e) = tick_route_tasks(
3129 &engine.backend,
3130 &engine.task_manager,
3131 &mut router_guard,
3132 &engine.store,
3133 &engine.repo,
3134 &mut routing_quota,
3135 )
3136 .await
3137 {
3138 tracing::warn!(err = %e, "routing phase failed");
3139 }
3140 })
3141 .await;
3142 super::log_phase_timings(
3143 "tick.routing",
3144 &engine.repo,
3145 &[("route", at.elapsed().as_millis())],
3146 );
3147 }
3148
3149 if let Err(e) =
3153 route_new_tasks_global(&mut router_guard, store, active_repos, &mut routing_quota).await
3154 {
3155 tracing::warn!(err = %e, "global new-task routing sweep failed");
3156 }
3157 }
3158
3159 let dispatch_mode = dispatch_mode_from_router(&*router.read().await);
3160
3161 let session_map = tmux.batch_session_active().await;
3164
3165 let mut passes = Vec::with_capacity(engines.len());
3166 for engine in engines {
3167 passes.push(REPO_CONTEXT.scope(
3168 engine.repo.clone(),
3169 tick_after_routing(
3170 &engine.backend,
3171 tmux,
3172 &engine.repo,
3173 &engine.runner,
3174 capture,
3175 semaphore,
3176 &engine.project_dir,
3177 &engine.task_manager,
3178 weight_tx,
3179 dispatch_mode,
3180 dispatching,
3181 &engine.store,
3182 transport,
3183 &session_map,
3184 ),
3185 ));
3186 }
3187 future::join_all(passes).await;
3188
3189 if let Err(e) = dispatch_routed_tasks_global(
3195 tmux,
3196 capture,
3197 semaphore,
3198 weight_tx,
3199 dispatch_mode,
3200 dispatching,
3201 store,
3202 &session_map,
3203 active_repos,
3204 )
3205 .await
3206 {
3207 tracing::warn!(err = %e, "global routed-task dispatch sweep failed");
3208 }
3209}
3210
3211pub(crate) fn display_status_for(
3214 signal: &WeightSignal,
3215 enable_review: bool,
3216 has_pr: bool,
3217) -> &'static str {
3218 match signal {
3219 WeightSignal::Success { .. } if enable_review => {
3220 if has_pr {
3221 "needs_review"
3222 } else {
3223 "done"
3224 }
3225 }
3226 WeightSignal::RateLimited { .. } | WeightSignal::Rerouted => "new",
3229 WeightSignal::Blocked => "blocked",
3230 WeightSignal::Failed { .. } => "needs_review",
3232 WeightSignal::None if enable_review => "needs_review",
3234 WeightSignal::Success { .. } | WeightSignal::None => "done",
3235 }
3236}
3237
3238#[cfg(test)]
3239mod tests {
3240 use super::*;
3241 use crate::backends::{ExternalId, ExternalTask, Mention, Status};
3242 use crate::channels::transport::Transport;
3243 use crate::engine::RunTimeouts;
3244 use crate::engine::cooldown;
3245 use crate::engine::router::Router;
3246 use crate::engine::router::RouterConfig;
3247 use crate::engine::tasks::TaskManager;
3248 use crate::store::DepSource;
3249 use crate::store::NewDependency;
3250 use crate::store::NewTask;
3251 use crate::store::StoreRoute;
3252 use crate::store::TaskStatus as DbStatus;
3253 use crate::store::TaskStatus;
3254 use crate::store::TaskStore;
3255 use crate::store::UpsertExternal;
3256 use crate::tmux::{SessionGuard, TmuxManager};
3257 use async_trait::async_trait;
3258 use dashmap::mapref::entry::Entry;
3259 use std::cell::Cell;
3260 use std::fs;
3261 use std::path::PathBuf;
3262 use std::sync::{Arc, Mutex};
3263 use tokio::sync::RwLock;
3264 use tokio::sync::oneshot;
3265 use tokio::task;
3266
3267 #[test]
3268 fn timed_out_run_without_pr_never_becomes_done() {
3269 let failed = WeightSignal::Failed {
3270 agent: "claude".to_string(),
3271 };
3272 assert_eq!(display_status_for(&failed, true, false), "needs_review");
3273 assert_eq!(display_status_for(&failed, false, false), "needs_review");
3274 }
3275
3276 #[test]
3277 fn done_needs_a_successful_run_with_no_pr() {
3278 let ok = WeightSignal::Success {
3279 agent: "claude".to_string(),
3280 };
3281 assert_eq!(display_status_for(&ok, true, true), "needs_review");
3282 assert_eq!(display_status_for(&ok, true, false), "done");
3283 assert_eq!(
3284 display_status_for(&WeightSignal::Rerouted, true, false),
3285 "new"
3286 );
3287 assert_eq!(
3288 display_status_for(&WeightSignal::None, true, false),
3289 "needs_review"
3290 );
3291 }
3292 #[test]
3293 fn skips_agent_for_human_and_no_agent_labels() {
3294 assert!(skips_agent(&["owner:human".to_string()]));
3295 assert!(skips_agent(&["no-agent".to_string()]));
3296 assert!(!skips_agent(&["status:new".to_string()]));
3297 }
3298
3299 fn cooled_review_router_for_test() -> Router {
3303 Router::new_for_test(RouterConfig::default(), vec!["claude".to_string()])
3304 }
3305
3306 struct MockBackend {
3310 blocked_tasks: Vec<ExternalTask>,
3312 sub_issues: HashMap<String, Vec<ExternalId>>,
3314 tasks_by_id: HashMap<String, ExternalTask>,
3316 get_sub_issues_calls: Arc<Mutex<Vec<String>>>,
3318 status_updates: Arc<Mutex<Vec<(String, Status)>>>,
3320 removed_labels: Arc<Mutex<Vec<(String, String)>>>,
3322 posted_comments: Arc<Mutex<Vec<(String, String)>>>,
3324 }
3325
3326 impl MockBackend {
3327 fn new() -> Self {
3328 Self {
3329 blocked_tasks: vec![],
3330 sub_issues: HashMap::new(),
3331 tasks_by_id: HashMap::new(),
3332 get_sub_issues_calls: Arc::new(Mutex::new(vec![])),
3333 status_updates: Arc::new(Mutex::new(vec![])),
3334 removed_labels: Arc::new(Mutex::new(vec![])),
3335 posted_comments: Arc::new(Mutex::new(vec![])),
3336 }
3337 }
3338 }
3339
3340 fn make_task(id: &str, labels: &[&str]) -> ExternalTask {
3341 ExternalTask {
3342 id: ExternalId(id.to_string()),
3343 title: format!("Task {id}"),
3344 body: String::new(),
3345 state: "open".to_string(),
3346 labels: labels.iter().map(ToString::to_string).collect(),
3347 author: "bot".to_string(),
3348 created_at: "2026-01-01T00:00:00Z".to_string(),
3349 updated_at: "2026-01-01T00:00:00Z".to_string(),
3350 url: format!("https://github.com/test/test/issues/{id}"),
3351 }
3352 }
3353
3354 #[async_trait]
3355 impl ExternalBackend for MockBackend {
3356 fn name(&self) -> &'static str {
3357 "mock"
3358 }
3359 async fn create_task(
3360 &self,
3361 _title: &str,
3362 _body: &str,
3363 _labels: &[String],
3364 ) -> anyhow::Result<ExternalId> {
3365 Ok(ExternalId("new".to_string()))
3366 }
3367 async fn get_task(&self, id: &ExternalId) -> anyhow::Result<ExternalTask> {
3368 self.tasks_by_id
3369 .get(&id.0)
3370 .cloned()
3371 .ok_or_else(|| anyhow::anyhow!("task not found: {}", id.0))
3372 }
3373 async fn list_by_status(&self, status: Status) -> anyhow::Result<Vec<ExternalTask>> {
3374 if status == Status::Blocked {
3375 Ok(self.blocked_tasks.clone())
3376 } else {
3377 Ok(vec![])
3378 }
3379 }
3380 async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
3381 Ok(vec![])
3382 }
3383 async fn post_comment(&self, _id: &ExternalId, _body: &str) -> anyhow::Result<()> {
3384 self.posted_comments
3385 .lock()
3386 .unwrap()
3387 .push((_id.0.clone(), _body.to_string()));
3388 Ok(())
3389 }
3390 async fn set_labels(&self, _id: &ExternalId, _labels: &[String]) -> anyhow::Result<()> {
3391 Ok(())
3392 }
3393 async fn remove_label(&self, _id: &ExternalId, _label: &str) -> anyhow::Result<()> {
3394 self.removed_labels
3395 .lock()
3396 .unwrap()
3397 .push((_id.0.clone(), _label.to_string()));
3398 Ok(())
3399 }
3400 async fn get_sub_issues(&self, id: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
3401 self.get_sub_issues_calls.lock().unwrap().push(id.0.clone());
3402 Ok(self.sub_issues.get(&id.0).cloned().unwrap_or_default())
3403 }
3404 async fn create_sub_task(
3405 &self,
3406 _parent: &ExternalId,
3407 _title: &str,
3408 _body: &str,
3409 _labels: &[String],
3410 ) -> anyhow::Result<ExternalId> {
3411 Ok(ExternalId("child".to_string()))
3412 }
3413 async fn ensure_status_label(&self, _label: &str) -> anyhow::Result<()> {
3414 Ok(())
3415 }
3416 async fn has_open_issue_with_title(
3417 &self,
3418 _title: &str,
3419 _label: &str,
3420 ) -> anyhow::Result<bool> {
3421 Ok(false)
3422 }
3423 async fn health_check(&self) -> anyhow::Result<()> {
3424 Ok(())
3425 }
3426 async fn is_pr_merged(&self, _branch: &str) -> anyhow::Result<bool> {
3427 Ok(false)
3428 }
3429 async fn get_authenticated_user(&self) -> anyhow::Result<Option<String>> {
3430 Ok(Some("testbot".to_string()))
3431 }
3432 async fn get_mentions(&self, _since: &str) -> anyhow::Result<Vec<Mention>> {
3433 Ok(vec![])
3434 }
3435 async fn update_status(&self, id: &ExternalId, status: Status) -> anyhow::Result<()> {
3436 self.status_updates
3437 .lock()
3438 .unwrap()
3439 .push((id.0.clone(), status));
3440 Ok(())
3441 }
3442 }
3443
3444 fn make_task_manager(backend: Arc<dyn ExternalBackend>) -> Arc<TaskManager> {
3445 Arc::new(TaskManager::new(backend))
3446 }
3447
3448 #[tokio::test]
3451 async fn unblock_parents_unblocks_when_all_children_done() {
3452 let mut mock = MockBackend::new();
3453
3454 let parent = make_task("10", &["status:blocked"]);
3456 mock.blocked_tasks.push(parent.clone());
3457 mock.sub_issues.insert(
3458 "10".to_string(),
3459 vec![ExternalId("11".to_string()), ExternalId("12".to_string())],
3460 );
3461 mock.tasks_by_id
3462 .insert("11".to_string(), make_task("11", &["status:done"]));
3463 mock.tasks_by_id
3464 .insert("12".to_string(), make_task("12", &["status:done"]));
3465
3466 let status_updates = Arc::clone(&mock.status_updates);
3467 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3468 let task_manager = make_task_manager(Arc::clone(&backend));
3469 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3470
3471 tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3472 .await
3473 .unwrap();
3474
3475 let updates = status_updates.lock().unwrap();
3476 assert_eq!(updates.len(), 1, "parent should be unblocked");
3477 assert_eq!(updates[0], ("10".to_string(), Status::New));
3478 }
3479
3480 #[tokio::test]
3481 async fn unblock_parents_skips_when_child_not_done() {
3482 let mut mock = MockBackend::new();
3483
3484 let parent = make_task("20", &["status:blocked"]);
3485 mock.blocked_tasks.push(parent.clone());
3486 mock.sub_issues.insert(
3487 "20".to_string(),
3488 vec![ExternalId("21".to_string()), ExternalId("22".to_string())],
3489 );
3490 mock.tasks_by_id
3492 .insert("21".to_string(), make_task("21", &["status:done"]));
3493 mock.tasks_by_id
3494 .insert("22".to_string(), make_task("22", &["status:in_progress"]));
3495
3496 let status_updates = Arc::clone(&mock.status_updates);
3497 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3498 let task_manager = make_task_manager(Arc::clone(&backend));
3499 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3500
3501 tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3502 .await
3503 .unwrap();
3504
3505 let updates = status_updates.lock().unwrap();
3506 assert!(
3507 updates.is_empty(),
3508 "should not unblock when a child is still running"
3509 );
3510 }
3511
3512 #[tokio::test]
3513 async fn unblock_parents_skips_task_with_no_children() {
3514 let mut mock = MockBackend::new();
3515
3516 let parent = make_task("30", &["status:blocked"]);
3518 mock.blocked_tasks.push(parent.clone());
3519 let status_updates = Arc::clone(&mock.status_updates);
3522 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3523 let task_manager = make_task_manager(Arc::clone(&backend));
3524 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3525
3526 tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3527 .await
3528 .unwrap();
3529
3530 let updates = status_updates.lock().unwrap();
3531 assert!(
3532 updates.is_empty(),
3533 "should not unblock tasks with no sub-issues"
3534 );
3535 }
3536
3537 #[tokio::test]
3538 async fn unblock_parents_skips_task_with_block_reason_in_store() {
3539 let mut mock = MockBackend::new();
3542
3543 let parent = make_task("60", &["status:blocked"]);
3545 mock.blocked_tasks.push(parent.clone());
3546 mock.sub_issues
3549 .insert("60".to_string(), vec![ExternalId("61".to_string())]);
3550 mock.tasks_by_id
3551 .insert("61".to_string(), make_task("61", &["status:done"]));
3552
3553 let status_updates = Arc::clone(&mock.status_updates);
3554 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3555 let task_manager = make_task_manager(Arc::clone(&backend));
3556
3557 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3559 let task_id = store
3560 .upsert_external(&crate::store::UpsertExternal {
3561 repo: "test/repo",
3562 ext_id: "60",
3563 title: "Task 60",
3564 body: "",
3565 author: "bot",
3566 url: "https://github.com/test/test/issues/60",
3567 labels: &[],
3568 origin: "github",
3569 })
3570 .await
3571 .unwrap();
3572 store
3573 .update_status(task_id, TaskStatus::Blocked)
3574 .await
3575 .unwrap();
3576 store
3577 .set_block_reason(task_id, Some("CI failure limit reached"))
3578 .await
3579 .unwrap();
3580
3581 tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3582 .await
3583 .unwrap();
3584
3585 let updates = status_updates.lock().unwrap();
3586 assert!(
3587 updates.is_empty(),
3588 "task with block_reason should be skipped — no API call, no unblock"
3589 );
3590 }
3591
3592 #[tokio::test]
3593 async fn unblock_parents_unblocks_internal_parent_from_store_children() {
3594 let mock = MockBackend::new();
3595 let get_sub_issues_calls = Arc::clone(&mock.get_sub_issues_calls);
3596 let status_updates = Arc::clone(&mock.status_updates);
3597 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3598 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3599 let task_manager = Arc::new(TaskManager::with_store(
3600 Arc::clone(&backend),
3601 Arc::clone(&store),
3602 "test/repo".to_string(),
3603 ));
3604
3605 let parent_id = store
3606 .create_internal("test/repo", "Parent", "", "job", "daily", None)
3607 .await
3608 .unwrap();
3609 store
3610 .update_status(parent_id, TaskStatus::Blocked)
3611 .await
3612 .unwrap();
3613 let child_id = store
3614 .create(&NewTask {
3615 external_id: Some("91".to_string()),
3616 repo: "test/repo".to_string(),
3617 origin: "github".to_string(),
3618 title: "Child".to_string(),
3619 body: String::new(),
3620 source: String::new(),
3621 source_id: String::new(),
3622 author: String::new(),
3623 url: String::new(),
3624 labels: vec![],
3625 parent_id: Some(parent_id),
3626 })
3627 .await
3628 .unwrap();
3629 store
3630 .update_status(child_id, TaskStatus::Done)
3631 .await
3632 .unwrap();
3633
3634 tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3635 .await
3636 .unwrap();
3637
3638 let parent = store.get(parent_id).await.unwrap();
3639 assert_eq!(parent.status, TaskStatus::New);
3640 assert!(
3641 status_updates.lock().unwrap().is_empty(),
3642 "internal parent should be updated in store without backend mirroring"
3643 );
3644 assert!(
3645 get_sub_issues_calls.lock().unwrap().is_empty(),
3646 "internal parent should not query backend sub-issues"
3647 );
3648 }
3649
3650 #[cfg(test)]
3654 async fn set_task_updated_at_past(store: &TaskStore, task_id: i64) {
3655 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
3656 .bind(task_id)
3657 .execute(store.pool())
3658 .await
3659 .unwrap();
3660 }
3661
3662 async fn set_in_review_entered_at_past(store: &TaskStore, task_id: i64) {
3664 sqlx::query(
3665 "UPDATE task_activity SET timestamp = '2020-01-01T00:00:00Z'
3666 WHERE task_id = ? AND to_status = 'in_review'",
3667 )
3668 .bind(task_id)
3669 .execute(store.pool())
3670 .await
3671 .unwrap();
3672 }
3673
3674 #[tokio::test]
3675 async fn recover_stuck_tasks_resets_external_in_review_to_needs_review() {
3676 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3677 let mock = MockBackend::new();
3678 let status_updates = Arc::clone(&mock.status_updates);
3679 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3680 let task_manager = Arc::new(TaskManager::with_store(
3681 Arc::clone(&backend),
3682 Arc::clone(&store),
3683 "owner/repo".to_string(),
3684 ));
3685 let tmux = Arc::new(TmuxManager::new());
3686 let config = EngineConfig {
3687 no_session_stuck_timeout: 600,
3688 stuck_timeout: 1800,
3689 ..EngineConfig::default()
3690 };
3691
3692 let id = store
3694 .upsert_external(&crate::store::UpsertExternal {
3695 repo: "owner/repo",
3696 ext_id: "99",
3697 title: "Stuck InReview",
3698 body: "",
3699 author: "",
3700 url: "",
3701 labels: &[],
3702 origin: "github",
3703 })
3704 .await
3705 .unwrap();
3706 store.update_status(id, TaskStatus::InReview).await.unwrap();
3707 set_in_review_entered_at_past(&store, id).await;
3708 set_task_updated_at_past(&store, id).await;
3709
3710 tick_recover_stuck_tasks(
3711 &backend,
3712 &tmux,
3713 "owner/repo",
3714 &task_manager,
3715 &config,
3716 &store,
3717 &HashMap::new(),
3718 &Arc::new(DashMap::new()),
3719 )
3720 .await
3721 .unwrap();
3722
3723 let updates = status_updates.lock().unwrap();
3724 assert_eq!(updates.len(), 1, "stuck InReview task should be recovered");
3725 assert_eq!(updates[0].0, "99");
3726 assert_eq!(updates[0].1, Status::NeedsReview);
3727 }
3728
3729 #[tokio::test]
3730 async fn recover_stuck_tasks_skips_recent_in_review() {
3731 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3732 let mock = MockBackend::new();
3733 let status_updates = Arc::clone(&mock.status_updates);
3734 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3735 let task_manager = Arc::new(TaskManager::with_store(
3736 Arc::clone(&backend),
3737 Arc::clone(&store),
3738 "owner/repo".to_string(),
3739 ));
3740 let tmux = Arc::new(TmuxManager::new());
3741 let config = EngineConfig {
3742 no_session_stuck_timeout: 600, stuck_timeout: 1800,
3744 ..EngineConfig::default()
3745 };
3746
3747 let id = store
3749 .upsert_external(&crate::store::UpsertExternal {
3750 repo: "owner/repo",
3751 ext_id: "100",
3752 title: "Recent InReview",
3753 body: "",
3754 author: "",
3755 url: "",
3756 labels: &[],
3757 origin: "github",
3758 })
3759 .await
3760 .unwrap();
3761 store.update_status(id, TaskStatus::InReview).await.unwrap();
3762 set_task_updated_at_past(&store, id).await;
3763
3764 tick_recover_stuck_tasks(
3765 &backend,
3766 &tmux,
3767 "owner/repo",
3768 &task_manager,
3769 &config,
3770 &store,
3771 &HashMap::new(),
3772 &Arc::new(DashMap::new()),
3773 )
3774 .await
3775 .unwrap();
3776
3777 let updates = status_updates.lock().unwrap();
3778 assert!(
3779 updates.is_empty(),
3780 "recent InReview task should not be recovered"
3781 );
3782 }
3783
3784 #[tokio::test]
3785 async fn recover_stuck_tasks_resets_internal_in_review_to_needs_review() {
3786 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3787 let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
3788 let task_manager = Arc::new(TaskManager::with_store(
3789 Arc::clone(&backend),
3790 Arc::clone(&store),
3791 "owner/repo".to_string(),
3792 ));
3793 let tmux = Arc::new(TmuxManager::new());
3794 let config = EngineConfig {
3795 no_session_stuck_timeout: 600,
3796 stuck_timeout: 1800,
3797 ..EngineConfig::default()
3798 };
3799
3800 let id = store
3802 .create_internal("owner/repo", "Internal InReview", "", "cron", "1", None)
3803 .await
3804 .unwrap();
3805 store.update_status(id, TaskStatus::InReview).await.unwrap();
3806 set_in_review_entered_at_past(&store, id).await;
3807 set_task_updated_at_past(&store, id).await;
3808
3809 tick_recover_stuck_tasks(
3810 &backend,
3811 &tmux,
3812 "owner/repo",
3813 &task_manager,
3814 &config,
3815 &store,
3816 &HashMap::new(),
3817 &Arc::new(DashMap::new()),
3818 )
3819 .await
3820 .unwrap();
3821
3822 let task = store.get(id).await.unwrap();
3823 assert_eq!(
3824 task.status,
3825 TaskStatus::NeedsReview,
3826 "stuck internal InReview task should be reset to NeedsReview"
3827 );
3828 }
3829
3830 #[tokio::test]
3831 async fn recover_stuck_tasks_ages_in_review_from_status_entry_not_updated_at() {
3832 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3833 let mock = MockBackend::new();
3834 let status_updates = Arc::clone(&mock.status_updates);
3835 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3836 let task_manager = Arc::new(TaskManager::with_store(
3837 Arc::clone(&backend),
3838 Arc::clone(&store),
3839 "owner/repo".to_string(),
3840 ));
3841 let tmux = Arc::new(TmuxManager::new());
3842 let config = EngineConfig {
3843 no_session_stuck_timeout: 600,
3844 stuck_timeout: 1800,
3845 ..EngineConfig::default()
3846 };
3847
3848 let id = store
3849 .upsert_external(&crate::store::UpsertExternal {
3850 repo: "owner/repo",
3851 ext_id: "102",
3852 title: "Fresh updated_at InReview",
3853 body: "",
3854 author: "",
3855 url: "",
3856 labels: &[],
3857 origin: "github",
3858 })
3859 .await
3860 .unwrap();
3861 store.update_status(id, TaskStatus::InReview).await.unwrap();
3862 set_in_review_entered_at_past(&store, id).await;
3863 store
3864 .set_fields(id, &[("pr_number", serde_json::json!(42))])
3865 .await
3866 .unwrap();
3867
3868 tick_recover_stuck_tasks(
3869 &backend,
3870 &tmux,
3871 "owner/repo",
3872 &task_manager,
3873 &config,
3874 &store,
3875 &HashMap::new(),
3876 &Arc::new(DashMap::new()),
3877 )
3878 .await
3879 .unwrap();
3880
3881 let updates = status_updates.lock().unwrap();
3882 assert_eq!(
3883 updates.len(),
3884 1,
3885 "in_review stuck recovery must age from the status entry even when updated_at is fresh"
3886 );
3887 assert_eq!(updates[0].0, "102");
3888 assert_eq!(updates[0].1, Status::NeedsReview);
3889 }
3890
3891 async fn orphaned_review_status_after_recovery(with_decision: bool) -> TaskStatus {
3894 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3895 let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
3896 let task_manager = Arc::new(TaskManager::with_store(
3897 Arc::clone(&backend),
3898 Arc::clone(&store),
3899 "owner/repo".to_string(),
3900 ));
3901 let tmux = Arc::new(TmuxManager::new());
3902
3903 let id = store
3904 .upsert_external(&crate::store::UpsertExternal {
3905 repo: "owner/repo",
3906 ext_id: "55",
3907 title: "Orphaned review",
3908 body: "",
3909 author: "",
3910 url: "",
3911 labels: &[],
3912 origin: "github",
3913 })
3914 .await
3915 .unwrap();
3916 store.update_status(id, TaskStatus::InReview).await.unwrap();
3917 set_in_review_entered_at_past(&store, id).await;
3919 if with_decision {
3920 store::store_log_activity(
3921 &Some(Arc::clone(&store)),
3922 "owner/repo",
3923 "55",
3924 "review_decision",
3925 None,
3926 None,
3927 None::<&str>,
3928 None::<&str>,
3929 Some(&serde_json::json!({"decision": "approve"})),
3930 )
3931 .await;
3932 }
3933
3934 tick_recover_stuck_tasks(
3935 &backend,
3936 &tmux,
3937 "owner/repo",
3938 &task_manager,
3939 &EngineConfig::default(),
3940 &store,
3941 &HashMap::new(),
3942 &Arc::new(DashMap::new()),
3943 )
3944 .await
3945 .unwrap();
3946
3947 store.get(id).await.unwrap().status
3948 }
3949
3950 #[tokio::test]
3951 async fn stale_in_review_recovery_resets_orphaned_task_without_session() {
3952 assert_eq!(
3953 orphaned_review_status_after_recovery(false).await,
3954 TaskStatus::NeedsReview
3955 );
3956 }
3957
3958 #[tokio::test]
3959 async fn stale_in_review_recovery_leaves_decided_task_waiting_on_merge() {
3960 assert_eq!(
3961 orphaned_review_status_after_recovery(true).await,
3962 TaskStatus::InReview
3963 );
3964 }
3965
3966 #[tokio::test]
3967 async fn recover_stuck_tasks_resets_review_agent_failures() {
3968 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3969 let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
3970 let task_manager = Arc::new(TaskManager::with_store(
3971 Arc::clone(&backend),
3972 Arc::clone(&store),
3973 "owner/repo".to_string(),
3974 ));
3975 let tmux = Arc::new(TmuxManager::new());
3976
3977 let id = store
3978 .upsert_external(&crate::store::UpsertExternal {
3979 repo: "owner/repo",
3980 ext_id: "104",
3981 title: "Orphaned review with failures",
3982 body: "",
3983 author: "",
3984 url: "",
3985 labels: &[],
3986 origin: "github",
3987 })
3988 .await
3989 .unwrap();
3990 store.update_status(id, TaskStatus::InReview).await.unwrap();
3991 store
3992 .set_fields(id, &[("review_agent_failures", serde_json::json!(2))])
3993 .await
3994 .unwrap();
3995 set_in_review_entered_at_past(&store, id).await;
3996
3997 tick_recover_stuck_tasks(
3998 &backend,
3999 &tmux,
4000 "owner/repo",
4001 &task_manager,
4002 &EngineConfig::default(),
4003 &store,
4004 &HashMap::new(),
4005 &Arc::new(DashMap::new()),
4006 )
4007 .await
4008 .unwrap();
4009
4010 let task = store.get(id).await.unwrap();
4011 assert_eq!(task.status, TaskStatus::NeedsReview);
4012 assert_eq!(
4013 task.review_agent_failures, 0,
4014 "a dead review session is an infrastructure event, not an agent failure"
4015 );
4016 }
4017
4018 #[tokio::test]
4019 async fn recover_stuck_tasks_reclaims_hung_in_review_session() {
4020 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4021 let mock = MockBackend::new();
4022 let status_updates = Arc::clone(&mock.status_updates);
4023 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4024 let task_manager = Arc::new(TaskManager::with_store(
4025 Arc::clone(&backend),
4026 Arc::clone(&store),
4027 "owner/repo".to_string(),
4028 ));
4029 let tmux = Arc::new(TmuxManager::new());
4030 let config = EngineConfig {
4031 no_session_stuck_timeout: 600,
4032 stuck_timeout: 1800,
4033 ..EngineConfig::default()
4034 };
4035
4036 let id = store
4037 .upsert_external(&crate::store::UpsertExternal {
4038 repo: "owner/repo",
4039 ext_id: "103",
4040 title: "Hung review session",
4041 body: "",
4042 author: "",
4043 url: "",
4044 labels: &[],
4045 origin: "github",
4046 })
4047 .await
4048 .unwrap();
4049 store.update_status(id, TaskStatus::InReview).await.unwrap();
4050 set_in_review_entered_at_past(&store, id).await;
4051 store
4052 .set_fields(id, &[("pr_number", serde_json::json!(43))])
4053 .await
4054 .unwrap();
4055
4056 let session_name = tmux.session_name("owner/repo", "103-review");
4057 let mut session_map = HashMap::new();
4058 session_map.insert(session_name, true);
4059
4060 tick_recover_stuck_tasks(
4061 &backend,
4062 &tmux,
4063 "owner/repo",
4064 &task_manager,
4065 &config,
4066 &store,
4067 &session_map,
4068 &Arc::new(DashMap::new()),
4069 )
4070 .await
4071 .unwrap();
4072
4073 let updates = status_updates.lock().unwrap();
4074 assert_eq!(
4075 updates.len(),
4076 1,
4077 "a live hung review session must be reclaimed once the in_review entry ages past stuck_timeout"
4078 );
4079 assert_eq!(updates[0].0, "103");
4080 assert_eq!(updates[0].1, Status::NeedsReview);
4081 }
4082
4083 #[tokio::test]
4084 async fn untracked_repo_recovery_resets_stale_in_progress_to_new() {
4085 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4086 let tmux = Arc::new(TmuxManager::new());
4087 let config = EngineConfig {
4088 no_session_stuck_timeout: 600,
4089 stuck_timeout: 1800,
4090 ..EngineConfig::default()
4091 };
4092
4093 let id = store
4095 .upsert_external(&crate::store::UpsertExternal {
4096 repo: "owner/other",
4097 ext_id: "777",
4098 title: "Cross-repo stuck task",
4099 body: "",
4100 author: "",
4101 url: "",
4102 labels: &[],
4103 origin: "github",
4104 })
4105 .await
4106 .unwrap();
4107 store
4108 .update_status(id, TaskStatus::InProgress)
4109 .await
4110 .unwrap();
4111 set_task_updated_at_past(&store, id).await;
4112
4113 let active = HashSet::from(["owner/repo".to_string()]);
4114 recover_untracked_repo_stuck_tasks(
4115 &tmux,
4116 &config,
4117 &store,
4118 &HashMap::new(),
4119 &Arc::new(DashMap::new()),
4120 &active,
4121 )
4122 .await;
4123
4124 let task = store.get(id).await.unwrap();
4125 assert_eq!(
4126 task.status,
4127 TaskStatus::New,
4128 "cross-repo stale in_progress task should be reset to New"
4129 );
4130 assert_eq!(task.route_attempts, 0, "route attempts should be cleared");
4131 assert!(task.agent.is_none(), "agent should be cleared");
4132 assert!(task.model.is_none(), "model should be cleared");
4133 }
4134
4135 #[tokio::test]
4136 async fn untracked_repo_recovery_leaves_active_repos_to_their_own_pass() {
4137 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4138 let tmux = Arc::new(TmuxManager::new());
4139 let config = EngineConfig {
4140 no_session_stuck_timeout: 600,
4141 stuck_timeout: 1800,
4142 ..EngineConfig::default()
4143 };
4144 let id = store
4145 .upsert_external(&crate::store::UpsertExternal {
4146 repo: "owner/other",
4147 ext_id: "778",
4148 title: "Stuck task in an active repo",
4149 body: "",
4150 author: "",
4151 url: "",
4152 labels: &[],
4153 origin: "github",
4154 })
4155 .await
4156 .unwrap();
4157 store
4158 .update_status(id, TaskStatus::InProgress)
4159 .await
4160 .unwrap();
4161 set_task_updated_at_past(&store, id).await;
4162
4163 let active = HashSet::from(["owner/repo".to_string(), "owner/other".to_string()]);
4165 recover_untracked_repo_stuck_tasks(
4166 &tmux,
4167 &config,
4168 &store,
4169 &HashMap::new(),
4170 &Arc::new(DashMap::new()),
4171 &active,
4172 )
4173 .await;
4174
4175 assert_eq!(store.get(id).await.unwrap().status, TaskStatus::InProgress);
4176 }
4177
4178 #[tokio::test]
4179 async fn recover_stuck_tasks_finalizes_open_task_runs_rows() {
4180 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4181 let mock = MockBackend::new();
4182 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4183 let task_manager = Arc::new(TaskManager::with_store(
4184 Arc::clone(&backend),
4185 Arc::clone(&store),
4186 "owner/repo".to_string(),
4187 ));
4188 let tmux = Arc::new(TmuxManager::new());
4189 let config = EngineConfig {
4190 no_session_stuck_timeout: 600,
4191 stuck_timeout: 1800,
4192 ..EngineConfig::default()
4193 };
4194
4195 let id = store
4196 .create_internal("owner/repo", "Stuck with open run", "", "cron", "1", None)
4197 .await
4198 .unwrap();
4199 store
4200 .set_fields(
4201 id,
4202 &[
4203 ("agent", serde_json::json!("claude")),
4204 ("model", serde_json::json!("sonnet")),
4205 ("route_attempts", serde_json::json!(1)),
4206 ],
4207 )
4208 .await
4209 .unwrap();
4210 store
4211 .update_status(id, TaskStatus::InProgress)
4212 .await
4213 .unwrap();
4214 set_task_updated_at_past(&store, id).await;
4215
4216 store
4218 .start_run(&crate::store::StartRun {
4219 task_id: id,
4220 attempt: 1,
4221 run_type: "agent",
4222 agent: "claude",
4223 model: "sonnet",
4224 command: "claude --model sonnet",
4225 prompt: "fix it",
4226 })
4227 .await
4228 .unwrap();
4229
4230 tick_recover_stuck_tasks(
4231 &backend,
4232 &tmux,
4233 "owner/repo",
4234 &task_manager,
4235 &config,
4236 &store,
4237 &HashMap::new(),
4238 &Arc::new(DashMap::new()),
4239 )
4240 .await
4241 .unwrap();
4242
4243 let task = store.get(id).await.unwrap();
4244 assert_eq!(task.status, TaskStatus::New);
4245
4246 let runs = store.get_runs(id).await.unwrap();
4247 assert_eq!(runs.len(), 1);
4248 assert_eq!(runs[0].outcome, "aborted");
4249 assert!(runs[0].completed_at.is_some());
4250 assert_eq!(
4251 runs[0].error,
4252 "stuck-task recovery: internal no session found"
4253 );
4254 }
4255
4256 #[tokio::test]
4273 async fn recover_stuck_tasks_skips_reclaim_while_dispatch_guard_held() {
4274 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4275 let mock = MockBackend::new();
4276 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4277 let task_manager = Arc::new(TaskManager::with_store(
4278 Arc::clone(&backend),
4279 Arc::clone(&store),
4280 "owner/repo".to_string(),
4281 ));
4282 let tmux = Arc::new(TmuxManager::new());
4283 let config = EngineConfig {
4284 no_session_stuck_timeout: 600,
4285 stuck_timeout: 1800,
4286 ..EngineConfig::default()
4287 };
4288
4289 let id = store
4290 .create_internal(
4291 "owner/repo",
4292 "Long-running daily job",
4293 "",
4294 "cron",
4295 "1",
4296 None,
4297 )
4298 .await
4299 .unwrap();
4300 store
4301 .set_fields(
4302 id,
4303 &[
4304 ("agent", serde_json::json!("claude")),
4305 ("model", serde_json::json!("sonnet")),
4306 ("route_attempts", serde_json::json!(1)),
4307 ],
4308 )
4309 .await
4310 .unwrap();
4311 store
4312 .update_status(id, TaskStatus::InProgress)
4313 .await
4314 .unwrap();
4315 set_task_updated_at_past(&store, id).await;
4318
4319 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4322 dispatching.insert(
4323 format!("owner/repo/internal:{id}"),
4324 format!("internal:{id}"),
4325 );
4326
4327 tick_recover_stuck_tasks(
4328 &backend,
4329 &tmux,
4330 "owner/repo",
4331 &task_manager,
4332 &config,
4333 &store,
4334 &HashMap::new(),
4335 &dispatching,
4336 )
4337 .await
4338 .unwrap();
4339
4340 let task = store.get(id).await.unwrap();
4341 assert_eq!(
4342 task.status,
4343 TaskStatus::InProgress,
4344 "task should not be reclaimed while its dispatch guard is still held"
4345 );
4346
4347 let runs = store.get_runs(id).await.unwrap();
4349 assert!(runs.is_empty());
4350
4351 dispatching.clear();
4354 tick_recover_stuck_tasks(
4355 &backend,
4356 &tmux,
4357 "owner/repo",
4358 &task_manager,
4359 &config,
4360 &store,
4361 &HashMap::new(),
4362 &dispatching,
4363 )
4364 .await
4365 .unwrap();
4366 let task = store.get(id).await.unwrap();
4367 assert_eq!(
4368 task.status,
4369 TaskStatus::New,
4370 "task should still be reclaimed once the dispatch guard is released"
4371 );
4372 }
4373
4374 #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
4390 async fn recover_stuck_tasks_never_reclaims_while_guard_held_under_real_concurrency() {
4391 const HAMMER_ITERATIONS: u32 = 300;
4392
4393 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4394 let mock = MockBackend::new();
4395 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4396 let task_manager = Arc::new(TaskManager::with_store(
4397 Arc::clone(&backend),
4398 Arc::clone(&store),
4399 "owner/repo".to_string(),
4400 ));
4401 let tmux = Arc::new(TmuxManager::new());
4402 let config = EngineConfig {
4403 no_session_stuck_timeout: 600,
4404 stuck_timeout: 1800,
4405 ..EngineConfig::default()
4406 };
4407
4408 let id = store
4409 .create_internal(
4410 "owner/repo",
4411 "Long-running daily job",
4412 "",
4413 "cron",
4414 "1",
4415 None,
4416 )
4417 .await
4418 .unwrap();
4419 store
4420 .set_fields(
4421 id,
4422 &[
4423 ("agent", serde_json::json!("claude")),
4424 ("model", serde_json::json!("sonnet")),
4425 ("route_attempts", serde_json::json!(1)),
4426 ],
4427 )
4428 .await
4429 .unwrap();
4430 store
4431 .update_status(id, TaskStatus::InProgress)
4432 .await
4433 .unwrap();
4434 set_task_updated_at_past(&store, id).await;
4435
4436 let dispatch_key = format!("owner/repo/internal:{id}");
4437 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4438
4439 let claim_task_id = format!("internal:{id}");
4446 {
4447 match dispatching.entry(dispatch_key.clone()) {
4448 Entry::Vacant(slot) => {
4449 slot.insert(claim_task_id);
4450 }
4451 Entry::Occupied(_) => panic!("dispatch key already claimed"),
4452 }
4453 }
4454 let guard = DispatchGuard::new(Arc::clone(&dispatching), dispatch_key.clone());
4455
4456 let (release_tx, release_rx) = oneshot::channel::<()>();
4467 let holder = tokio::spawn(async move {
4468 let _ = release_rx.await;
4469 drop(guard);
4470 });
4471
4472 for i in 0..HAMMER_ITERATIONS {
4480 tick_recover_stuck_tasks(
4481 &backend,
4482 &tmux,
4483 "owner/repo",
4484 &task_manager,
4485 &config,
4486 &store,
4487 &HashMap::new(),
4488 &dispatching,
4489 )
4490 .await
4491 .unwrap();
4492
4493 let task = store.get(id).await.unwrap();
4494 assert_eq!(
4495 task.status,
4496 TaskStatus::InProgress,
4497 "task must never be reclaimed while the dispatch guard is deterministically \
4498 still held, even under real cross-thread concurrency (dispatch_key={dispatch_key}, \
4499 iteration={i}, contains_key_now={}, map_len_now={})",
4500 dispatching.contains_key(&dispatch_key),
4501 dispatching.len(),
4502 );
4503 task::yield_now().await;
4504 }
4505
4506 let _ = release_tx.send(());
4508 holder.await.unwrap();
4509
4510 tick_recover_stuck_tasks(
4512 &backend,
4513 &tmux,
4514 "owner/repo",
4515 &task_manager,
4516 &config,
4517 &store,
4518 &HashMap::new(),
4519 &dispatching,
4520 )
4521 .await
4522 .unwrap();
4523 let task = store.get(id).await.unwrap();
4524 assert_eq!(
4525 task.status,
4526 TaskStatus::New,
4527 "task should be reclaimed once the dispatch guard is released"
4528 );
4529 }
4530
4531 #[tokio::test]
4532 async fn recover_stuck_tasks_skips_external_in_review_waiting_for_human_review() {
4533 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4534 let mock = MockBackend::new();
4535 let status_updates = Arc::clone(&mock.status_updates);
4536 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4537 let task_manager = Arc::new(TaskManager::with_store(
4538 Arc::clone(&backend),
4539 Arc::clone(&store),
4540 "owner/repo".to_string(),
4541 ));
4542 let tmux = Arc::new(TmuxManager::new());
4543 let config = EngineConfig {
4544 no_session_stuck_timeout: 600,
4545 stuck_timeout: 1800,
4546 ..EngineConfig::default()
4547 };
4548
4549 let id = store
4550 .upsert_external(&crate::store::UpsertExternal {
4551 repo: "owner/repo",
4552 ext_id: "101",
4553 title: "Waiting For Human Review",
4554 body: "",
4555 author: "",
4556 url: "",
4557 labels: &[],
4558 origin: "github",
4559 })
4560 .await
4561 .unwrap();
4562 store.update_status(id, TaskStatus::InReview).await.unwrap();
4563 set_task_updated_at_past(&store, id).await;
4564
4565 tick_recover_stuck_tasks(
4566 &backend,
4567 &tmux,
4568 "owner/repo",
4569 &task_manager,
4570 &config,
4571 &store,
4572 &HashMap::new(),
4573 &Arc::new(DashMap::new()),
4574 )
4575 .await
4576 .unwrap();
4577
4578 let updates = status_updates.lock().unwrap();
4579 assert!(
4580 updates.is_empty(),
4581 "InReview task waiting for human review should not be reset"
4582 );
4583 }
4584
4585 #[tokio::test]
4586 async fn unblock_parents_no_blocked_tasks() {
4587 let mock = MockBackend::new(); let status_updates = Arc::clone(&mock.status_updates);
4589 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4590 let task_manager = make_task_manager(Arc::clone(&backend));
4591 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4592
4593 tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
4594 .await
4595 .unwrap();
4596
4597 assert!(status_updates.lock().unwrap().is_empty());
4598 }
4599
4600 #[tokio::test]
4601 async fn unblock_parents_handles_multiple_blocked_tasks() {
4602 let mut mock = MockBackend::new();
4603
4604 let t40 = make_task("40", &["status:blocked"]);
4606 mock.blocked_tasks.push(t40);
4607 mock.sub_issues
4608 .insert("40".to_string(), vec![ExternalId("41".to_string())]);
4609 mock.tasks_by_id
4610 .insert("41".to_string(), make_task("41", &["status:done"]));
4611
4612 let t50 = make_task("50", &["status:blocked"]);
4614 mock.blocked_tasks.push(t50);
4615 mock.sub_issues
4616 .insert("50".to_string(), vec![ExternalId("51".to_string())]);
4617 mock.tasks_by_id
4618 .insert("51".to_string(), make_task("51", &["status:in_progress"]));
4619
4620 let status_updates = Arc::clone(&mock.status_updates);
4621 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4622 let task_manager = make_task_manager(Arc::clone(&backend));
4623 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4624
4625 tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
4626 .await
4627 .unwrap();
4628
4629 let updates = status_updates.lock().unwrap();
4630 assert_eq!(updates.len(), 1, "only one parent should be unblocked");
4631 assert_eq!(updates[0].0, "40", "task 40 should be unblocked");
4632 assert_eq!(updates[0].1, Status::New);
4633 }
4634
4635 #[tokio::test]
4638 async fn tick_all_is_a_no_op_without_projects() {
4639 let transport = Arc::new(Transport::new());
4640 let capture = Arc::new(CaptureService::new(Arc::clone(&transport)));
4641 let tmux = Arc::new(TmuxManager::new());
4642 let semaphore = Arc::new(Semaphore::new(1));
4643 let router = Arc::new(RwLock::new(cooled_review_router_for_test()));
4644 let (weight_tx, mut weight_rx) = mpsc::channel(8);
4645 let dispatching = Arc::new(DashMap::new());
4646 let auto_merge_in_flight = Arc::new(DashSet::new());
4647
4648 tick_all(
4649 &[],
4650 &tmux,
4651 &capture,
4652 &semaphore,
4653 &EngineConfig::default(),
4654 &router,
4655 &weight_tx,
4656 &dispatching,
4657 Some(&transport),
4658 &HashSet::new(),
4659 &auto_merge_in_flight,
4660 )
4661 .await;
4662
4663 assert!(dispatching.is_empty(), "no project, so no dispatch claimed");
4664 assert!(
4665 weight_rx.try_recv().is_err(),
4666 "no project, so no weight signal"
4667 );
4668 }
4669
4670 #[serial_test::serial(cooldown_state)]
4675 #[tokio::test]
4676 async fn tick_all_runs_the_global_needs_review_sweep_once_for_two_projects() {
4677 cooldown::reset_global_state().await;
4678
4679 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4680 let id = store
4681 .upsert_external(&UpsertExternal {
4682 repo: "owner/one",
4683 ext_id: "7",
4684 title: "Stale needs_review task",
4685 body: "",
4686 author: "user",
4687 url: "https://github.com/owner/one/issues/7",
4688 labels: &[],
4689 origin: "github",
4690 })
4691 .await
4692 .unwrap();
4693 store
4694 .update_status(id, TaskStatus::NeedsReview)
4695 .await
4696 .unwrap();
4697 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4698 .bind(id)
4699 .execute(store.pool())
4700 .await
4701 .unwrap();
4702
4703 let engines: Vec<ProjectEngine> = ["owner/one", "owner/two"]
4704 .iter()
4705 .map(|repo| {
4706 let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
4707 ProjectEngine {
4708 repo: repo.to_string(),
4709 project_dir: PathBuf::from("/nonexistent"),
4710 backend: Arc::clone(&backend),
4711 task_manager: Arc::new(TaskManager::with_store(
4712 backend,
4713 Arc::clone(&store),
4714 repo.to_string(),
4715 )),
4716 runner: Arc::new(TaskRunner::new(repo.to_string())),
4717 store: Arc::clone(&store),
4718 }
4719 })
4720 .collect();
4721
4722 let mut config = RouterConfig::default();
4724 config
4725 .model_map
4726 .entry("review".to_string())
4727 .or_default()
4728 .insert("claude".to_string(), vec!["sonnet".to_string()]);
4729 let router = Arc::new(RwLock::new(Router::new_for_test(
4730 config,
4731 vec!["claude".to_string()],
4732 )));
4733
4734 let transport = Arc::new(Transport::new());
4735 let capture = Arc::new(CaptureService::new(Arc::clone(&transport)));
4736 let (weight_tx, _weight_rx) = mpsc::channel(8);
4737
4738 tick_all(
4739 &engines,
4740 &Arc::new(TmuxManager::new()),
4741 &capture,
4742 &Arc::new(Semaphore::new(1)),
4743 &EngineConfig::default(),
4744 &router,
4745 &weight_tx,
4746 &Arc::new(DashMap::new()),
4747 Some(&transport),
4748 &engines.iter().map(|e| e.repo.clone()).collect(),
4749 &Arc::new(DashSet::new()),
4750 )
4751 .await;
4752
4753 assert_eq!(
4754 store.get(id).await.unwrap().needs_review_refires,
4755 1,
4756 "two projects must share one pass of the global NeedsReview sweep"
4757 );
4758 }
4759
4760 #[tokio::test]
4763 async fn tick_all_does_not_kill_sessions_it_did_not_create() {
4764 let tmux = Arc::new(TmuxManager::new());
4765 let Ok(session_name) = tmux
4766 .create_session("other/foreign", "3889", "/tmp", "sleep 30", &[])
4767 .await
4768 else {
4769 eprintln!("Skipping test: tmux not available or failed to create test session");
4770 return;
4771 };
4772 let _guard = SessionGuard(session_name.clone());
4773
4774 time::sleep(Duration::from_millis(1100)).await;
4777
4778 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4779 let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
4780 let engines = vec![ProjectEngine {
4781 repo: "owner/one".to_string(),
4782 project_dir: PathBuf::from("/nonexistent"),
4783 backend: Arc::clone(&backend),
4784 task_manager: Arc::new(TaskManager::with_store(
4785 backend,
4786 Arc::clone(&store),
4787 "owner/one".to_string(),
4788 )),
4789 runner: Arc::new(TaskRunner::new("owner/one".to_string())),
4790 store,
4791 }];
4792
4793 let transport = Arc::new(Transport::new());
4794 let capture = Arc::new(CaptureService::new(Arc::clone(&transport)));
4795 let (weight_tx, _weight_rx) = mpsc::channel(8);
4796 let router = Arc::new(RwLock::new(cooled_review_router_for_test()));
4797
4798 tick_all(
4799 &engines,
4800 &tmux,
4801 &capture,
4802 &Arc::new(Semaphore::new(1)),
4803 &EngineConfig::default(),
4804 &router,
4805 &weight_tx,
4806 &Arc::new(DashMap::new()),
4807 Some(&transport),
4808 &engines.iter().map(|e| e.repo.clone()).collect(),
4809 &Arc::new(DashSet::new()),
4810 )
4811 .await;
4812
4813 assert!(
4814 tmux.session_exists(&session_name).await,
4815 "tick_all must not kill a tmux session it did not start"
4816 );
4817 }
4818
4819 #[tokio::test]
4820 async fn detect_silent_agents_skips_backend_for_internal_tasks() {
4821 let transport = Arc::new(Transport::new());
4822 let capture = Arc::new(CaptureService::new(transport));
4823 let mock = Arc::new(MockBackend::new());
4824 let backend: Arc<dyn ExternalBackend> = Arc::<MockBackend>::clone(&mock);
4825 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4826 let task_manager = Arc::new(TaskManager::with_store(
4827 Arc::clone(&backend),
4828 Arc::clone(&store),
4829 "owner/repo".to_string(),
4830 ));
4831 let tmux = Arc::new(TmuxManager::new());
4832 let config = EngineConfig {
4833 silence_grace_period: 0,
4834 ..EngineConfig::default()
4835 };
4836
4837 let internal_id = store
4838 .create_internal("owner/repo", "Silent internal", "", "cron", "1", None)
4839 .await
4840 .unwrap();
4841 store
4842 .update_status(internal_id, TaskStatus::InProgress)
4843 .await
4844 .unwrap();
4845
4846 let task_id = format!("internal:{internal_id}");
4847 capture
4848 .register_session("owner/repo", &task_id, "orch-test-internal")
4849 .await;
4850 time::sleep(Duration::from_millis(1100)).await;
4851
4852 tick_detect_silent_agents(
4853 &tmux,
4854 "owner/repo",
4855 &capture,
4856 &backend,
4857 &task_manager,
4858 &config,
4859 &store,
4860 )
4861 .await
4862 .unwrap();
4863
4864 assert!(
4865 mock.removed_labels.lock().unwrap().is_empty(),
4866 "internal tasks should skip backend label removal"
4867 );
4868 assert!(
4869 mock.posted_comments.lock().unwrap().is_empty(),
4870 "internal tasks should skip backend comments"
4871 );
4872
4873 let task = store.get(internal_id).await.unwrap();
4874 assert!(
4878 matches!(task.status, TaskStatus::Routed | TaskStatus::New),
4879 "internal task should be routed to fallback or reset to new, got {:?}",
4880 task.status
4881 );
4882 }
4883
4884 #[tokio::test]
4885 async fn detect_silent_agents_finalizes_open_task_runs_rows() {
4886 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4887 let mock = MockBackend::new();
4888 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4889 let task_manager = Arc::new(TaskManager::with_store(
4890 Arc::clone(&backend),
4891 Arc::clone(&store),
4892 "owner/repo".to_string(),
4893 ));
4894 let transport = Arc::new(Transport::new());
4895 let capture = Arc::new(CaptureService::new(transport));
4896 let tmux = Arc::new(TmuxManager::new());
4897 let config = EngineConfig {
4898 silence_grace_period: 60,
4899 silence_cooldown: 300,
4900 ..EngineConfig::default()
4901 };
4902
4903 let internal_id = store
4904 .create_internal("owner/repo", "Silent with open run", "", "cron", "1", None)
4905 .await
4906 .unwrap();
4907 store
4908 .set_fields(
4909 internal_id,
4910 &[
4911 ("agent", serde_json::json!("claude")),
4912 ("model", serde_json::json!("sonnet")),
4913 ],
4914 )
4915 .await
4916 .unwrap();
4917 store
4918 .update_status(internal_id, TaskStatus::InProgress)
4919 .await
4920 .unwrap();
4921
4922 let task_id = format!("internal:{internal_id}");
4923 capture
4924 .register_session("owner/repo", &task_id, "orch-test-silent-run")
4925 .await;
4926 capture
4927 .set_buffer_state_for_test(
4928 "owner/repo",
4929 &task_id,
4930 false,
4931 false,
4932 chrono::Utc::now() - chrono::Duration::seconds(120),
4933 )
4934 .await;
4935
4936 store
4938 .start_run(&crate::store::StartRun {
4939 task_id: internal_id,
4940 attempt: 1,
4941 run_type: "agent",
4942 agent: "claude",
4943 model: "sonnet",
4944 command: "claude --model sonnet",
4945 prompt: "fix it",
4946 })
4947 .await
4948 .unwrap();
4949
4950 tick_detect_silent_agents(
4951 &tmux,
4952 "owner/repo",
4953 &capture,
4954 &backend,
4955 &task_manager,
4956 &config,
4957 &store,
4958 )
4959 .await
4960 .unwrap();
4961
4962 let runs = store.get_runs(internal_id).await.unwrap();
4963 assert_eq!(runs.len(), 1);
4964 assert_eq!(runs[0].outcome, "aborted");
4965 assert!(runs[0].completed_at.is_some());
4966 assert_eq!(runs[0].error, "silence detection: session killed");
4967 }
4968
4969 #[tokio::test]
4980 async fn phase1_session_completion_touch_prevents_phase2_reclaim() {
4981 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4982 let mock = MockBackend::new();
4983 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4984 let task_manager = Arc::new(TaskManager::with_store(
4985 Arc::clone(&backend),
4986 Arc::clone(&store),
4987 "owner/repo".to_string(),
4988 ));
4989 let tmux = Arc::new(TmuxManager::new());
4990 let config = EngineConfig {
4991 no_session_stuck_timeout: 600,
4992 stuck_timeout: 1800,
4993 ..EngineConfig::default()
4994 };
4995
4996 let id = store
5000 .create_internal("owner/repo", "Completed task", "", "cron", "1", None)
5001 .await
5002 .unwrap();
5003 store
5004 .update_status(id, TaskStatus::InProgress)
5005 .await
5006 .unwrap();
5007 set_task_updated_at_past(&store, id).await;
5008
5009 let task_id = format!("internal:{id}");
5012 store_touch_updated_at(&Some(Arc::clone(&store)), "owner/repo", &task_id).await;
5013
5014 tick_recover_stuck_tasks(
5016 &backend,
5017 &tmux,
5018 "owner/repo",
5019 &task_manager,
5020 &config,
5021 &store,
5022 &HashMap::new(),
5023 &Arc::new(DashMap::new()),
5024 )
5025 .await
5026 .unwrap();
5027
5028 let task = store.get(id).await.unwrap();
5029 assert_eq!(
5030 task.status,
5031 TaskStatus::InProgress,
5032 "Phase 2 must not reclaim a task whose updated_at was just touched by Phase 1"
5033 );
5034 }
5035
5036 #[tokio::test]
5037 async fn stuck_task_timing_treats_dead_session_as_missing() {
5038 let tmux = Arc::new(TmuxManager::new());
5039 let repo = "owner/repo";
5040 let task_id = "internal:33498";
5041 let session_name = tmux.session_name(repo, task_id);
5042
5043 let create_result = tmux
5044 .command()
5045 .args(["new-session", "-d", "-s", &session_name, "-c", "/tmp"])
5046 .output()
5047 .await;
5048
5049 match create_result {
5050 Ok(o) if o.status.success() => {}
5051 _ => {
5052 eprintln!("Skipping test: tmux not available or failed to create test session");
5053 return;
5054 }
5055 }
5056
5057 let _guard = SessionGuard(session_name.clone());
5058
5059 let set_option_result = tmux
5060 .command()
5061 .args(["set-option", "-t", &session_name, "remain-on-exit", "on"])
5062 .output()
5063 .await;
5064 if !matches!(set_option_result, Ok(ref o) if o.status.success()) {
5065 eprintln!("Skipping test: unable to set tmux remain-on-exit option");
5066 return;
5067 }
5068
5069 let send_exit_result = tmux
5070 .command()
5071 .args(["send-keys", "-t", &session_name, "exit", "Enter"])
5072 .output()
5073 .await;
5074 if !matches!(send_exit_result, Ok(ref o) if o.status.success()) {
5075 eprintln!("Skipping test: unable to exit tmux pane");
5076 return;
5077 }
5078
5079 for _ in 0..20 {
5081 time::sleep(Duration::from_millis(100)).await;
5082 if !tmux.session_is_running(&session_name).await {
5083 break;
5084 }
5085 }
5086
5087 let mut session_map = tmux.batch_session_active().await;
5092 for _ in 0..20 {
5093 if !session_map.get(&session_name).copied().unwrap_or(false) {
5094 break;
5095 }
5096 time::sleep(Duration::from_millis(100)).await;
5097 session_map = tmux.batch_session_active().await;
5098 }
5099 if session_map.get(&session_name).copied().unwrap_or(false) {
5100 eprintln!(
5101 "Skipping test: session still alive in batch_session_active after retries (CI tmux lag)"
5102 );
5103 return;
5104 }
5105
5106 let config = EngineConfig {
5107 no_session_stuck_timeout: 600,
5108 stuck_timeout: 1800,
5109 ..EngineConfig::default()
5110 };
5111 let updated_at = (chrono::Utc::now()
5112 - chrono::Duration::seconds(
5113 i64::try_from(config.no_session_stuck_timeout).expect("timeout fits in i64") + 5,
5114 ))
5115 .to_rfc3339();
5116
5117 let timing = stuck_task_timing_from_map(
5118 &tmux,
5119 repo,
5120 task_id,
5121 &updated_at,
5122 &config,
5123 "parse failure",
5124 &session_map,
5125 )
5126 .expect("dead session should be treated as missing");
5127
5128 assert!(!timing.has_session);
5129 assert_eq!(timing.threshold, config.no_session_stuck_timeout);
5130 }
5131
5132 #[tokio::test]
5133 #[serial_test::serial(suspend_state)]
5134 async fn stuck_task_timing_discounts_host_suspend_gap() {
5135 super::super::suspend::clear_for_test();
5136
5137 let tmux = Arc::new(TmuxManager::new());
5138 let repo = "owner/repo";
5139 let task_id = "internal:99001";
5140 let session_map = HashMap::new();
5141
5142 let config = EngineConfig {
5143 no_session_stuck_timeout: 1800,
5144 stuck_timeout: 1800,
5145 ..EngineConfig::default()
5146 };
5147
5148 let updated_at = (chrono::Utc::now() - chrono::Duration::minutes(33)).to_rfc3339();
5152 let suspend_gap_detected_at = chrono::Utc::now() - chrono::Duration::minutes(1);
5153 super::super::suspend::inject_gap_for_test(
5154 suspend_gap_detected_at,
5155 chrono::Duration::minutes(30),
5156 );
5157
5158 let timing = stuck_task_timing_from_map(
5159 &tmux,
5160 repo,
5161 task_id,
5162 &updated_at,
5163 &config,
5164 "parse failure",
5165 &session_map,
5166 );
5167
5168 assert!(
5169 timing.is_none(),
5170 "task age should be discounted below the stuck threshold once suspend time is subtracted, got {timing:?}"
5171 );
5172
5173 super::super::suspend::clear_for_test();
5174 }
5175
5176 #[tokio::test]
5177 async fn recover_stuck_tasks_reclaims_routed_task_blocked_by_stale_session() {
5178 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5179 let mock = MockBackend::new();
5180 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
5181 let task_manager = Arc::new(TaskManager::with_store(
5182 Arc::clone(&backend),
5183 Arc::clone(&store),
5184 "owner/repo".to_string(),
5185 ));
5186 let tmux = Arc::new(TmuxManager::new());
5187 let config = EngineConfig {
5188 no_session_stuck_timeout: 600,
5189 stuck_timeout: 1,
5190 ..EngineConfig::default()
5191 };
5192
5193 let id = store
5194 .create_internal("owner/repo", "Blocked routed task", "", "cron", "1", None)
5195 .await
5196 .unwrap();
5197 store.update_status(id, TaskStatus::Routed).await.unwrap();
5198 set_task_updated_at_past(&store, id).await;
5199
5200 let task_id = format!("internal:{id}");
5201 let session_name = tmux.session_name("owner/repo", &task_id);
5202 let create_result = tmux
5203 .command()
5204 .args([
5205 "new-session",
5206 "-d",
5207 "-s",
5208 &session_name,
5209 "-c",
5210 "/tmp",
5211 "sleep 60",
5212 ])
5213 .output()
5214 .await;
5215
5216 match create_result {
5217 Ok(o) if o.status.success() => {}
5218 _ => {
5219 eprintln!("Skipping test: tmux not available or failed to create test session");
5220 return;
5221 }
5222 }
5223
5224 let session_map = tmux.batch_session_active().await;
5225 assert_eq!(session_map.get(&session_name).copied(), Some(true));
5226
5227 tick_recover_stuck_tasks(
5228 &backend,
5229 &tmux,
5230 "owner/repo",
5231 &task_manager,
5232 &config,
5233 &store,
5234 &session_map,
5235 &Arc::new(DashMap::new()),
5236 )
5237 .await
5238 .unwrap();
5239
5240 let task = store.get(id).await.unwrap();
5241 assert_eq!(
5242 task.status,
5243 TaskStatus::New,
5244 "routed task blocked by a stale session should be reset for re-routing"
5245 );
5246 assert!(
5247 !tmux.session_exists(&session_name).await,
5248 "recovery should kill the stale tmux session so dispatch can proceed"
5249 );
5250 }
5251
5252 async fn live_complex_run(
5255 store: &TaskStore,
5256 tmux: &TmuxManager,
5257 minutes: i64,
5258 updated_minutes_ago: i64,
5259 ) -> Option<(i64, String)> {
5260 let id = store
5261 .create_internal("owner/repo", "Long complex run", "", "cron", "1", None)
5262 .await
5263 .unwrap();
5264 store
5265 .set_fields(id, &[("complexity", serde_json::json!("complex"))])
5266 .await
5267 .unwrap();
5268 store
5269 .update_status(id, TaskStatus::InProgress)
5270 .await
5271 .unwrap();
5272 let ago = |m: i64| {
5273 (chrono::Utc::now() - chrono::Duration::minutes(m))
5274 .format("%Y-%m-%dT%H:%M:%SZ")
5275 .to_string()
5276 };
5277 let entered = ago(minutes);
5278 sqlx::query(
5279 "UPDATE task_activity SET timestamp = ? WHERE task_id = ? AND to_status = 'in_progress'",
5280 )
5281 .bind(&entered)
5282 .bind(id)
5283 .execute(store.pool())
5284 .await
5285 .unwrap();
5286 sqlx::query("UPDATE tasks SET updated_at = ? WHERE id = ?")
5287 .bind(ago(updated_minutes_ago))
5288 .bind(id)
5289 .execute(store.pool())
5290 .await
5291 .unwrap();
5292
5293 let session_name = tmux.session_name("owner/repo", &format!("internal:{id}"));
5294 let created = tmux
5295 .command()
5296 .args([
5297 "new-session",
5298 "-d",
5299 "-s",
5300 &session_name,
5301 "-c",
5302 "/tmp",
5303 "sleep 60",
5304 ])
5305 .output()
5306 .await;
5307 match created {
5308 Ok(o) if o.status.success() => Some((id, session_name)),
5309 _ => None,
5310 }
5311 }
5312
5313 fn long_run_config() -> EngineConfig {
5314 EngineConfig {
5315 stuck_timeout: 1800,
5316 run_timeouts: RunTimeouts {
5317 base: 1800,
5318 medium: 2700,
5319 complex: 3600,
5320 },
5321 ..EngineConfig::default()
5322 }
5323 }
5324
5325 #[test]
5326 fn tick_router_loads_the_config_once_for_many_live_runs() {
5327 let cell = OnceLock::new();
5328 let loads = Cell::new(0);
5329 for _ in 0..4 {
5330 tick_router(&cell, || {
5331 loads.set(loads.get() + 1);
5332 RouterConfig::default()
5333 });
5334 }
5335 assert_eq!(loads.get(), 1);
5336 }
5337
5338 #[tokio::test]
5339 async fn run_tier_follows_the_tier_of_the_stored_model() {
5340 let mut router = RouterConfig::default();
5341 router.model_map.clear();
5342 for (tier, model) in [("simple", "rt-s"), ("complex", "rt-c")] {
5343 router
5344 .model_map
5345 .entry(tier.to_string())
5346 .or_default()
5347 .insert("rtagent".to_string(), vec![model.to_string()]);
5348 }
5349 let store = TaskStore::open_memory().await.unwrap();
5350 let id = store
5351 .create_internal(
5352 "owner/repo",
5353 "Simple on a complex model",
5354 "",
5355 "cron",
5356 "1",
5357 None,
5358 )
5359 .await
5360 .unwrap();
5361 store
5362 .set_fields(
5363 id,
5364 &[
5365 ("complexity", serde_json::json!("simple")),
5366 ("agent", serde_json::json!("rtagent")),
5367 ("model", serde_json::json!("rt-c")),
5368 ],
5369 )
5370 .await
5371 .unwrap();
5372 let task = store.get(id).await.unwrap();
5373 assert_eq!(run_tier(&router, &task), "complex");
5374
5375 store
5376 .set_fields(id, &[("model", serde_json::json!("rt-s"))])
5377 .await
5378 .unwrap();
5379 let task = store.get(id).await.unwrap();
5380 assert_eq!(run_tier(&router, &task), "simple");
5381 }
5382
5383 #[tokio::test]
5384 async fn recover_stuck_tasks_keeps_a_live_complex_run_inside_its_timeout() {
5385 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5386 let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
5387 let task_manager = Arc::new(TaskManager::with_store(
5388 Arc::clone(&backend),
5389 Arc::clone(&store),
5390 "owner/repo".to_string(),
5391 ));
5392 let tmux = Arc::new(TmuxManager::new());
5393 let Some((id, session_name)) = live_complex_run(&store, &tmux, 31, 31).await else {
5394 eprintln!("Skipping test: tmux not available or failed to create test session");
5395 return;
5396 };
5397 let session_map = tmux.batch_session_active().await;
5398
5399 tick_recover_stuck_tasks(
5400 &backend,
5401 &tmux,
5402 "owner/repo",
5403 &task_manager,
5404 &long_run_config(),
5405 &store,
5406 &session_map,
5407 &Arc::new(DashMap::new()),
5408 )
5409 .await
5410 .unwrap();
5411
5412 let alive = tmux.session_exists(&session_name).await;
5413 let _ = tmux.kill_session(&session_name).await;
5414 assert_eq!(
5415 store.get(id).await.unwrap().status,
5416 TaskStatus::InProgress,
5417 "a complex run at 31 minutes is inside its 3600s timeout"
5418 );
5419 assert!(alive, "recovery must not touch a run inside its timeout");
5420 }
5421
5422 #[tokio::test]
5423 async fn recover_stuck_tasks_kills_a_live_complex_run_past_its_timeout() {
5424 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5425 let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
5426 let task_manager = Arc::new(TaskManager::with_store(
5427 Arc::clone(&backend),
5428 Arc::clone(&store),
5429 "owner/repo".to_string(),
5430 ));
5431 let tmux = Arc::new(TmuxManager::new());
5432 let Some((id, session_name)) = live_complex_run(&store, &tmux, 66, 0).await else {
5433 eprintln!("Skipping test: tmux not available or failed to create test session");
5434 return;
5435 };
5436 let session_map = tmux.batch_session_active().await;
5437
5438 tick_recover_stuck_tasks(
5439 &backend,
5440 &tmux,
5441 "owner/repo",
5442 &task_manager,
5443 &long_run_config(),
5444 &store,
5445 &session_map,
5446 &Arc::new(DashMap::new()),
5447 )
5448 .await
5449 .unwrap();
5450
5451 let alive = tmux.session_exists(&session_name).await;
5452 let _ = tmux.kill_session(&session_name).await;
5453 assert_eq!(
5454 store.get(id).await.unwrap().status,
5455 TaskStatus::New,
5456 "a complex run past its timeout plus grace is stuck"
5457 );
5458 assert!(
5459 !alive,
5460 "recovery must kill the session it reports as killed"
5461 );
5462 }
5463
5464 #[tokio::test]
5478 async fn dispatch_does_not_deadlock_under_write_lock() {
5479 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5480 let mock = MockBackend::new();
5481 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
5482 let task_manager = Arc::new(TaskManager::with_store(
5483 Arc::clone(&backend),
5484 Arc::clone(&store),
5485 "owner/repo".to_string(),
5486 ));
5487 let tmux = Arc::new(TmuxManager::new());
5488 let semaphore = Arc::new(Semaphore::new(4));
5489 let (weight_tx, _weight_rx) = mpsc::channel(16);
5490 let dispatching = Arc::new(DashMap::new());
5491 let transport = Arc::new(Transport::new());
5492 let capture = Arc::new(CaptureService::new(transport));
5493
5494 let router_config = RouterConfig::default();
5495 let router = Router::new_for_test(router_config, vec!["claude".to_string()]);
5496 let router_arc = Arc::new(RwLock::new(router));
5497
5498 let write_guard = router_arc.write().await;
5502 let dispatch_mode = dispatch_mode_from_router(&write_guard);
5503
5504 let result = time::timeout(
5505 Duration::from_secs(2),
5506 tick_dispatch_tasks(
5507 &backend,
5508 &tmux,
5509 "owner/repo",
5510 &Arc::new(TaskRunner::new("owner/repo".to_string())),
5511 &capture,
5512 &semaphore,
5513 &task_manager,
5514 &weight_tx,
5515 dispatch_mode,
5516 &dispatching,
5517 &store,
5518 &HashMap::new(),
5519 ),
5520 )
5521 .await;
5522
5523 assert!(
5524 result.is_ok(),
5525 "tick_dispatch_tasks deadlocked! It tried to acquire a read lock \
5526 on router_arc while a write lock was already held (issue #1361)"
5527 );
5528 }
5529
5530 #[tokio::test]
5538 async fn dispatch_skips_task_whose_routed_model_entered_cooldown() {
5539 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5540 let mock = MockBackend::new();
5541 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
5542 let task_manager = Arc::new(TaskManager::with_store(
5543 Arc::clone(&backend),
5544 Arc::clone(&store),
5545 "owner/repo".to_string(),
5546 ));
5547 let tmux = Arc::new(TmuxManager::new());
5548 let runner = Arc::new(TaskRunner::new("owner/repo".to_string()));
5549 let semaphore = Arc::new(Semaphore::new(4));
5550 let (weight_tx, _weight_rx) = mpsc::channel(16);
5551 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
5552 let transport = Arc::new(Transport::new());
5553 let capture = Arc::new(CaptureService::new(transport));
5554
5555 let id = store
5556 .upsert_external(&UpsertExternal {
5557 repo: "owner/repo",
5558 ext_id: "999",
5559 title: "Stale routed task",
5560 body: "",
5561 author: "user",
5562 url: "https://github.com/owner/repo/issues/999",
5563 labels: &[],
5564 origin: "github",
5565 })
5566 .await
5567 .unwrap();
5568 store
5569 .store_route(&StoreRoute {
5570 id,
5571 agent: "minimax",
5572 model: Some("opus"),
5573 complexity: "medium",
5574 estimate: 0,
5575 reason: "test",
5576 profile: "{}",
5577 })
5578 .await
5579 .unwrap();
5580 store.update_status(id, DbStatus::Routed).await.unwrap();
5581
5582 cooldown::set_model_cooldown("minimax", "opus", 300, "rate_limit_usage_window").await;
5585
5586 let dispatch_mode = DispatchMode {
5587 is_degraded: false,
5588 healthy_agents: 1,
5589 threshold: 1,
5590 };
5591
5592 tick_dispatch_tasks(
5593 &backend,
5594 &tmux,
5595 "owner/repo",
5596 &runner,
5597 &capture,
5598 &semaphore,
5599 &task_manager,
5600 &weight_tx,
5601 dispatch_mode,
5602 &dispatching,
5603 &store,
5604 &HashMap::new(),
5605 )
5606 .await
5607 .unwrap();
5608
5609 let task = store.get(id).await.unwrap();
5610 assert_eq!(
5611 task.status,
5612 DbStatus::Routed,
5613 "task whose routed model is now cooled down must stay Routed, not be dispatched"
5614 );
5615 assert!(
5616 dispatching.is_empty(),
5617 "dispatch guard must be released when skipping a stale-cooldown task"
5618 );
5619 }
5620
5621 #[tokio::test]
5622 async fn dispatch_skips_a_task_with_an_open_dependency() {
5623 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5624 let mock = MockBackend::new();
5625 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
5626 let task_manager = Arc::new(TaskManager::with_store(
5627 Arc::clone(&backend),
5628 Arc::clone(&store),
5629 "owner/repo".to_string(),
5630 ));
5631 let tmux = Arc::new(TmuxManager::new());
5632 let runner = Arc::new(TaskRunner::new("owner/repo".to_string()));
5633 let semaphore = Arc::new(Semaphore::new(4));
5634 let (weight_tx, _weight_rx) = mpsc::channel(16);
5635 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
5636 let transport = Arc::new(Transport::new());
5637 let capture = Arc::new(CaptureService::new(transport));
5638
5639 let id = store
5640 .upsert_external(&UpsertExternal {
5641 repo: "owner/repo",
5642 ext_id: "42",
5643 title: "Waiting routed task",
5644 body: "",
5645 author: "user",
5646 url: "https://github.com/owner/repo/issues/42",
5647 labels: &[],
5648 origin: "github",
5649 })
5650 .await
5651 .unwrap();
5652 store
5653 .replace_dependencies(
5654 id,
5655 &[NewDependency {
5656 repo: "owner/repo".into(),
5657 number: 7,
5658 source: DepSource::Github,
5659 }],
5660 )
5661 .await
5662 .unwrap();
5663 store
5664 .store_route(&StoreRoute {
5665 id,
5666 agent: "claude",
5667 model: Some("haiku"),
5668 complexity: "simple",
5669 estimate: 0,
5670 reason: "test",
5671 profile: "{}",
5672 })
5673 .await
5674 .unwrap();
5675 store.update_status(id, DbStatus::Routed).await.unwrap();
5676
5677 let dispatch_mode = DispatchMode {
5678 is_degraded: false,
5679 healthy_agents: 1,
5680 threshold: 1,
5681 };
5682 tick_dispatch_tasks(
5683 &backend,
5684 &tmux,
5685 "owner/repo",
5686 &runner,
5687 &capture,
5688 &semaphore,
5689 &task_manager,
5690 &weight_tx,
5691 dispatch_mode,
5692 &dispatching,
5693 &store,
5694 &HashMap::new(),
5695 )
5696 .await
5697 .unwrap();
5698
5699 let task = store.get(id).await.unwrap();
5700 assert_eq!(task.status, DbStatus::Waiting);
5701 assert!(dispatching.is_empty());
5702 }
5703
5704 #[tokio::test]
5717 async fn completed_session_does_not_cause_stuck_reclaim_race() {
5718 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5719 let mock = MockBackend::new();
5720 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
5721 let task_manager = Arc::new(TaskManager::with_store(
5722 Arc::clone(&backend),
5723 Arc::clone(&store),
5724 "owner/repo".to_string(),
5725 ));
5726 let tmux = Arc::new(TmuxManager::new());
5727 let transport = Arc::new(Transport::new());
5728 let capture = Arc::new(CaptureService::new(transport));
5729
5730 let id = store
5732 .upsert_external(&crate::store::UpsertExternal {
5733 repo: "owner/repo",
5734 ext_id: "200",
5735 title: "Race Task",
5736 body: "",
5737 author: "",
5738 url: "",
5739 labels: &[],
5740 origin: "github",
5741 })
5742 .await
5743 .unwrap();
5744 store
5745 .update_status(id, TaskStatus::InProgress)
5746 .await
5747 .unwrap();
5748
5749 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
5751 .bind(id)
5752 .execute(store.pool())
5753 .await
5754 .unwrap();
5755
5756 let task_id = "200";
5758 let session_name = tmux.session_name("owner/repo", task_id);
5759 let create_result = tmux
5760 .command()
5761 .args(["new-session", "-d", "-s", &session_name, "-c", "/tmp"])
5762 .output()
5763 .await;
5764 match create_result {
5765 Ok(o) if o.status.success() => {}
5766 _ => {
5767 eprintln!("Skipping test: tmux not available or failed to create test session");
5768 return;
5769 }
5770 }
5771 let _guard = SessionGuard(session_name.clone());
5772
5773 let _ = tmux
5775 .command()
5776 .args(["set-option", "-t", &session_name, "remain-on-exit", "on"])
5777 .output()
5778 .await;
5779 let _ = tmux
5780 .command()
5781 .args(["send-keys", "-t", &session_name, "exit", "Enter"])
5782 .output()
5783 .await;
5784
5785 for _ in 0..20 {
5787 time::sleep(Duration::from_millis(100)).await;
5788 if !tmux.session_is_running(&session_name).await {
5789 break;
5790 }
5791 }
5792
5793 let task_key = task_id.to_string();
5795 capture
5796 .register_session("owner/repo", &task_key, &session_name)
5797 .await;
5798
5799 let config = EngineConfig {
5801 no_session_stuck_timeout: 600,
5802 stuck_timeout: 1800,
5803 ..EngineConfig::default()
5804 };
5805
5806 tick_check_session_completions(&tmux, "owner/repo", &capture, &store)
5807 .await
5808 .unwrap();
5809
5810 {
5816 let task_state = store.get(id).await.unwrap();
5817 if task_state.updated_at.starts_with("2020") {
5818 eprintln!(
5819 "Skipping test: tick_check_session_completions did not update stored \
5820 updated_at — session was not in snapshot (CI tmux environment issue)"
5821 );
5822 return;
5823 }
5824 }
5825
5826 tick_recover_stuck_tasks(
5828 &backend,
5829 &tmux,
5830 "owner/repo",
5831 &task_manager,
5832 &config,
5833 &store,
5834 &HashMap::new(),
5835 &Arc::new(DashMap::new()),
5836 )
5837 .await
5838 .unwrap();
5839
5840 let task = store.get(id).await.unwrap();
5842 assert_eq!(
5843 task.status,
5844 TaskStatus::InProgress,
5845 "task should not be reclaimed to New"
5846 );
5847 }
5848
5849 #[tokio::test]
5859 async fn session_completion_matches_repo_configured_with_git_suffix() {
5860 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5861 let tmux = Arc::new(TmuxManager::new());
5862 let transport = Arc::new(Transport::new());
5863 let capture = Arc::new(CaptureService::new(transport));
5864
5865 let repo = "owner/repo.git";
5866 let id = store
5867 .upsert_external(&crate::store::UpsertExternal {
5868 repo,
5869 ext_id: "201",
5870 title: "Git-suffix repo task",
5871 body: "",
5872 author: "",
5873 url: "",
5874 labels: &[],
5875 origin: "github",
5876 })
5877 .await
5878 .unwrap();
5879 store
5880 .update_status(id, TaskStatus::InProgress)
5881 .await
5882 .unwrap();
5883 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
5884 .bind(id)
5885 .execute(store.pool())
5886 .await
5887 .unwrap();
5888
5889 let task_id = "201";
5892 let session_name = tmux.session_name(repo, task_id);
5893 let create_result = tmux
5894 .command()
5895 .args(["new-session", "-d", "-s", &session_name, "-c", "/tmp"])
5896 .output()
5897 .await;
5898 match create_result {
5899 Ok(o) if o.status.success() => {}
5900 _ => {
5901 eprintln!("Skipping test: tmux not available or failed to create test session");
5902 return;
5903 }
5904 }
5905 let _guard = SessionGuard(session_name.clone());
5906
5907 let _ = tmux
5908 .command()
5909 .args(["set-option", "-t", &session_name, "remain-on-exit", "on"])
5910 .output()
5911 .await;
5912 let _ = tmux
5913 .command()
5914 .args(["send-keys", "-t", &session_name, "exit", "Enter"])
5915 .output()
5916 .await;
5917
5918 for _ in 0..20 {
5919 time::sleep(Duration::from_millis(100)).await;
5920 if !tmux.session_is_running(&session_name).await {
5921 break;
5922 }
5923 }
5924
5925 capture.register_session(repo, task_id, &session_name).await;
5926
5927 tick_check_session_completions(&tmux, repo, &capture, &store)
5928 .await
5929 .unwrap();
5930
5931 let task_state = store.get(id).await.unwrap();
5932 if task_state.updated_at.starts_with("2020") {
5933 eprintln!(
5934 "Skipping test: tick_check_session_completions did not update stored \
5935 updated_at — session was not in snapshot (CI tmux environment issue)"
5936 );
5937 return;
5938 }
5939 assert!(
5940 !task_state.updated_at.starts_with("2020"),
5941 "tick_check_session_completions must match its own repo's live sessions \
5942 even when the repo is configured with a \".git\" suffix"
5943 );
5944 }
5945
5946 #[tokio::test]
5952 async fn job_scheduler_returns_ok_on_duplicate_job_config() {
5953 let tmp = tempfile::tempdir().unwrap();
5954 let cfg = tmp.path().join(".orch.yml");
5955 fs::write(
5956 &cfg,
5957 "jobs:\n - id: trading-scan\n schedule: \"0 9 * * *\"\n task:\n title: Scan\n body: do it\n",
5958 )
5959 .unwrap();
5960 let jobs_dir = tmp.path().join("prompts").join("jobs");
5961 fs::create_dir_all(&jobs_dir).unwrap();
5962 fs::write(
5963 jobs_dir.join("trading-scan.md"),
5964 "---\nid: trading-scan\nschedule: '0 9 * * *'\ntitle: Trading Scan\n---\n\nDo trading scan.\n",
5965 )
5966 .unwrap();
5967
5968 let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
5969 let result = tick_job_scheduler(&cfg, &backend, None, "test/repo", None).await;
5970 assert!(
5971 result.is_ok(),
5972 "duplicate job id must not propagate as Err: {result:?}"
5973 );
5974 }
5975
5976 #[tokio::test]
5980 async fn job_scheduler_processes_valid_project_when_other_has_duplicate() {
5981 let bad_tmp = tempfile::tempdir().unwrap();
5983 let bad_cfg = bad_tmp.path().join(".orch.yml");
5984 fs::write(
5985 &bad_cfg,
5986 "jobs:\n - id: dup\n schedule: \"0 9 * * *\"\n task:\n title: Dup\n body: x\n",
5987 )
5988 .unwrap();
5989 let bad_jobs_dir = bad_tmp.path().join("prompts").join("jobs");
5990 fs::create_dir_all(&bad_jobs_dir).unwrap();
5991 fs::write(
5992 bad_jobs_dir.join("dup.md"),
5993 "---\nid: dup\nschedule: '0 9 * * *'\ntitle: Dup\n---\n\nbody\n",
5994 )
5995 .unwrap();
5996
5997 let good_tmp = tempfile::tempdir().unwrap();
5999 let good_cfg = good_tmp.path().join(".orch.yml");
6000 fs::write(
6001 &good_cfg,
6002 "jobs:\n - id: valid-job\n schedule: \"0 9 * * *\"\n task:\n title: Valid\n body: ok\n",
6003 )
6004 .unwrap();
6005
6006 let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
6007
6008 let bad_result = tick_job_scheduler(&bad_cfg, &backend, None, "test/bad-repo", None).await;
6009 assert!(
6010 bad_result.is_ok(),
6011 "bad project must not abort scheduler: {bad_result:?}"
6012 );
6013
6014 let good_result =
6015 tick_job_scheduler(&good_cfg, &backend, None, "test/good-repo", None).await;
6016 assert!(
6017 good_result.is_ok(),
6018 "valid project must still run after bad project: {good_result:?}"
6019 );
6020 }
6021
6022 #[serial_test::serial(cooldown_state)]
6030 #[tokio::test]
6031 async fn global_sweep_routes_inactive_repo_tasks() {
6032 cooldown::reset_global_state().await;
6033
6034 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6035
6036 let active_id = store
6038 .upsert_external(&UpsertExternal {
6039 repo: "active/repo",
6040 ext_id: "1",
6041 title: "Active task",
6042 body: "",
6043 author: "user",
6044 url: "https://github.com/active/repo/issues/1",
6045 labels: &[],
6046 origin: "github",
6047 })
6048 .await
6049 .unwrap();
6050
6051 let inactive_id = store
6053 .upsert_external(&UpsertExternal {
6054 repo: "inactive/repo",
6055 ext_id: "42",
6056 title: "Orphaned task",
6057 body: "",
6058 author: "user",
6059 url: "https://github.com/inactive/repo/issues/42",
6060 labels: &[],
6061 origin: "github",
6062 })
6063 .await
6064 .unwrap();
6065
6066 assert_eq!(store.get(active_id).await.unwrap().status, TaskStatus::New);
6067 assert_eq!(
6068 store.get(inactive_id).await.unwrap().status,
6069 TaskStatus::New
6070 );
6071
6072 let mut model_map: HashMap<String, HashMap<String, Vec<String>>> = HashMap::new();
6073 let mut tier = HashMap::new();
6074 tier.insert("claude".to_string(), vec!["haiku".to_string()]);
6075 model_map.insert("medium".to_string(), tier);
6076 let config = RouterConfig {
6077 mode: "round_robin".to_string(),
6078 agents: vec!["claude".to_string()],
6079 model_map,
6080 ..Default::default()
6081 };
6082 let mut router = Router::new_for_test(config, vec!["claude".to_string()]);
6083
6084 let active_repos: HashSet<String> = ["active/repo".to_string()].into_iter().collect();
6085 let mut routing_quota = RoutingTickQuota::new(1);
6086 route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
6087 .await
6088 .unwrap();
6089
6090 assert_eq!(
6092 store.get(active_id).await.unwrap().status,
6093 TaskStatus::New,
6094 "global sweep must not touch tasks for the active repo"
6095 );
6096
6097 let after = store.get(inactive_id).await.unwrap();
6099 assert_eq!(
6100 after.status,
6101 TaskStatus::Routed,
6102 "global sweep must route tasks from inactive repos"
6103 );
6104 assert!(
6105 matches!(&after.agent, Some(a) if !a.is_empty()),
6106 "agent must be set after routing"
6107 );
6108 }
6109
6110 #[serial_test::serial(cooldown_state)]
6118 #[tokio::test]
6119 async fn global_sweep_does_not_touch_other_active_repo_tasks() {
6120 cooldown::reset_global_state().await;
6121
6122 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6123
6124 let other_active_id = store
6127 .upsert_external(&UpsertExternal {
6128 repo: "other-active/repo",
6129 ext_id: "7",
6130 title: "Other active project's task",
6131 body: "",
6132 author: "user",
6133 url: "https://github.com/other-active/repo/issues/7",
6134 labels: &[],
6135 origin: "github",
6136 })
6137 .await
6138 .unwrap();
6139
6140 let inactive_id = store
6142 .upsert_external(&UpsertExternal {
6143 repo: "inactive/repo",
6144 ext_id: "42",
6145 title: "Orphaned task",
6146 body: "",
6147 author: "user",
6148 url: "https://github.com/inactive/repo/issues/42",
6149 labels: &[],
6150 origin: "github",
6151 })
6152 .await
6153 .unwrap();
6154
6155 let mut model_map: HashMap<String, HashMap<String, Vec<String>>> = HashMap::new();
6156 let mut tier = HashMap::new();
6157 tier.insert("claude".to_string(), vec!["haiku".to_string()]);
6158 model_map.insert("medium".to_string(), tier);
6159 let config = RouterConfig {
6160 mode: "round_robin".to_string(),
6161 agents: vec!["claude".to_string()],
6162 model_map,
6163 ..Default::default()
6164 };
6165 let mut router = Router::new_for_test(config, vec!["claude".to_string()]);
6166
6167 let active_repos: HashSet<String> =
6171 ["active/repo".to_string(), "other-active/repo".to_string()]
6172 .into_iter()
6173 .collect();
6174 let mut routing_quota = RoutingTickQuota::new(1);
6175 route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
6176 .await
6177 .unwrap();
6178
6179 assert_eq!(
6180 store.get(other_active_id).await.unwrap().status,
6181 TaskStatus::New,
6182 "global sweep run from a different active repo's tick must not route \
6183 another active repo's new tasks — that repo's own tick will route them"
6184 );
6185 assert_eq!(
6186 store.get(inactive_id).await.unwrap().status,
6187 TaskStatus::Routed,
6188 "global sweep must still route tasks from genuinely orphaned repos"
6189 );
6190 }
6191
6192 #[serial_test::serial(cooldown_state)]
6194 #[tokio::test]
6195 async fn global_sweep_skips_no_agent_tasks() {
6196 cooldown::reset_global_state().await;
6197
6198 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6199
6200 let id = store
6201 .upsert_external(&UpsertExternal {
6202 repo: "inactive/repo",
6203 ext_id: "99",
6204 title: "No-agent task",
6205 body: "",
6206 author: "user",
6207 url: "https://github.com/inactive/repo/issues/99",
6208 labels: &["no-agent".to_string()],
6209 origin: "github",
6210 })
6211 .await
6212 .unwrap();
6213
6214 let mut model_map: HashMap<String, HashMap<String, Vec<String>>> = HashMap::new();
6215 let mut tier = HashMap::new();
6216 tier.insert("claude".to_string(), vec!["haiku".to_string()]);
6217 model_map.insert("medium".to_string(), tier);
6218 let config = RouterConfig {
6219 mode: "round_robin".to_string(),
6220 agents: vec!["claude".to_string()],
6221 model_map,
6222 ..Default::default()
6223 };
6224 let mut router = Router::new_for_test(config, vec!["claude".to_string()]);
6225
6226 let active_repos: HashSet<String> = ["active/repo".to_string()].into_iter().collect();
6227 let mut routing_quota = RoutingTickQuota::new(1);
6228 route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
6229 .await
6230 .unwrap();
6231
6232 assert_eq!(
6233 store.get(id).await.unwrap().status,
6234 TaskStatus::New,
6235 "no-agent tasks must not be routed"
6236 );
6237 }
6238
6239 #[serial_test::serial(cooldown_state)]
6244 #[tokio::test]
6245 async fn global_sweep_honors_shared_routing_quota() {
6246 cooldown::reset_global_state().await;
6247
6248 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6249 for ext_id in ["1", "2"] {
6250 store
6251 .upsert_external(&UpsertExternal {
6252 repo: "inactive/repo",
6253 ext_id,
6254 title: "Orphaned task",
6255 body: "",
6256 author: "user",
6257 url: &format!("https://github.com/inactive/repo/issues/{ext_id}"),
6258 labels: &[],
6259 origin: "github",
6260 })
6261 .await
6262 .unwrap();
6263 }
6264
6265 let mut model_map: HashMap<String, HashMap<String, Vec<String>>> = HashMap::new();
6266 let mut tier = HashMap::new();
6267 tier.insert("claude".to_string(), vec!["haiku".to_string()]);
6268 model_map.insert("medium".to_string(), tier);
6269 let config = RouterConfig {
6270 mode: "round_robin".to_string(),
6271 agents: vec!["claude".to_string()],
6272 model_map,
6273 ..Default::default()
6274 };
6275 let mut router = Router::new_for_test(config, vec!["claude".to_string()]);
6276
6277 let active_repos: HashSet<String> = ["active/repo".to_string()].into_iter().collect();
6278 let mut routing_quota = RoutingTickQuota::new(1);
6279 route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
6280 .await
6281 .unwrap();
6282
6283 let all_tasks = store.list_all_global().await.unwrap();
6284 let routed = all_tasks
6285 .iter()
6286 .filter(|task| task.status == TaskStatus::Routed)
6287 .count();
6288 let still_new = all_tasks
6289 .iter()
6290 .filter(|task| task.status == TaskStatus::New)
6291 .count();
6292
6293 assert_eq!(routed, 1, "only one task may consume the shared quota");
6294 assert_eq!(still_new, 1, "remaining tasks wait for the next tick");
6295
6296 route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
6297 .await
6298 .unwrap();
6299 let after_exhausted = store.list_all_global().await.unwrap();
6300 assert_eq!(
6301 after_exhausted
6302 .iter()
6303 .filter(|task| task.status == TaskStatus::Routed)
6304 .count(),
6305 1,
6306 "an exhausted shared quota must not reset within the same tick"
6307 );
6308 }
6309
6310 #[tokio::test]
6313 async fn block_task_without_project_blocks_with_the_cause() {
6314 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6315 let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
6316 let task_manager = Arc::new(TaskManager::with_store(
6317 backend,
6318 Arc::clone(&store),
6319 "removed/repo".to_string(),
6320 ));
6321 let id = store
6322 .create_internal("removed/repo", "Orphan", "", "cron", "1", None)
6323 .await
6324 .unwrap();
6325 store.update_status(id, TaskStatus::Routed).await.unwrap();
6326
6327 block_task_without_project(
6328 &task_manager,
6329 &store,
6330 "removed/repo",
6331 &format!("internal:{id}"),
6332 )
6333 .await;
6334
6335 let task = store.get(id).await.unwrap();
6336 assert_eq!(task.status, TaskStatus::Blocked);
6337 assert_eq!(
6338 task.block_reason.as_deref(),
6339 Some(NoProjectDir::BLOCK_REASON)
6340 );
6341 }
6342
6343 #[tokio::test]
6360 async fn group_orphaned_routed_tasks_selects_inactive_repo_tasks() {
6361 async fn make_routed(store: &TaskStore, repo: &str, ext_id: &str) -> i64 {
6362 let id = store
6363 .upsert_external(&UpsertExternal {
6364 repo,
6365 ext_id,
6366 title: "Routed task",
6367 body: "",
6368 author: "user",
6369 url: &format!("https://github.com/{repo}/issues/{ext_id}"),
6370 labels: &[],
6371 origin: "github",
6372 })
6373 .await
6374 .unwrap();
6375 store
6376 .store_route(&StoreRoute {
6377 id,
6378 agent: "claude",
6379 model: Some("sonnet"),
6380 complexity: "medium",
6381 estimate: 0,
6382 reason: "test",
6383 profile: "{}",
6384 })
6385 .await
6386 .unwrap();
6387 store.update_status(id, TaskStatus::Routed).await.unwrap();
6388 id
6389 }
6390
6391 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6392
6393 let active_id = make_routed(&store, "active/repo", "1").await;
6394 let inactive_id_a = make_routed(&store, "inactive/repo", "2").await;
6395 let other_inactive_id = make_routed(&store, "other-inactive/repo", "3").await;
6396 let inactive_id_b = make_routed(&store, "inactive/repo", "4").await;
6397
6398 let all_routed = store
6399 .list_all_by_status_global(TaskStatus::Routed)
6400 .await
6401 .unwrap();
6402 let active_repos: HashSet<String> = ["active/repo".to_string()].into_iter().collect();
6403 let by_repo = group_orphaned_routed_tasks(all_routed, &active_repos);
6404
6405 assert!(
6406 !by_repo.contains_key("active/repo"),
6407 "global sweep must not select tasks for the active repo"
6408 );
6409 let inactive_ids: HashSet<i64> = by_repo
6410 .get("inactive/repo")
6411 .expect("inactive/repo must be selected")
6412 .iter()
6413 .map(|t| t.id)
6414 .collect();
6415 assert_eq!(
6416 inactive_ids,
6417 [inactive_id_a, inactive_id_b].into_iter().collect(),
6418 "global sweep must group all orphaned tasks for a repo together"
6419 );
6420 assert_eq!(
6421 by_repo
6422 .get("other-inactive/repo")
6423 .map(|tasks| tasks.iter().map(|t| t.id).collect::<Vec<_>>()),
6424 Some(vec![other_inactive_id])
6425 );
6426
6427 assert_eq!(
6430 store.get(active_id).await.unwrap().status,
6431 TaskStatus::Routed
6432 );
6433 }
6434
6435 #[tokio::test]
6440 async fn group_orphaned_routed_tasks_excludes_all_active_repos_not_just_current() {
6441 async fn make_routed(store: &TaskStore, repo: &str, ext_id: &str) -> i64 {
6442 let id = store
6443 .upsert_external(&UpsertExternal {
6444 repo,
6445 ext_id,
6446 title: "Routed task",
6447 body: "",
6448 author: "user",
6449 url: &format!("https://github.com/{repo}/issues/{ext_id}"),
6450 labels: &[],
6451 origin: "github",
6452 })
6453 .await
6454 .unwrap();
6455 store
6456 .store_route(&StoreRoute {
6457 id,
6458 agent: "claude",
6459 model: Some("sonnet"),
6460 complexity: "medium",
6461 estimate: 0,
6462 reason: "test",
6463 profile: "{}",
6464 })
6465 .await
6466 .unwrap();
6467 store.update_status(id, TaskStatus::Routed).await.unwrap();
6468 id
6469 }
6470
6471 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6472
6473 let repo_a_id = make_routed(&store, "active/repo-a", "1").await;
6474 let repo_b_id = make_routed(&store, "active/repo-b", "2").await;
6475 let orphaned_id = make_routed(&store, "inactive/repo", "3").await;
6476
6477 let all_routed = store
6478 .list_all_by_status_global(TaskStatus::Routed)
6479 .await
6480 .unwrap();
6481 let active_repos: HashSet<String> =
6484 ["active/repo-a".to_string(), "active/repo-b".to_string()]
6485 .into_iter()
6486 .collect();
6487 let by_repo = group_orphaned_routed_tasks(all_routed, &active_repos);
6488
6489 assert!(
6490 !by_repo.contains_key("active/repo-a"),
6491 "the current repo's own tasks must not be swept"
6492 );
6493 assert!(
6494 !by_repo.contains_key("active/repo-b"),
6495 "another active repo's tasks must not be swept during repo A's tick"
6496 );
6497 assert_eq!(
6498 by_repo
6499 .get("inactive/repo")
6500 .map(|tasks| tasks.iter().map(|t| t.id).collect::<Vec<_>>()),
6501 Some(vec![orphaned_id]),
6502 "a genuinely inactive repo's tasks must still be swept"
6503 );
6504
6505 assert_eq!(
6507 store.get(repo_a_id).await.unwrap().status,
6508 TaskStatus::Routed
6509 );
6510 assert_eq!(
6511 store.get(repo_b_id).await.unwrap().status,
6512 TaskStatus::Routed
6513 );
6514 }
6515
6516 #[tokio::test]
6518 async fn global_dispatch_sweep_skips_no_agent_tasks() {
6519 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6520
6521 let id = store
6522 .upsert_external(&UpsertExternal {
6523 repo: "inactive/repo",
6524 ext_id: "99",
6525 title: "No-agent routed task",
6526 body: "",
6527 author: "user",
6528 url: "https://github.com/inactive/repo/issues/99",
6529 labels: &["no-agent".to_string()],
6530 origin: "github",
6531 })
6532 .await
6533 .unwrap();
6534 store
6535 .store_route(&StoreRoute {
6536 id,
6537 agent: "claude",
6538 model: Some("sonnet"),
6539 complexity: "medium",
6540 estimate: 0,
6541 reason: "test",
6542 profile: "{}",
6543 })
6544 .await
6545 .unwrap();
6546 store.update_status(id, TaskStatus::Routed).await.unwrap();
6547
6548 let tmux = Arc::new(TmuxManager::new());
6553 let transport = Arc::new(Transport::new());
6554 let capture = Arc::new(CaptureService::new(transport));
6555 let semaphore = Arc::new(Semaphore::new(4));
6556 let (weight_tx, _weight_rx) = mpsc::channel(16);
6557 let dispatching = Arc::new(DashMap::new());
6558 let dispatch_mode = DispatchMode {
6559 is_degraded: false,
6560 healthy_agents: 1,
6561 threshold: 1,
6562 };
6563
6564 let active_repos: HashSet<String> = ["active/repo".to_string()].into_iter().collect();
6565 dispatch_routed_tasks_global(
6566 &tmux,
6567 &capture,
6568 &semaphore,
6569 &weight_tx,
6570 dispatch_mode,
6571 &dispatching,
6572 &store,
6573 &HashMap::new(),
6574 &active_repos,
6575 )
6576 .await
6577 .unwrap();
6578
6579 assert_eq!(
6580 store.get(id).await.unwrap().status,
6581 TaskStatus::Routed,
6582 "no-agent tasks must not be dispatched"
6583 );
6584 }
6585}