1use crate::backends::{ExternalBackend, ExternalId, ExternalTask, Status};
13use crate::channels::capture::CaptureService;
14use crate::config;
15use crate::engine::cooldown::{
16 github_circuit_remaining_secs, is_agent_in_cooldown, is_github_circuit_open,
17 is_model_in_cooldown, record_agent_failure_with_message, record_silence_detection,
18 set_agent_cooldown, set_model_cooldown, SILENCE_AGENT_COOLDOWN_SECS,
19 SILENCE_EXTENDED_COOLDOWN_SECS,
20};
21use crate::engine::dispatch_guard::DispatchGuard;
22use crate::engine::jobs;
23use crate::engine::router::{get_route_result, AllAgentsCooledError, RouteResultError, Router};
24use crate::engine::runner::{TaskRunner, WeightSignal};
25use crate::engine::tasks::{is_internal_id, TaskManager};
26use crate::repo_context::REPO_CONTEXT;
27use crate::store::TaskStore;
28use crate::tmux::TmuxManager;
29use dashmap::{DashMap, DashSet};
30use std::sync::atomic::{AtomicBool, Ordering};
31use std::sync::Arc;
32use std::sync::LazyLock;
33use std::time::{Instant, SystemTime};
34use tokio::sync::{mpsc, Semaphore};
35
36static ENGINE_START_TIME: LazyLock<u64> = LazyLock::new(|| {
39 SystemTime::now()
40 .duration_since(SystemTime::UNIX_EPOCH)
41 .unwrap_or_default()
42 .as_secs()
43});
44
45static STARTUP_CLEANUP_DONE: AtomicBool = AtomicBool::new(false);
47
48pub(crate) struct RoutingTickQuota {
55 remaining: usize,
56}
57
58impl RoutingTickQuota {
59 pub(crate) fn new(max: usize) -> Self {
60 Self { remaining: max }
61 }
62
63 fn is_exhausted(&self) -> bool {
64 self.remaining == 0
65 }
66
67 fn remaining(&self) -> usize {
68 self.remaining
69 }
70
71 fn consume_one(&mut self) -> bool {
72 if self.remaining == 0 {
73 return false;
74 }
75 self.remaining -= 1;
76 true
77 }
78}
79
80async fn startup_cleanup(tmux: &TmuxManager) {
83 if STARTUP_CLEANUP_DONE
85 .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
86 .is_err()
87 {
88 return;
89 }
90
91 let start_time = *ENGINE_START_TIME;
92 tracing::info!(
93 start_time,
94 "running startup cleanup for stale tmux sessions"
95 );
96
97 match tmux.kill_stale_sessions(start_time).await {
98 Ok(killed) => {
99 if killed > 0 {
100 tracing::info!(killed, "startup cleanup killed stale sessions");
101 } else {
102 tracing::debug!("no stale tmux sessions found on startup");
103 }
104 }
105 Err(e) => {
106 tracing::warn!(error = %e, "failed to run startup cleanup");
107 }
108 }
109}
110
111use super::EngineConfig;
112use crate::store::{set_review_session_expected, store_set_by_id, store_touch_updated_at};
113
114async fn recover_routed_blocked_dispatch(
115 backend: &Arc<dyn ExternalBackend>,
116 store: &Arc<TaskStore>,
117 task_manager: &Arc<TaskManager>,
118 repo: &str,
119 task: &ExternalTask,
120 timing: &StuckTaskTiming,
121) {
122 tracing::warn!(
123 task_id = task.id.0,
124 age_mins = timing.age.num_minutes(),
125 threshold_mins = timing.threshold / 60,
126 "recovering routed task: dispatch blocked by stale tmux session → new"
127 );
128
129 let resolved_store_id = match store.resolve_task_id(repo, &task.id.0).await {
130 Ok(Some(id)) => Some(id),
131 Ok(None) => {
132 tracing::warn!(
133 task_id = %task.id.0,
134 repo,
135 "resolve_task_id returned None during routed-task recovery"
136 );
137 None
138 }
139 Err(e) => {
140 tracing::error!(
141 task_id = %task.id.0,
142 repo,
143 error = %e,
144 "resolve_task_id failed during routed-task recovery"
145 );
146 None
147 }
148 };
149
150 if let Some(store_id) = resolved_store_id {
151 if let Err(e) = store_set_by_id(
152 &Some(store),
153 store_id,
154 &[
155 ("agent", serde_json::Value::Null),
156 ("model", serde_json::Value::Null),
157 ("route_attempts", serde_json::json!(0)),
158 ],
159 )
160 .await
161 {
162 tracing::warn!(
163 task_id = task.id.0,
164 ?e,
165 "failed to clear routing fields for routed stuck task — will retry next tick"
166 );
167 return;
168 }
169 finalize_open_runs_for_recovery(
170 store,
171 store_id,
172 "stuck-task recovery: stale session blocking dispatch killed",
173 )
174 .await;
175 }
176
177 if let Err(e) = task_manager.update_task_status(&task.id, Status::New).await {
178 tracing::warn!(
179 task_id = task.id.0,
180 ?e,
181 "failed to reset routed stuck task status"
182 );
183 return;
184 }
185
186 crate::store::store_log_activity(
187 &Some(Arc::clone(store)),
188 repo,
189 &task.id.0,
190 "rerouted",
191 Some("routed"),
192 Some("new"),
193 None::<&str>,
194 None::<&str>,
195 Some(&serde_json::json!({
196 "failure_reason": "routed_blocked_by_existing_session",
197 "age_minutes": timing.age.num_minutes(),
198 "had_session": true,
199 })),
200 )
201 .await;
202
203 if !is_internal_id(&task.id.0) {
204 let backend_clone = Arc::clone(backend);
205 let task_id_clone = task.id.clone();
206 let body = format!(
207 "[{}] recovered: stuck routed task — existing tmux session blocked dispatch for {}m; cleared route for retry{}",
208 chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
209 timing.age.num_minutes(),
210 crate::engine::orch_footer()
211 );
212 tokio::spawn(async move {
213 if let Err(e) = backend_clone.post_comment(&task_id_clone, &body).await {
214 tracing::warn!(
215 task_id = task_id_clone.0,
216 ?e,
217 "failed to post routed-task recovery comment"
218 );
219 }
220 });
221 }
222}
223
224pub(crate) async fn tick_check_session_completions(
226 tmux: &Arc<TmuxManager>,
227 repo: &str,
228 capture: &Arc<CaptureService>,
229 store: &Arc<TaskStore>,
230) -> anyhow::Result<()> {
231 let _span = tracing::info_span!("engine.tick.phase1.sessions").entered();
232 let session_snapshot = tmux.snapshot().await;
233 let repo_name = repo
239 .rsplit('/')
240 .next()
241 .unwrap_or(repo)
242 .trim_end_matches(".git");
243 for (session, active) in session_snapshot {
244 if session.project != repo_name {
246 continue;
247 }
248
249 if !active {
250 tracing::info!(
251 session = %session.name,
252 task_id = %session.task_id,
253 "session completed, collecting results"
254 );
255 let store_task_id = {
266 let base = session
267 .task_id
268 .strip_suffix("-review")
269 .unwrap_or(&session.task_id);
270 if let Some(n) = base.strip_prefix("internal-") {
271 format!("internal:{n}")
272 } else {
273 base.to_string()
274 }
275 };
276 store_touch_updated_at(&Some(Arc::clone(store)), repo, &store_task_id).await;
277 capture.unregister_session(repo, &session.task_id).await;
280 if let Err(e) = tmux.kill_session(&session.name).await {
282 tracing::debug!(
283 session = %session.name,
284 ?e,
285 "kill_session failed (session may already be gone)"
286 );
287 }
288 }
289 }
290 Ok(())
291}
292
293#[derive(Debug)]
294struct StuckTaskTiming {
295 has_session: bool,
296 age: chrono::Duration,
297 threshold: u64,
298}
299
300#[allow(clippy::too_many_arguments)]
301fn stuck_task_timing_from_map(
302 tmux: &Arc<TmuxManager>,
303 repo: &str,
304 session_task_id: &str,
305 updated_at: &str,
306 config: &EngineConfig,
307 parse_error_message: &'static str,
308 session_map: &std::collections::HashMap<String, bool>,
309) -> Option<StuckTaskTiming> {
310 let session_name = tmux.session_name(repo, session_task_id);
311 let has_session = session_map.get(&session_name).copied().unwrap_or(false);
313 let threshold = if has_session {
314 config.stuck_timeout
315 } else {
316 config.no_session_stuck_timeout
317 };
318
319 let updated = match chrono::DateTime::parse_from_rfc3339(updated_at) {
320 Ok(dt) => dt.with_timezone(&chrono::Utc),
321 Err(e) => {
322 tracing::warn!(
323 task_id = session_task_id,
324 updated_at,
325 ?e,
326 error_message = parse_error_message,
327 "cannot parse updated_at"
328 );
329 return None;
330 }
331 };
332
333 let raw_age = chrono::Utc::now() - updated;
334 let suspended = super::suspend::suspended_duration_since(updated);
338 let age = (raw_age - suspended).max(chrono::Duration::zero());
339 if age.num_seconds() <= threshold as i64 {
340 return None;
341 }
342
343 Some(StuckTaskTiming {
344 has_session,
345 age,
346 threshold,
347 })
348}
349
350async fn finalize_open_runs_for_recovery(store: &TaskStore, store_id: i64, reason: &str) {
353 if let Err(e) = store
354 .finalize_incomplete_runs(store_id, "aborted", reason)
355 .await
356 {
357 tracing::warn!(
358 store_id,
359 error = %e,
360 reason,
361 "failed to finalize incomplete runs during recovery"
362 );
363 }
364}
365
366pub(crate) async fn tick_detect_silent_agents(
375 tmux: &Arc<TmuxManager>,
376 repo: &str,
377 capture: &Arc<CaptureService>,
378 backend: &Arc<dyn ExternalBackend>,
379 task_manager: &Arc<TaskManager>,
380 config: &EngineConfig,
381 store: &Arc<TaskStore>,
382) -> anyhow::Result<()> {
383 let _span = tracing::info_span!("engine.tick.phase1b.silence").entered();
384 let grace = std::time::Duration::from_secs(config.silence_grace_period);
385 let silent_sessions = capture.get_silent_sessions_for_repo(repo, grace).await;
386
387 for (task_id, session_name, silence_age_secs) in silent_sessions {
388 let use_backend = should_use_backend(&task_id);
389 let store_task = match store.resolve_task_id(repo, &task_id).await {
391 Ok(Some(store_id)) => match store.get(store_id).await {
392 Ok(task) => Some(task),
393 Err(e) => {
394 tracing::warn!(
395 task_id,
396 repo,
397 store_id,
398 operation = "store.get",
399 error = %e,
400 "silence detection: db error fetching task — agent/model cooldown will be skipped"
401 );
402 None
403 }
404 },
405 Ok(None) => {
406 tracing::debug!(
407 task_id,
408 repo,
409 operation = "resolve_task_id",
410 "silence detection: task not found in store (may have been cleaned up)"
411 );
412 None
413 }
414 Err(e) => {
415 tracing::warn!(
416 task_id,
417 repo,
418 operation = "resolve_task_id",
419 error = %e,
420 "silence detection: db error resolving task id — agent/model cooldown will be skipped"
421 );
422 None
423 }
424 };
425 let agent_name = store_task
426 .as_ref()
427 .and_then(|t| t.agent.clone())
428 .unwrap_or_default();
429 let model_name = store_task
430 .as_ref()
431 .and_then(|t| t.model.clone())
432 .unwrap_or_default();
433
434 tracing::warn!(
435 task_id,
436 agent = %agent_name,
437 model = %model_name,
438 silence_age_secs,
439 grace_secs = config.silence_grace_period,
440 cooldown_secs = config.silence_cooldown,
441 "agent silent since session start — killing session, cooling down model + agent, failing over"
442 );
443
444 if let Err(e) = tmux.kill_session(&session_name).await {
446 tracing::debug!(
447 task_id,
448 ?e,
449 "kill_session failed for silent agent (may already be gone)"
450 );
451 }
452
453 capture.unregister_session(repo, &task_id).await;
455
456 if let Some(store_id) = store_task.as_ref().map(|t| t.id) {
458 finalize_open_runs_for_recovery(store, store_id, "silence detection: session killed")
459 .await;
460 }
461
462 let mut extended_note = String::new();
463
464 if !agent_name.is_empty() && !model_name.is_empty() {
466 set_model_cooldown(
467 &agent_name,
468 &model_name,
469 config.silence_cooldown,
470 "silence_detected",
471 )
472 .await;
473 if let Some(result) = record_silence_detection(&agent_name, &model_name).await {
474 if result.extended_cooldown_applied {
475 extended_note = format!(
476 " ({} silences in 24h -> extended cooldown {}s)",
477 result.count, SILENCE_EXTENDED_COOLDOWN_SECS
478 );
479 }
480 }
481 }
482
483 if !agent_name.is_empty() {
487 set_agent_cooldown(&agent_name, SILENCE_AGENT_COOLDOWN_SECS).await;
488 }
489
490 let task_eid = ExternalId(task_id.clone());
495
496 let available: Vec<String> = crate::engine::configured_agents()
498 .into_iter()
499 .filter(|a| crate::cmd_cache::command_exists(a))
500 .collect();
501 let chain = crate::engine::runner::response::get_reroute_chain(
502 &task_id,
503 &Some(Arc::clone(store)),
504 repo,
505 )
506 .await;
507 let chain = crate::engine::runner::response::update_reroute_chain(
508 &task_id,
509 &agent_name,
510 &chain,
511 &Some(Arc::clone(store)),
512 repo,
513 )
514 .await;
515
516 let (next_status, next_agent) = if let Some(fallback) =
517 crate::engine::runner::response::pick_fallback_agent(&agent_name, &chain, &available)
518 {
519 tracing::info!(
520 task_id,
521 from = %agent_name,
522 to = %fallback,
523 "silence detection: failover to different agent, setting routed"
524 );
525 (Status::Routed, Some(fallback))
526 } else {
527 tracing::warn!(
528 task_id,
529 "silence detection: no fallback agents available, resetting to new for re-routing"
530 );
531 (Status::New, None)
532 };
533
534 if use_backend {
535 if let Some(ref st) = store_task {
536 let stale_labels: Vec<String> = st
538 .labels
539 .iter()
540 .filter(|l| {
541 l.starts_with("agent:")
542 || l.starts_with("complexity:")
543 || l.starts_with("model:")
544 })
545 .cloned()
546 .collect();
547 if !stale_labels.is_empty() {
548 let backend_clone = Arc::clone(backend);
549 let task_eid_clone = task_eid.clone();
550 tokio::spawn(async move {
551 for label in &stale_labels {
552 if let Err(e) = backend_clone.remove_label(&task_eid_clone, label).await
553 {
554 tracing::warn!(task_id = task_eid_clone.0, label, error = %e, "failed to remove label during silence detection re-route");
555 }
556 }
557 });
558 }
559 }
560 }
561
562 if let Some(store_id) = store_task.as_ref().map(|t| t.id) {
563 if let Some(ref fallback) = next_agent {
564 let _ = store_set_by_id(
568 &Some(store),
569 store_id,
570 &[
571 ("agent", serde_json::json!(fallback)),
572 ("model", serde_json::json!("")),
573 (
574 "last_error",
575 serde_json::json!(format!(
576 "silence detected after {}s, failing over to {}",
577 config.silence_grace_period, fallback
578 )),
579 ),
580 ],
581 )
582 .await;
583 } else {
584 let _ = store_set_by_id(
585 &Some(store),
586 store_id,
587 &[
588 ("agent", serde_json::Value::Null),
589 ("model", serde_json::Value::Null),
590 (
591 "last_error",
592 serde_json::json!(format!(
593 "silence detected after {}s, no fallback agents available",
594 config.silence_grace_period
595 )),
596 ),
597 ],
598 )
599 .await;
600 }
601 }
602
603 if let Err(e) = task_manager
604 .update_task_status(&task_eid, next_status)
605 .await
606 {
607 tracing::warn!(task_id, ?e, "failed to update silent task status");
608 continue;
609 }
610
611 let details = serde_json::json!({
613 "failure_reason": "agent_silence",
614 "silence_duration_secs": config.silence_grace_period,
615 "cooldown_applied": !agent_name.is_empty() && !model_name.is_empty(),
616 "fallback_available": next_agent.is_some(),
617 });
618 crate::store::store_log_activity(
619 &Some(Arc::clone(store)),
620 repo,
621 &task_id,
622 "rerouted",
623 Some("in_progress"),
624 Some(next_status.as_str()),
625 next_agent.as_deref(),
626 None,
627 Some(&details),
628 )
629 .await;
630
631 let action = if let Some(ref fallback) = next_agent {
633 format!("failing over to {fallback}")
634 } else {
635 "marking needs_review (no fallback agents)".to_string()
636 };
637 let comment = format!(
638 "[{}] agent silent for {}s since session start — killed session, cooled down model `{}:{}` for {}s{}, agent `{}` for {}s, {}{}",
639 chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
640 config.silence_grace_period,
641 agent_name,
642 model_name,
643 config.silence_cooldown,
644 extended_note,
645 agent_name,
646 SILENCE_AGENT_COOLDOWN_SECS,
647 action,
648 crate::engine::orch_footer(),
649 );
650 if use_backend {
651 let backend_clone = Arc::clone(backend);
653 let task_eid_clone = task_eid.clone();
654 tokio::spawn(async move {
655 if let Err(e) = backend_clone.post_comment(&task_eid_clone, &comment).await {
656 tracing::warn!(
657 task_id = task_eid_clone.0,
658 ?e,
659 "failed to post silence detection comment"
660 );
661 }
662 });
663 }
664 }
665
666 Ok(())
667}
668
669fn should_use_backend(task_id: &str) -> bool {
670 !is_internal_id(task_id)
671}
672
673#[allow(clippy::too_many_arguments)]
678pub(crate) async fn tick_recover_stuck_tasks(
679 backend: &Arc<dyn ExternalBackend>,
680 tmux: &Arc<TmuxManager>,
681 repo: &str,
682 task_manager: &Arc<TaskManager>,
683 config: &EngineConfig,
684 store: &Arc<TaskStore>,
685 session_map: &std::collections::HashMap<String, bool>,
686 router: &Router,
687 dispatching: &Arc<DashMap<String, String>>,
688 auto_merge_in_flight: &Arc<DashSet<String>>,
689) -> anyhow::Result<()> {
690 let _span = tracing::info_span!("engine.tick.phase2.stuck_tasks").entered();
691
692 let in_progress = match task_manager
693 .list_external_by_status(Status::InProgress)
694 .await
695 {
696 Ok(tasks) => tasks,
697 Err(e) => {
698 tracing::warn!(?e, "failed to list in_progress tasks for stuck recovery");
699 vec![]
700 }
701 };
702 for task in &in_progress {
703 let Some(timing) = stuck_task_timing_from_map(
704 tmux,
705 repo,
706 &task.id.0,
707 &task.updated_at,
708 config,
709 "cannot parse updated_at, skipping stuck-task check",
710 session_map,
711 ) else {
712 continue;
713 };
714
715 if !timing.has_session {
722 let dispatch_key = format!("{repo}/{}", task.id.0);
723 if dispatching.contains_key(&dispatch_key) {
724 tracing::info!(
732 task_id = task.id.0,
733 dispatch_key,
734 age_mins = timing.age.num_minutes(),
735 "skipping stuck-task reclaim: runner still actively processing this task"
736 );
737 continue;
738 }
739 }
740
741 if timing.has_session {
742 tracing::warn!(
743 task_id = task.id.0,
744 age_mins = timing.age.num_minutes(),
745 threshold_mins = timing.threshold / 60,
746 "recovering stuck task: timed out with active session → new"
747 );
748 } else {
749 tracing::warn!(
750 task_id = task.id.0,
751 dispatch_key = format!("{repo}/{}", task.id.0),
752 age_mins = timing.age.num_minutes(),
753 threshold_mins = timing.threshold / 60,
754 "recovering stuck task: no session found — reclaiming early → new"
755 );
756 }
757 let mut cached_store_id: Option<i64> = None;
762 if timing.has_session {
763 let store_task = match store.resolve_task_id(repo, &task.id.0).await {
764 Ok(Some(store_id)) => {
765 cached_store_id = Some(store_id);
766 match store.get(store_id).await {
767 Ok(t) => Some(t),
768 Err(e) => {
769 tracing::warn!(task_id = task.id.0, error = %e, "failed to fetch task from store for stuck-task cooldown");
770 None
771 }
772 }
773 }
774 Ok(None) => None,
775 Err(e) => {
776 tracing::warn!(task_id = task.id.0, error = %e, "failed to resolve task id for stuck-task cooldown");
777 None
778 }
779 };
780 let agent_name = store_task
781 .as_ref()
782 .and_then(|t| t.agent.clone())
783 .unwrap_or_default();
784 let model_name = store_task
785 .as_ref()
786 .and_then(|t| t.model.clone())
787 .unwrap_or_default();
788
789 if !agent_name.is_empty() && !model_name.is_empty() {
790 set_model_cooldown(
791 &agent_name,
792 &model_name,
793 config.silence_cooldown,
794 "silence_detected",
795 )
796 .await;
797 record_agent_failure_with_message(
798 &agent_name,
799 &format!(
800 "stuck with active session after {}m",
801 timing.age.num_minutes()
802 ),
803 )
804 .await;
805 }
806 if !agent_name.is_empty() {
807 set_agent_cooldown(&agent_name, SILENCE_AGENT_COOLDOWN_SECS).await;
808 }
809 tracing::warn!(
810 task_id = task.id.0,
811 agent = %agent_name,
812 model = %model_name,
813 cooldown_secs = config.silence_cooldown,
814 "stuck-task cooldown applied — router will avoid this agent/model on next attempt"
815 );
816 }
817
818 {
821 let stale_labels: Vec<String> = task
822 .labels
823 .iter()
824 .filter(|l| l.starts_with("agent:") || l.starts_with("model:"))
825 .cloned()
826 .collect();
827 if !stale_labels.is_empty() {
828 let backend_clone = Arc::clone(backend);
829 let task_id_clone = task.id.clone();
830 tokio::spawn(async move {
831 for label in &stale_labels {
832 if let Err(e) = backend_clone.remove_label(&task_id_clone, label).await {
833 tracing::warn!(task_id = task_id_clone.0, label, error = %e, "failed to remove stale routing label during stuck-task recovery");
834 }
835 }
836 });
837 }
838 }
839 let resolved_store_id = match cached_store_id {
840 Some(id) => Some(id),
841 None => match store.resolve_task_id(repo, &task.id.0).await {
842 Ok(Some(id)) => Some(id),
843 Ok(None) => {
844 tracing::warn!(
845 task_id = %task.id.0,
846 repo,
847 "resolve_task_id returned None during stuck-task recovery"
848 );
849 None
850 }
851 Err(e) => {
852 tracing::error!(
853 task_id = %task.id.0,
854 repo,
855 error = %e,
856 "resolve_task_id failed during stuck-task recovery"
857 );
858 None
859 }
860 },
861 };
862 if let Some(store_id) = resolved_store_id {
863 if let Err(e) = store_set_by_id(
864 &Some(store),
865 store_id,
866 &[
867 ("agent", serde_json::Value::Null),
868 ("model", serde_json::Value::Null),
869 ("route_attempts", serde_json::json!(0)),
870 ],
871 )
872 .await
873 {
874 tracing::warn!(
875 task_id = task.id.0,
876 ?e,
877 "failed to clear routing fields for stuck task — will retry next tick"
878 );
879 continue;
880 }
881 finalize_open_runs_for_recovery(
882 store,
883 store_id,
884 if timing.has_session {
885 "stuck-task recovery: active session killed"
886 } else {
887 "stuck-task recovery: no session found"
888 },
889 )
890 .await;
891 }
892 if let Err(e) = task_manager.update_task_status(&task.id, Status::New).await {
893 tracing::warn!(task_id = task.id.0, ?e, "failed to reset stuck task status");
894 continue;
895 }
896
897 let details = serde_json::json!({
899 "failure_reason": if timing.has_session { "stuck_with_session" } else { "stuck_no_session" },
900 "age_minutes": timing.age.num_minutes(),
901 "had_session": timing.has_session,
902 });
903 crate::store::store_log_activity(
904 &Some(Arc::clone(store)),
905 repo,
906 &task.id.0,
907 "rerouted",
908 Some("in_progress"),
909 Some("new"),
910 None::<&str>,
911 None::<&str>,
912 Some(&details),
913 )
914 .await;
915
916 let reason = if timing.has_session {
917 format!(
918 "timed out after {}m with active session (cleared agent for re-routing)",
919 timing.age.num_minutes()
920 )
921 } else {
922 format!(
923 "no session found — reclaiming early after {}m (cleared agent for re-routing)",
924 timing.age.num_minutes()
925 )
926 };
927 {
929 let backend_clone = Arc::clone(backend);
930 let task_id_clone = task.id.clone();
931 let body = format!(
932 "[{}] recovered: stuck in_progress — {}{}",
933 chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
934 reason,
935 crate::engine::orch_footer()
936 );
937 tokio::spawn(async move {
938 if let Err(e) = backend_clone.post_comment(&task_id_clone, &body).await {
939 tracing::warn!(
940 task_id = task_id_clone.0,
941 ?e,
942 "failed to post stuck-task recovery comment"
943 );
944 }
945 });
946 }
947 }
948
949 let mut routed = match task_manager.list_external_by_status(Status::Routed).await {
950 Ok(tasks) => tasks,
951 Err(e) => {
952 tracing::warn!(?e, "failed to list routed tasks for stuck recovery");
953 vec![]
954 }
955 };
956 match task_manager
957 .list_internal_by_status(crate::store::TaskStatus::Routed)
958 .await
959 {
960 Ok(tasks) => routed.extend(tasks),
961 Err(e) => tracing::warn!(
962 ?e,
963 "failed to list internal routed tasks for stuck recovery"
964 ),
965 }
966 for task in &routed {
967 let Some(timing) = stuck_task_timing_from_map(
968 tmux,
969 repo,
970 &task.id.0,
971 &task.updated_at,
972 config,
973 "cannot parse updated_at, skipping routed stuck-task check",
974 session_map,
975 ) else {
976 continue;
977 };
978
979 if !timing.has_session {
980 continue;
981 }
982
983 let session_name = tmux.session_name(repo, &task.id.0);
984 if let Err(e) = tmux.kill_session(&session_name).await {
985 tracing::warn!(
986 task_id = task.id.0,
987 session = %session_name,
988 error = %e,
989 "failed to kill stale tmux session blocking routed task"
990 );
991 continue;
992 }
993
994 recover_routed_blocked_dispatch(backend, store, task_manager, repo, task, &timing).await;
995 }
996
997 let all_external_in_progress = match store
1001 .list_all_by_status_global(crate::store::TaskStatus::InProgress)
1002 .await
1003 {
1004 Ok(tasks) => tasks,
1005 Err(e) => {
1006 tracing::warn!(
1007 ?e,
1008 "failed to list global in_progress tasks for cross-repo stuck recovery"
1009 );
1010 vec![]
1011 }
1012 };
1013 for task in all_external_in_progress
1014 .iter()
1015 .filter(|t| t.origin != "internal" && t.repo != repo)
1016 {
1017 let external_id = task
1018 .external_id
1019 .clone()
1020 .unwrap_or_else(|| task.id.to_string());
1021 let Some(timing) = stuck_task_timing_from_map(
1022 tmux,
1023 &task.repo,
1024 &external_id,
1025 &task.updated_at,
1026 config,
1027 "cannot parse updated_at, skipping cross-repo stuck-task check",
1028 session_map,
1029 ) else {
1030 continue;
1031 };
1032
1033 if !timing.has_session {
1037 let dispatch_key = format!("{}/{external_id}", task.repo);
1038 if dispatching.contains_key(&dispatch_key) {
1039 tracing::info!(
1043 task_id = task.id,
1044 repo = %task.repo,
1045 dispatch_key,
1046 age_mins = timing.age.num_minutes(),
1047 "skipping cross-repo stuck-task reclaim: runner still actively processing this task"
1048 );
1049 continue;
1050 }
1051 }
1052
1053 if timing.has_session {
1054 tracing::warn!(
1055 task_id = task.id,
1056 repo = %task.repo,
1057 age_mins = timing.age.num_minutes(),
1058 threshold_mins = timing.threshold / 60,
1059 "recovering cross-repo stuck task: timed out with active session → new"
1060 );
1061 } else {
1062 tracing::warn!(
1063 task_id = task.id,
1064 repo = %task.repo,
1065 dispatch_key = format!("{}/{external_id}", task.repo),
1066 age_mins = timing.age.num_minutes(),
1067 threshold_mins = timing.threshold / 60,
1068 "recovering cross-repo stuck task: no session found — reclaiming early → new"
1069 );
1070 }
1071
1072 if let Err(e) = store_set_by_id(
1073 &Some(store),
1074 task.id,
1075 &[
1076 ("agent", serde_json::Value::Null),
1077 ("model", serde_json::Value::Null),
1078 ("route_attempts", serde_json::json!(0)),
1079 ],
1080 )
1081 .await
1082 {
1083 tracing::warn!(
1084 task_id = task.id,
1085 repo = %task.repo,
1086 ?e,
1087 "failed to clear routing fields for cross-repo stuck task — will retry next tick"
1088 );
1089 continue;
1090 }
1091
1092 finalize_open_runs_for_recovery(
1093 store,
1094 task.id,
1095 if timing.has_session {
1096 "stuck-task recovery: cross-repo active session killed"
1097 } else {
1098 "stuck-task recovery: cross-repo no session found"
1099 },
1100 )
1101 .await;
1102
1103 if let Err(e) = store
1104 .update_status(task.id, crate::store::TaskStatus::New)
1105 .await
1106 {
1107 tracing::warn!(
1108 task_id = task.id,
1109 repo = %task.repo,
1110 ?e,
1111 "failed to reset cross-repo stuck task status"
1112 );
1113 continue;
1114 }
1115
1116 let details = serde_json::json!({
1117 "failure_reason": if timing.has_session { "stuck_with_session" } else { "stuck_no_session" },
1118 "age_minutes": timing.age.num_minutes(),
1119 "had_session": timing.has_session,
1120 "scope": "cross_repo_global_sweep",
1121 });
1122 crate::store::store_log_activity(
1123 &Some(Arc::clone(store)),
1124 &task.repo,
1125 &external_id,
1126 "rerouted",
1127 Some("in_progress"),
1128 Some("new"),
1129 None::<&str>,
1130 None::<&str>,
1131 Some(&details),
1132 )
1133 .await;
1134 }
1135
1136 use crate::store::TaskStatus as DbStatus;
1139 let internal_in_progress = match task_manager
1140 .list_internal_by_status(DbStatus::InProgress)
1141 .await
1142 {
1143 Ok(tasks) => tasks,
1144 Err(e) => {
1145 tracing::warn!(
1146 ?e,
1147 "failed to list internal in_progress tasks for stuck recovery"
1148 );
1149 vec![]
1150 }
1151 };
1152 for task in &internal_in_progress {
1153 let task_id = task.id.0.clone();
1154 let Some(timing) = stuck_task_timing_from_map(
1155 tmux,
1156 repo,
1157 &task_id,
1158 &task.updated_at,
1159 config,
1160 "cannot parse updated_at, skipping stuck internal-task check",
1161 session_map,
1162 ) else {
1163 continue;
1164 };
1165
1166 if !timing.has_session {
1172 let dispatch_key = format!("{repo}/{task_id}");
1173 if dispatching.contains_key(&dispatch_key) {
1174 tracing::info!(
1184 task_id,
1185 dispatch_key,
1186 age_mins = timing.age.num_minutes(),
1187 "skipping stuck-task reclaim: runner still actively processing this task"
1188 );
1189 continue;
1190 }
1191 }
1192
1193 if timing.has_session {
1194 tracing::warn!(
1195 task_id,
1196 age_mins = timing.age.num_minutes(),
1197 threshold_mins = timing.threshold / 60,
1198 "recovering stuck task: timed out with active session → new"
1199 );
1200 } else {
1201 tracing::warn!(
1202 task_id,
1203 dispatch_key = format!("{repo}/{task_id}"),
1204 age_mins = timing.age.num_minutes(),
1205 threshold_mins = timing.threshold / 60,
1206 "recovering stuck task: no session found — reclaiming early → new"
1207 );
1208 }
1209 let mut cached_store_id: Option<i64> = None;
1213 if timing.has_session {
1214 let store_task = match store.resolve_task_id(repo, &task_id).await {
1215 Ok(Some(store_id)) => {
1216 cached_store_id = Some(store_id);
1217 match store.get(store_id).await {
1218 Ok(t) => Some(t),
1219 Err(e) => {
1220 tracing::warn!(task_id, error = %e, "failed to fetch task from store for stuck internal-task cooldown");
1221 None
1222 }
1223 }
1224 }
1225 Ok(None) => None,
1226 Err(e) => {
1227 tracing::warn!(task_id, error = %e, "failed to resolve task id for stuck internal-task cooldown");
1228 None
1229 }
1230 };
1231 let agent_name = store_task
1232 .as_ref()
1233 .and_then(|t| t.agent.clone())
1234 .unwrap_or_default();
1235 let model_name = store_task
1236 .as_ref()
1237 .and_then(|t| t.model.clone())
1238 .unwrap_or_default();
1239
1240 if !agent_name.is_empty() && !model_name.is_empty() {
1241 set_model_cooldown(
1242 &agent_name,
1243 &model_name,
1244 config.silence_cooldown,
1245 "silence_detected",
1246 )
1247 .await;
1248 record_agent_failure_with_message(
1249 &agent_name,
1250 &format!(
1251 "internal task stuck with active session after {}m",
1252 timing.age.num_minutes()
1253 ),
1254 )
1255 .await;
1256 }
1257 if !agent_name.is_empty() {
1258 set_agent_cooldown(&agent_name, SILENCE_AGENT_COOLDOWN_SECS).await;
1259 }
1260 tracing::warn!(
1261 task_id,
1262 agent = %agent_name,
1263 model = %model_name,
1264 cooldown_secs = config.silence_cooldown,
1265 "internal stuck-task cooldown applied — router will avoid this agent/model on next attempt"
1266 );
1267 }
1268 let resolved_store_id = match cached_store_id {
1271 Some(id) => Some(id),
1272 None => match store.resolve_task_id(repo, &task_id).await {
1273 Ok(Some(id)) => Some(id),
1274 Ok(None) => {
1275 tracing::warn!(
1276 task_id = %task_id,
1277 repo,
1278 "resolve_task_id returned None during stuck internal-task recovery"
1279 );
1280 None
1281 }
1282 Err(e) => {
1283 tracing::error!(
1284 task_id = %task_id,
1285 repo,
1286 error = %e,
1287 "resolve_task_id failed during stuck internal-task recovery"
1288 );
1289 None
1290 }
1291 },
1292 };
1293 if let Some(store_id) = resolved_store_id {
1294 if let Err(e) = store_set_by_id(
1295 &Some(store),
1296 store_id,
1297 &[
1298 ("agent", serde_json::Value::Null),
1299 ("model", serde_json::Value::Null),
1300 ("route_attempts", serde_json::json!(0)),
1301 ],
1302 )
1303 .await
1304 {
1305 tracing::warn!(
1306 task_id,
1307 ?e,
1308 "failed to clear routing fields for stuck internal task — will retry next tick"
1309 );
1310 continue;
1311 }
1312 finalize_open_runs_for_recovery(
1313 store,
1314 store_id,
1315 if timing.has_session {
1316 "stuck-task recovery: internal active session killed"
1317 } else {
1318 "stuck-task recovery: internal no session found"
1319 },
1320 )
1321 .await;
1322 }
1323 if let Err(e) = task_manager
1324 .update_task_status(&ExternalId(task_id.clone()), Status::New)
1325 .await
1326 {
1327 tracing::warn!(task_id, ?e, "failed to reset stuck internal task status");
1328 }
1329 }
1330
1331 let in_review = match task_manager.list_external_by_status(Status::InReview).await {
1335 Ok(tasks) => tasks,
1336 Err(e) => {
1337 tracing::warn!(?e, "failed to list in_review tasks for stuck recovery");
1338 vec![]
1339 }
1340 };
1341 let store_in_review = match store
1345 .list_by_status(repo, crate::store::TaskStatus::InReview)
1346 .await
1347 {
1348 Ok(v) => v,
1349 Err(e) => {
1350 tracing::warn!(
1351 ?e,
1352 "failed to list store in_review tasks for review_session_expected prefetch"
1353 );
1354 vec![]
1355 }
1356 };
1357 let review_expected_set: std::collections::HashSet<String> = store_in_review
1358 .into_iter()
1359 .filter(|t| t.review_session_expected)
1360 .filter_map(|t| t.external_id)
1361 .collect();
1362 for task in &in_review {
1363 if !review_expected_set.contains(&task.id.0) {
1364 tracing::debug!(
1365 task_id = task.id.0,
1366 "in_review task is waiting on PR review, skipping stuck-session recovery"
1367 );
1368 continue;
1369 }
1370
1371 let review_task_id = format!("{}-review", task.id.0);
1372 let in_review_config = crate::engine::EngineConfig {
1380 no_session_stuck_timeout: config.in_review_no_session_stuck_timeout,
1381 ..config.clone()
1382 };
1383 let Some(timing) = stuck_task_timing_from_map(
1384 tmux,
1385 repo,
1386 &review_task_id,
1387 &task.updated_at,
1388 &in_review_config,
1389 "cannot parse updated_at, skipping stuck in_review check",
1390 session_map,
1391 ) else {
1392 continue;
1393 };
1394
1395 tracing::warn!(
1396 task_id = task.id.0,
1397 age_mins = timing.age.num_minutes(),
1398 threshold_mins = timing.threshold / 60,
1399 "recovering stuck in_review task: no session found — resetting to needs_review"
1400 );
1401 if let Err(e) = task_manager
1402 .update_task_status(&task.id, Status::NeedsReview)
1403 .await
1404 {
1405 tracing::warn!(
1406 task_id = task.id.0,
1407 ?e,
1408 "failed to reset stuck in_review task status"
1409 );
1410 continue;
1411 } else {
1412 set_review_session_expected(store, repo, &task.id.0, false).await;
1413 }
1414
1415 if let Ok(Some(store_id)) = store.resolve_task_id(repo, &task.id.0).await {
1417 finalize_open_runs_for_recovery(
1418 store,
1419 store_id,
1420 "stuck-task recovery: in_review session killed",
1421 )
1422 .await;
1423 }
1424
1425 let details = serde_json::json!({
1427 "failure_reason": "stuck_in_review",
1428 "age_minutes": timing.age.num_minutes(),
1429 "had_session": timing.has_session,
1430 });
1431 crate::store::store_log_activity(
1432 &Some(Arc::clone(store)),
1433 repo,
1434 &task.id.0,
1435 "rerouted",
1436 Some("in_review"),
1437 Some("needs_review"),
1438 None::<&str>,
1439 None::<&str>,
1440 Some(&details),
1441 )
1442 .await;
1443 if let Err(e) = backend
1444 .post_comment(
1445 &task.id,
1446 &format!(
1447 "[{}] recovered: stuck in_review — no session found after {}m, resetting to needs_review{}",
1448 chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
1449 timing.age.num_minutes(),
1450 crate::engine::orch_footer()
1451 ),
1452 )
1453 .await
1454 {
1455 tracing::warn!(
1456 task_id = task.id.0,
1457 ?e,
1458 "failed to post stuck in_review recovery comment"
1459 );
1460 }
1461 }
1462
1463 let internal_in_review = match task_manager
1465 .list_internal_by_status(DbStatus::InReview)
1466 .await
1467 {
1468 Ok(tasks) => tasks,
1469 Err(e) => {
1470 tracing::warn!(
1471 ?e,
1472 "failed to list internal in_review tasks for stuck recovery"
1473 );
1474 vec![]
1475 }
1476 };
1477 for task in &internal_in_review {
1478 let task_id = task.id.0.clone();
1479 if !review_expected_set.contains(&task_id) {
1480 tracing::debug!(
1481 task_id,
1482 "internal in_review task is waiting on PR review, skipping stuck-session recovery"
1483 );
1484 continue;
1485 }
1486
1487 let review_task_id = format!("{}-review", task_id);
1488 let in_review_config = crate::engine::EngineConfig {
1489 no_session_stuck_timeout: config.in_review_no_session_stuck_timeout,
1490 ..config.clone()
1491 };
1492 let Some(timing) = stuck_task_timing_from_map(
1493 tmux,
1494 repo,
1495 &review_task_id,
1496 &task.updated_at,
1497 &in_review_config,
1498 "cannot parse updated_at, skipping stuck internal in_review check",
1499 session_map,
1500 ) else {
1501 continue;
1502 };
1503
1504 tracing::warn!(
1505 task_id,
1506 age_mins = timing.age.num_minutes(),
1507 threshold_mins = timing.threshold / 60,
1508 "recovering stuck internal in_review task: no session found — resetting to needs_review"
1509 );
1510 if let Err(e) = task_manager
1511 .update_task_status(&ExternalId(task_id.clone()), Status::NeedsReview)
1512 .await
1513 {
1514 tracing::warn!(
1515 task_id,
1516 ?e,
1517 "failed to reset stuck internal in_review task status"
1518 );
1519 } else {
1520 set_review_session_expected(store, repo, &task_id, false).await;
1521 }
1522
1523 if let Ok(Some(store_id)) = store.resolve_task_id(repo, &task_id).await {
1525 finalize_open_runs_for_recovery(
1526 store,
1527 store_id,
1528 "stuck-task recovery: internal in_review session killed",
1529 )
1530 .await;
1531 }
1532 }
1533
1534 if let Err(e) = crate::engine::sync::auto_unblock_ci_failure_blocked_tasks_global(
1539 task_manager,
1540 store,
1541 auto_merge_in_flight,
1542 )
1543 .await
1544 {
1545 tracing::warn!(err = %e, "global CI-failure blocked sweep failed");
1546 }
1547
1548 if let Err(e) = crate::engine::sync::refire_and_escalate_stale_needs_review_global(
1555 task_manager,
1556 store,
1557 router,
1558 dispatching,
1559 )
1560 .await
1561 {
1562 tracing::warn!(err = %e, "global NeedsReview refire/escalation sweep failed");
1563 }
1564
1565 if let Err(e) = crate::engine::sync::auto_recover_rebroadcast_blocked_tasks_global(
1573 router,
1574 task_manager,
1575 store,
1576 )
1577 .await
1578 {
1579 tracing::warn!(err = %e, "global rebroadcast-blocked recovery sweep failed");
1580 }
1581
1582 Ok(())
1583}
1584
1585pub(crate) async fn tick_route_tasks(
1587 backend: &Arc<dyn ExternalBackend>,
1588 task_manager: &Arc<TaskManager>,
1589 router: &mut Router,
1590 store: &Arc<TaskStore>,
1591 repo: &str,
1592 routing_quota: &mut RoutingTickQuota,
1593) -> anyhow::Result<()> {
1594 let _span = tracing::info_span!("engine.tick.phase3a.route").entered();
1595 if let Err(err) = crate::engine::dependencies::apply_claim_gates(store).await {
1596 tracing::warn!(err = %err, "dependency claim gate failed");
1597 }
1598 if routing_quota.is_exhausted() {
1599 tracing::debug!("routing quota exhausted for this tick — skipping repo routing phase");
1600 return Ok(());
1601 }
1602
1603 if is_github_circuit_open() {
1607 let remaining = github_circuit_remaining_secs();
1608 tracing::info!(
1609 remaining_secs = remaining,
1610 "GitHub 5xx circuit breaker open — skipping routing phase"
1611 );
1612 return Ok(());
1613 }
1614 let new_tasks = task_manager.list_routable().await?;
1615 let routable: Vec<&ExternalTask> = new_tasks
1616 .iter()
1617 .filter(|t| !skips_agent(&t.labels))
1618 .collect();
1619
1620 if !routable.is_empty() {
1624 let start = std::time::Instant::now();
1625 router.refresh_health(store).await;
1626 tracing::debug!(
1627 duration_ms = start.elapsed().as_millis(),
1628 "router.refresh_health completed"
1629 );
1630 }
1631
1632 for task in routable.into_iter().take(routing_quota.remaining()) {
1634 if !routing_quota.consume_one() {
1635 break;
1636 }
1637 let _task_span = tracing::info_span!("engine.route", task_id = %task.id.0).entered();
1638
1639 let task_start = Instant::now();
1640 match router.route(task, store, repo).await {
1641 Ok(result) => {
1642 tracing::debug!(task_id = %task.id.0, duration_ms = task_start.elapsed().as_millis(), "route completed");
1643 if let Err(e) = router
1645 .store_route_result(&task.id.0, &result, store, repo)
1646 .await
1647 {
1648 tracing::warn!(
1649 task_id = task.id.0,
1650 ?e,
1651 "failed to store route result; skipping Routed transition"
1652 );
1653 continue;
1654 }
1655
1656 if is_internal_id(&task.id.0) {
1657 if let Err(e) = task_manager
1659 .update_task_status(&task.id, Status::Routed)
1660 .await
1661 {
1662 tracing::warn!(
1663 task_id = task.id.0,
1664 ?e,
1665 "failed to set internal status:routed"
1666 );
1667 continue;
1668 }
1669 } else {
1670 let backend_clone = Arc::clone(backend);
1674 let task_id_clone = task.id.clone();
1675 let task_labels: Vec<String> = task.labels.clone();
1676 let new_labels = {
1677 let mut labels = vec![
1678 format!("agent:{}", result.agent),
1679 format!("complexity:{}", result.complexity),
1680 ];
1681 if let Some(ref model) = result.model {
1682 labels.push(format!("model:{model}"));
1683 }
1684 labels
1685 };
1686 tokio::spawn(async move {
1687 for label in &task_labels {
1689 if label.starts_with("agent:")
1690 || label.starts_with("complexity:")
1691 || label.starts_with("model:")
1692 {
1693 if let Err(e) =
1694 backend_clone.remove_label(&task_id_clone, label).await
1695 {
1696 tracing::warn!(task_id = task_id_clone.0, label, error = %e, "failed to remove stale routing label during re-route");
1697 }
1698 }
1699 }
1700 if let Err(e) = backend_clone.set_labels(&task_id_clone, &new_labels).await
1702 {
1703 tracing::warn!(
1704 task_id = task_id_clone.0,
1705 ?e,
1706 "failed to set routing labels"
1707 );
1708 }
1709 });
1710
1711 if let Err(e) = task_manager
1713 .update_task_status(&task.id, Status::Routed)
1714 .await
1715 {
1716 tracing::warn!(task_id = task.id.0, ?e, "failed to set status:routed");
1717 continue;
1718 }
1719 }
1720
1721 match store.ensure_external_task(repo, task).await {
1723 Ok(store_id) => {
1724 let profile_json =
1725 serde_json::to_string(&result.profile).unwrap_or_default();
1726 let skills_json =
1727 serde_json::to_string(&result.selected_skills).unwrap_or_default();
1728 if let Err(e) = store
1729 .store_route(&crate::store::StoreRoute {
1730 id: store_id,
1731 agent: &result.agent,
1732 model: result.model.as_deref(),
1733 complexity: &result.complexity,
1734 estimate: result.estimate,
1735 reason: &result.reason,
1736 profile: &profile_json,
1737 skills: &skills_json,
1738 })
1739 .await
1740 {
1741 tracing::debug!(
1742 task_id = task.id.0,
1743 ?e,
1744 "dual-write: store_route failed"
1745 );
1746 }
1747 if let Err(e) = store
1748 .update_status(store_id, crate::store::TaskStatus::Routed)
1749 .await
1750 {
1751 tracing::debug!(
1752 task_id = task.id.0,
1753 ?e,
1754 "dual-write: update_status failed"
1755 );
1756 }
1757 if result.estimate > 0 {
1761 let backend_clone = Arc::clone(backend);
1762 let task_id = task.id.clone();
1763 let estimate = result.estimate;
1764 tokio::spawn(async move {
1765 if let Err(e) = backend_clone
1766 .sync_estimate_to_project(&task_id, estimate)
1767 .await
1768 {
1769 tracing::debug!(
1770 task_id = task_id.0,
1771 estimate,
1772 err = %e,
1773 "dual-write: estimate project sync failed"
1774 );
1775 }
1776 });
1777 }
1778 }
1779 Err(e) => {
1780 tracing::debug!(
1781 task_id = task.id.0,
1782 ?e,
1783 "dual-write: ensure_external_task failed"
1784 );
1785 }
1786 }
1787
1788 if let Some(ref warning) = result.warning {
1789 tracing::warn!(task_id = task.id.0, warning, "routing sanity warning");
1790 }
1791
1792 tracing::info!(
1793 task_id = task.id.0,
1794 agent = %result.agent,
1795 complexity = %result.complexity,
1796 reason = %result.reason,
1797 "task routed"
1798 );
1799 }
1800 Err(e) => {
1801 if let Some(cooled) = e.downcast_ref::<AllAgentsCooledError>() {
1802 tracing::error!(
1803 task_id = %task.id.0,
1804 scope = %cooled.scope(),
1805 remaining_secs = ?cooled.remaining_secs(),
1806 "ALL AGENTS COOLED — no agent available to route this task. \
1807 Task stays in 'new' and will be retried on every tick. \
1808 Check `orch cooldown list` to see what is blocking dispatch."
1809 );
1810 continue;
1811 }
1812 if let Some(waiting) = e.downcast_ref::<crate::store::WaitingOnDependency>() {
1813 tracing::info!(
1814 task_id = %task.id.0,
1815 blockers = %waiting.blockers.join(", "),
1816 "task is waiting on a dependency"
1817 );
1818 continue;
1819 }
1820 tracing::error!(task_id = task.id.0, ?e, "routing failed, skipping task");
1821 }
1822 }
1823 }
1824 Ok(())
1825}
1826
1827pub(crate) async fn route_new_tasks_global(
1843 router: &mut Router,
1844 store: &Arc<TaskStore>,
1845 active_repos: &std::collections::HashSet<String>,
1846 routing_quota: &mut RoutingTickQuota,
1847) -> anyhow::Result<()> {
1848 use crate::store::{StoreRoute, TaskStatus};
1849
1850 if routing_quota.is_exhausted() {
1851 tracing::debug!("routing quota exhausted for this tick — skipping global routing sweep");
1852 return Ok(());
1853 }
1854
1855 let all_new = store.list_all_by_status_global(TaskStatus::New).await?;
1856 let orphaned: Vec<&crate::store::Task> = all_new
1857 .iter()
1858 .filter(|t| !active_repos.contains(&t.repo))
1859 .filter(|t| !skips_agent(&t.labels))
1860 .collect();
1861
1862 if orphaned.is_empty() {
1863 return Ok(());
1864 }
1865
1866 tracing::info!(
1867 count = orphaned.len(),
1868 "global routing sweep: found new tasks from inactive repos"
1869 );
1870
1871 router.refresh_health(store).await;
1872
1873 for task in orphaned.into_iter().take(routing_quota.remaining()) {
1874 if !routing_quota.consume_one() {
1875 break;
1876 }
1877 let ext_task = crate::engine::tasks::store_task_to_external(task);
1878 let task_start = Instant::now();
1879 match router.route(&ext_task, store, &task.repo).await {
1880 Ok(result) => {
1881 tracing::debug!(
1882 task_id = %ext_task.id.0,
1883 repo = %task.repo,
1884 duration_ms = task_start.elapsed().as_millis(),
1885 "global sweep: route completed"
1886 );
1887 let profile_json = serde_json::to_string(&result.profile).unwrap_or_default();
1888 let skills_json =
1889 serde_json::to_string(&result.selected_skills).unwrap_or_default();
1890 if let Err(e) = store
1891 .store_route(&StoreRoute {
1892 id: task.id,
1893 agent: &result.agent,
1894 model: result.model.as_deref(),
1895 complexity: &result.complexity,
1896 estimate: result.estimate,
1897 reason: &result.reason,
1898 profile: &profile_json,
1899 skills: &skills_json,
1900 })
1901 .await
1902 {
1903 tracing::warn!(
1904 task_id = %ext_task.id.0,
1905 repo = %task.repo,
1906 err = %e,
1907 "global sweep: failed to store route result; skipping status update"
1908 );
1909 continue;
1910 }
1911 if let Err(e) = store.update_status(task.id, TaskStatus::Routed).await {
1912 tracing::warn!(
1913 task_id = %ext_task.id.0,
1914 repo = %task.repo,
1915 err = %e,
1916 "global sweep: failed to set status:routed"
1917 );
1918 } else {
1919 tracing::info!(
1920 task_id = %ext_task.id.0,
1921 repo = %task.repo,
1922 agent = %result.agent,
1923 complexity = %result.complexity,
1924 "global sweep: task routed"
1925 );
1926 }
1927 if let Some(ref warning) = result.warning {
1928 tracing::warn!(task_id = %ext_task.id.0, warning, "global sweep: routing sanity warning");
1929 }
1930 }
1931 Err(e) => {
1932 if let Some(cooled) = e.downcast_ref::<AllAgentsCooledError>() {
1933 tracing::error!(
1934 task_id = %ext_task.id.0,
1935 repo = %task.repo,
1936 scope = %cooled.scope(),
1937 remaining_secs = ?cooled.remaining_secs(),
1938 "global sweep: ALL AGENTS COOLED — task stays in 'new'"
1939 );
1940 continue;
1941 }
1942 if let Some(waiting) = e.downcast_ref::<crate::store::WaitingOnDependency>() {
1943 tracing::info!(
1944 task_id = %ext_task.id.0,
1945 repo = %task.repo,
1946 blockers = %waiting.blockers.join(", "),
1947 "global sweep: task is waiting on a dependency"
1948 );
1949 continue;
1950 }
1951 tracing::error!(
1952 task_id = %ext_task.id.0,
1953 repo = %task.repo,
1954 err = ?e,
1955 "global sweep: routing failed, skipping task"
1956 );
1957 }
1958 }
1959 }
1960
1961 Ok(())
1962}
1963
1964#[derive(Debug, Clone, Copy)]
1966pub(crate) struct DispatchMode {
1967 pub(crate) is_degraded: bool,
1968 pub(crate) healthy_agents: usize,
1969 pub(crate) threshold: usize,
1970}
1971
1972pub(crate) fn dispatch_mode_from_router(router: &Router) -> DispatchMode {
1973 let threshold = crate::engine::router::config::min_healthy_agents_threshold();
1974 let healthy_agents = router.healthy_agent_count("simple");
1975 let is_degraded = router.is_degraded(threshold);
1976 DispatchMode {
1977 is_degraded,
1978 healthy_agents,
1979 threshold,
1980 }
1981}
1982
1983#[allow(clippy::too_many_arguments)]
1985#[tracing::instrument(skip_all, name = "engine.tick.phase3b.dispatch")]
1986pub(crate) async fn tick_dispatch_tasks(
1987 backend: &Arc<dyn ExternalBackend>,
1988 tmux: &Arc<TmuxManager>,
1989 repo: &str,
1990 runner: &Arc<TaskRunner>,
1991 capture: &Arc<CaptureService>,
1992 semaphore: &Arc<Semaphore>,
1993 task_manager: &Arc<TaskManager>,
1994 weight_tx: &mpsc::Sender<WeightSignal>,
1995 dispatch_mode: DispatchMode,
1996 dispatching: &Arc<DashMap<String, String>>,
1997 store: &Arc<TaskStore>,
1998 session_map: &std::collections::HashMap<String, bool>,
1999) -> anyhow::Result<()> {
2000 let mut routed_tasks = task_manager.list_external_by_status(Status::Routed).await?;
2002
2003 use crate::store::TaskStatus as DbStatus;
2005 let internal_routed = task_manager
2006 .list_internal_by_status(DbStatus::Routed)
2007 .await?;
2008 routed_tasks.extend(internal_routed);
2009
2010 let dispatchable: Vec<ExternalTask> = routed_tasks
2012 .into_iter()
2013 .filter(|t| !skips_agent(&t.labels))
2014 .collect();
2015
2016 dispatch_tasks_for_repo(
2017 backend,
2018 tmux,
2019 repo,
2020 runner,
2021 capture,
2022 semaphore,
2023 task_manager,
2024 weight_tx,
2025 dispatch_mode,
2026 dispatching,
2027 store,
2028 session_map,
2029 dispatchable,
2030 )
2031 .await
2032}
2033
2034#[allow(clippy::too_many_arguments)]
2040async fn dispatch_tasks_for_repo(
2041 backend: &Arc<dyn ExternalBackend>,
2042 tmux: &Arc<TmuxManager>,
2043 repo: &str,
2044 runner: &Arc<TaskRunner>,
2045 capture: &Arc<CaptureService>,
2046 semaphore: &Arc<Semaphore>,
2047 task_manager: &Arc<TaskManager>,
2048 weight_tx: &mpsc::Sender<WeightSignal>,
2049 dispatch_mode: DispatchMode,
2050 dispatching: &Arc<DashMap<String, String>>,
2051 store: &Arc<TaskStore>,
2052 session_map: &std::collections::HashMap<String, bool>,
2053 dispatchable: Vec<ExternalTask>,
2054) -> anyhow::Result<()> {
2055 if dispatchable.is_empty() {
2056 tracing::debug!(count = 0, "dispatchable tasks found");
2057 return Ok(());
2058 }
2059
2060 tracing::info!(count = dispatchable.len(), "dispatchable tasks found");
2061 if dispatch_mode.is_degraded {
2062 tracing::warn!(
2063 healthy_agents = dispatch_mode.healthy_agents,
2064 threshold = dispatch_mode.threshold,
2065 "degraded mode: using sequential dispatch"
2066 );
2067 }
2068
2069 let sequential_delay = if dispatch_mode.is_degraded {
2070 crate::engine::router::config::sequential_dispatch_delay_ms()
2071 } else {
2072 0
2073 };
2074
2075 for (idx, task) in dispatchable.into_iter().enumerate() {
2076 if crate::engine::dependencies::dispatch_is_waiting(store, repo, &task.id.0).await {
2077 continue;
2078 }
2079 if idx > 0 && dispatch_mode.is_degraded {
2081 let delay_ms = sequential_delay;
2082 tracing::debug!(
2083 task_id = task.id.0,
2084 delay_ms = delay_ms,
2085 "sequential dispatch: waiting before dispatch"
2086 );
2087 tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
2088 }
2089 let dispatch_key = format!("{}/{}", repo, task.id.0);
2096 {
2101 use dashmap::mapref::entry::Entry;
2102 match dispatching.entry(dispatch_key.clone()) {
2103 Entry::Occupied(existing) => {
2104 let existing_id = existing.get().clone();
2105 drop(existing); if existing_id == task.id.0 {
2107 tracing::debug!(
2108 task_id = task.id.0,
2109 "task already dispatching, skipping duplicate"
2110 );
2111 } else {
2112 tracing::warn!(
2113 task_id = task.id.0,
2114 existing_task_id = existing_id,
2115 dispatch_key,
2116 "dispatch key collision: unexpected task already holds this key"
2117 );
2118 }
2119 continue;
2120 }
2121 Entry::Vacant(slot) => {
2122 slot.insert(task.id.0.clone());
2123 tracing::info!(task_id = task.id.0, dispatch_key, "dispatch guard claimed");
2130 }
2131 }
2132 }
2133 let dispatch_guard = DispatchGuard::new(dispatching.clone(), dispatch_key.clone());
2135 let task_id = task.id.0.clone();
2136
2137 let route_result = match get_route_result(store, repo, &task_id).await {
2140 Ok(r) => Some(r),
2141 Err(RouteResultError::NoAgent { .. }) => {
2142 tracing::warn!(
2143 task_id,
2144 "routed task missing agent — resetting to new for re-routing (#1604)"
2145 );
2146 if let Err(e2) = task_manager.update_task_status(&task.id, Status::New).await {
2147 tracing::error!(task_id, error = %e2, "failed to reset task to new");
2148 }
2149 continue; }
2151 Err(e) => {
2152 tracing::warn!(task_id, error = %e, "get_route_result failed — dispatching without route info");
2153 None
2154 }
2155 };
2156
2157 if let Some(route) = route_result.as_ref() {
2163 let agent_cooled = is_agent_in_cooldown(&route.agent);
2164 let model_cooled = route
2165 .model
2166 .as_deref()
2167 .is_some_and(|model| is_model_in_cooldown(&route.agent, model));
2168 if agent_cooled || model_cooled {
2169 tracing::warn!(
2170 task_id,
2171 agent = route.agent,
2172 model = route.model.as_deref().unwrap_or(""),
2173 "routed agent/model entered cooldown after routing — deferring dispatch to next tick"
2174 );
2175 continue; }
2177 }
2178
2179 let session_name = tmux.session_name(repo, &task_id);
2181 if tmux
2182 .session_blocks_dispatch_from_map(&session_name, session_map)
2183 .await
2184 {
2185 tracing::warn!(
2186 task_id,
2187 session_name,
2188 "task has existing tmux session, skipping dispatch"
2189 );
2190 continue; }
2192
2193 let permit = match semaphore.clone().try_acquire_owned() {
2195 Ok(p) => p,
2196 Err(_) => {
2197 tracing::debug!("all parallel slots busy, skipping remaining tasks");
2198 break; }
2200 };
2201
2202 let set_in_progress_result = task_manager
2204 .update_task_status(&task.id, Status::InProgress)
2205 .await;
2206 if let Err(e) = set_in_progress_result {
2207 tracing::error!(task_id, ?e, "failed to set in_progress, skipping dispatch");
2208 drop(permit);
2209 continue; }
2211 tracing::info!(task_id, "dispatching task");
2212
2213 capture
2215 .register_session(repo, &task_id, &session_name)
2216 .await;
2217
2218 let runner = runner.clone();
2220 let backend = backend.clone();
2221 let tmux = tmux.clone();
2222 let capture = capture.clone();
2223 let task_id_for_cleanup = task_id.clone();
2224 let task_owned = task;
2225 let weight_tx = weight_tx.clone();
2226 let repo_owned = repo.to_string();
2227 let task_manager_for_spawn = task_manager.clone();
2228 let store_for_spawn = store.clone();
2229
2230 let repo_ctx = repo_owned.clone();
2231 tokio::spawn(REPO_CONTEXT.scope(repo_ctx, async move {
2232 let _dispatch_guard = dispatch_guard; let dispatch_start = std::time::Instant::now();
2234 match runner
2235 .run_with_context(&task_owned, &backend, &tmux, route_result.as_ref())
2236 .await
2237 {
2238 Ok(signal) => {
2239 tracing::info!(task_id, "task runner completed");
2240
2241 let duration = dispatch_start.elapsed().as_secs_f64();
2242
2243 let enable_review = config::get("workflow.enable_review_agent")
2249 .map(|v| v != "false")
2250 .unwrap_or(true);
2251 let display_status = match &signal {
2252 WeightSignal::Success { .. } if enable_review => {
2253 let has_pr = match store_for_spawn
2257 .resolve_task_id(&repo_owned, &task_id)
2258 .await
2259 {
2260 Ok(Some(store_id)) => match store_for_spawn
2261 .get(store_id)
2262 .await
2263 {
2264 Ok(t) => t.pr_number.is_some(),
2265 Err(e) => {
2266 tracing::warn!(task_id, err = %e, "store.get() failed — defaulting to needs_review");
2267 true
2268 }
2269 },
2270 Ok(None) => false,
2271 Err(e) => {
2272 tracing::warn!(task_id, err = %e, "failed to check PR status — defaulting to needs_review");
2273 true
2274 }
2275 };
2276 if has_pr {
2277 "needs_review"
2278 } else {
2279 "done"
2280 }
2281 }
2282 WeightSignal::Success { .. } => "done",
2283 WeightSignal::RateLimited { .. } => "new",
2284 WeightSignal::Blocked => "blocked",
2285 WeightSignal::Rerouted => "new",
2290 WeightSignal::None if enable_review => "needs_review",
2295 WeightSignal::None => "done",
2296 };
2297
2298 let _ = weight_tx.send(signal).await;
2300
2301 if display_status == "needs_review" {
2305 if let Err(e) = task_manager_for_spawn
2308 .update_task_status_with_duration(
2309 &ExternalId(task_id.clone()),
2310 Status::NeedsReview,
2311 Some(duration),
2312 )
2313 .await
2314 {
2315 tracing::error!(task_id, err = %e, "update_task_status(NeedsReview) failed — task may be stuck");
2316 }
2317 } else {
2319 let final_status = match display_status {
2321 "done" => Status::Done,
2322 "blocked" => Status::Blocked,
2323 _ => Status::New,
2324 };
2325 if let Err(e) = task_manager_for_spawn
2326 .update_task_status_with_duration(
2327 &ExternalId(task_id.clone()),
2328 final_status,
2329 Some(duration),
2330 )
2331 .await
2332 {
2333 tracing::warn!(task_id, ?e, "failed to update task status after completion");
2334 }
2335 }
2336 }
2337 Err(e) => {
2338 tracing::error!(task_id, ?e, "task runner failed");
2339 let duration = dispatch_start.elapsed().as_secs_f64();
2340 if is_internal_id(&task_id) {
2341 if let Err(ue) = task_manager_for_spawn
2343 .update_task_status_with_duration(
2344 &ExternalId(task_id.clone()),
2345 Status::NeedsReview,
2346 Some(duration),
2347 )
2348 .await
2349 {
2350 tracing::error!(task_id, err = %ue, "update_task_status(NeedsReview) failed — task may be stuck");
2351 }
2352 } else {
2353 if let Err(comment_err) = backend
2355 .post_comment(
2356 &ExternalId(task_id.clone()),
2357 &format!(
2358 "[{}] error: task runner failed: {e}{}",
2359 chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
2360 crate::engine::orch_footer()
2361 ),
2362 )
2363 .await
2364 {
2365 tracing::warn!(
2366 task_id,
2367 ?comment_err,
2368 "failed to post error comment to GitHub"
2369 );
2370 }
2371 if let Err(ue) = task_manager_for_spawn
2375 .update_task_status_with_duration(
2376 &ExternalId(task_id.clone()),
2377 Status::NeedsReview,
2378 Some(duration),
2379 )
2380 .await
2381 {
2382 tracing::error!(task_id, err = %ue, "update_task_status(NeedsReview) failed — task may be stuck");
2383 }
2384 }
2385 }
2386 }
2387
2388 capture.unregister_session(&repo_owned, &task_id_for_cleanup).await;
2390
2391 drop(permit);
2393 }));
2395 }
2396 Ok(())
2397}
2398
2399fn skips_agent(labels: &[String]) -> bool {
2400 labels
2401 .iter()
2402 .any(|label| label == "no-agent" || label == "owner:human")
2403}
2404
2405fn group_orphaned_routed_tasks(
2412 all_routed: Vec<crate::store::Task>,
2413 active_repos: &std::collections::HashSet<String>,
2414) -> std::collections::HashMap<String, Vec<crate::store::Task>> {
2415 let mut by_repo: std::collections::HashMap<String, Vec<crate::store::Task>> =
2416 std::collections::HashMap::new();
2417 for task in all_routed
2418 .into_iter()
2419 .filter(|t| !active_repos.contains(&t.repo))
2420 .filter(|t| !skips_agent(&t.labels))
2421 {
2422 by_repo.entry(task.repo.clone()).or_default().push(task);
2423 }
2424 by_repo
2425}
2426
2427#[allow(clippy::too_many_arguments)]
2447pub(crate) async fn dispatch_routed_tasks_global(
2448 tmux: &Arc<TmuxManager>,
2449 capture: &Arc<CaptureService>,
2450 semaphore: &Arc<Semaphore>,
2451 weight_tx: &mpsc::Sender<WeightSignal>,
2452 dispatch_mode: DispatchMode,
2453 dispatching: &Arc<DashMap<String, String>>,
2454 store: &Arc<TaskStore>,
2455 session_map: &std::collections::HashMap<String, bool>,
2456 active_repos: &std::collections::HashSet<String>,
2457) -> anyhow::Result<()> {
2458 use crate::store::TaskStatus;
2459
2460 let all_routed = store.list_all_by_status_global(TaskStatus::Routed).await?;
2461 let by_repo = group_orphaned_routed_tasks(all_routed, active_repos);
2462
2463 if by_repo.is_empty() {
2464 return Ok(());
2465 }
2466
2467 tracing::info!(
2468 repos = by_repo.len(),
2469 "global dispatch sweep: found routed tasks from inactive repos"
2470 );
2471
2472 for (repo, tasks) in by_repo {
2473 let backend: Arc<dyn ExternalBackend> = match crate::backends::github::GitHubBackend::new(
2474 repo.clone(),
2475 ) {
2476 Ok(b) => Arc::new(b),
2477 Err(e) => {
2478 tracing::warn!(repo = %repo, err = %e, "global sweep: failed to construct backend, skipping repo");
2479 continue;
2480 }
2481 };
2482 let task_manager = Arc::new(TaskManager::with_store(
2483 backend.clone(),
2484 store.clone(),
2485 repo.clone(),
2486 ));
2487 let runner = Arc::new(TaskRunner::new(repo.clone()).with_store(store.clone()));
2488 let dispatchable: Vec<ExternalTask> = tasks
2489 .iter()
2490 .map(crate::engine::tasks::store_task_to_external)
2491 .collect();
2492
2493 if let Err(e) = dispatch_tasks_for_repo(
2494 &backend,
2495 tmux,
2496 &repo,
2497 &runner,
2498 capture,
2499 semaphore,
2500 &task_manager,
2501 weight_tx,
2502 dispatch_mode,
2503 dispatching,
2504 store,
2505 session_map,
2506 dispatchable,
2507 )
2508 .await
2509 {
2510 tracing::warn!(repo = %repo, err = %e, "global sweep: dispatch failed for repo");
2511 }
2512 }
2513
2514 Ok(())
2515}
2516
2517pub(crate) async fn tick_unblock_parents(
2519 backend: &Arc<dyn ExternalBackend>,
2520 task_manager: &Arc<TaskManager>,
2521 store: &Arc<TaskStore>,
2522 repo: &str,
2523) -> anyhow::Result<()> {
2524 let blocked = task_manager.list_all_by_status(Status::Blocked).await?;
2525
2526 let store_blocked = store
2530 .list_by_status(repo, crate::store::TaskStatus::Blocked)
2531 .await?;
2532 let has_block_reason: std::collections::HashSet<String> = store_blocked
2533 .iter()
2534 .filter(|t| t.block_reason.is_some())
2535 .filter_map(|t| t.external_id.clone())
2536 .collect();
2537
2538 let handles: Vec<_> = blocked
2542 .into_iter()
2543 .filter(|task| {
2544 if has_block_reason.contains(&task.id.0) {
2545 tracing::debug!(
2546 task_id = task.id.0,
2547 "skipping blocked task with known block_reason"
2548 );
2549 false
2550 } else {
2551 true
2552 }
2553 })
2554 .map(|task| {
2555 let backend_clone = Arc::clone(backend);
2556 let task_manager_clone = Arc::clone(task_manager);
2557 let store_clone = Arc::clone(store);
2558 let repo_owned = repo.to_string();
2559 tokio::spawn(async move {
2560 let task_id = &task.id;
2561
2562 let mut children: Vec<ExternalId> = Vec::new();
2563 let mut child_statuses: std::collections::HashMap<
2564 String,
2565 crate::store::TaskStatus,
2566 > = std::collections::HashMap::new();
2567
2568 match store_clone.resolve_task_id(&repo_owned, &task_id.0).await {
2569 Ok(Some(parent_store_id)) => {
2570 match store_clone.list_children(&repo_owned, parent_store_id).await {
2571 Ok(store_children) => {
2572 for child in store_children {
2573 let Some(child_external_id) = child.external_id else {
2574 continue;
2575 };
2576 child_statuses.insert(child_external_id.clone(), child.status);
2577 children.push(ExternalId(child_external_id));
2578 }
2579 }
2580 Err(e) => {
2581 tracing::debug!(
2582 task_id = task_id.0,
2583 ?e,
2584 "failed to list store-linked children"
2585 );
2586 }
2587 }
2588 }
2589 Ok(None) => {}
2590 Err(e) => {
2591 tracing::debug!(
2592 task_id = task_id.0,
2593 ?e,
2594 "failed to resolve blocked task in store for child lookup"
2595 );
2596 }
2597 }
2598
2599 if !is_internal_id(&task_id.0) {
2600 match backend_clone.get_sub_issues(task_id).await {
2601 Ok(ids) => {
2602 let mut seen: std::collections::HashSet<String> =
2603 children.iter().map(|child| child.0.clone()).collect();
2604 for id in ids {
2605 if seen.insert(id.0.clone()) {
2606 children.push(id);
2607 }
2608 }
2609 }
2610 Err(e) => {
2611 tracing::debug!(task_id = task_id.0, ?e, "failed to get sub-issues");
2612 }
2613 }
2614 }
2615
2616 if children.is_empty() {
2618 return;
2619 }
2620
2621 let mut all_done = true;
2624 let child_ext_strs: Vec<String> =
2626 children.iter().map(|c| c.0.clone()).collect();
2627 let child_exts: Vec<&str> =
2628 child_ext_strs.iter().map(|s| s.as_str()).collect();
2629 if let Ok(map) = store_clone
2631 .get_statuses_by_external_ids(&repo_owned, &child_exts)
2632 .await
2633 {
2634 for (k, v) in map.into_iter() {
2635 child_statuses.insert(k, v);
2636 }
2637 } else {
2638 tracing::debug!(
2639 task_id = task_id.0,
2640 "failed to batch-query store for child statuses; falling back to per-child lookups"
2641 );
2642 }
2643
2644 for child_id in &children {
2645 if let Some(status) = child_statuses.get(&child_id.0) {
2647 if *status != crate::store::TaskStatus::Done {
2648 all_done = false;
2649 break;
2650 }
2651 continue;
2652 }
2653
2654 match backend_clone.get_task(child_id).await {
2656 Ok(child) => {
2657 if !child.labels.iter().any(|l| l == Status::Done.as_label()) {
2658 all_done = false;
2659 break;
2660 }
2661 }
2662 Err(e) => {
2663 tracing::debug!(task_id = task_id.0, child = %child_id.0, ?e, "failed to fetch child task from backend — treating as not done");
2664 all_done = false;
2665 break;
2666 }
2667 }
2668 }
2669
2670 if all_done {
2671 tracing::info!(
2672 task_id = task_id.0,
2673 children = children.len(),
2674 "all children done, unblocking parent"
2675 );
2676 if let Err(e) =
2677 task_manager_clone.update_task_status(task_id, Status::New).await
2678 {
2679 tracing::warn!(task_id = task_id.0, ?e, "failed to unblock parent");
2680 }
2681 }
2682 })
2683 })
2684 .collect();
2685
2686 for handle in handles {
2687 if let Err(e) = handle.await {
2688 tracing::warn!(?e, "phase 4 sub-issue check panicked");
2689 }
2690 }
2691 Ok(())
2692}
2693
2694pub(crate) async fn tick_job_scheduler(
2696 jobs_path: &std::path::Path,
2697 backend: &Arc<dyn ExternalBackend>,
2698 store: Option<&Arc<TaskStore>>,
2699 repo: &str,
2700 transport: Option<&Arc<crate::channels::transport::Transport>>,
2701) -> anyhow::Result<()> {
2702 jobs::tick(jobs_path, backend, store, repo, transport).await
2703}
2704
2705#[allow(clippy::too_many_arguments)]
2715pub(crate) async fn tick(
2716 backend: &Arc<dyn ExternalBackend>,
2717 tmux: &Arc<TmuxManager>,
2718 repo: &str,
2719 runner: &Arc<TaskRunner>,
2720 capture: &Arc<CaptureService>,
2721 semaphore: &Arc<Semaphore>,
2722 config: &EngineConfig,
2723 jobs_path: &std::path::Path,
2724 router: &mut Router,
2725 task_manager: &Arc<TaskManager>,
2726 weight_tx: &mpsc::Sender<WeightSignal>,
2727 dispatching: &Arc<DashMap<String, String>>,
2728 store: &Arc<TaskStore>,
2729 transport: Option<&Arc<crate::channels::transport::Transport>>,
2730 active_repos: &std::collections::HashSet<String>,
2731 auto_merge_in_flight: &Arc<DashSet<String>>,
2732 routing_quota: &mut RoutingTickQuota,
2733) -> anyhow::Result<()> {
2734 let _tick_span = tracing::info_span!("engine.tick").entered();
2735
2736 startup_cleanup(tmux).await;
2738
2739 tick_check_session_completions(tmux, repo, capture, store).await?;
2740 tick_detect_silent_agents(tmux, repo, capture, backend, task_manager, config, store).await?;
2741 let session_map = tmux.batch_session_active().await;
2744 tick_recover_stuck_tasks(
2745 backend,
2746 tmux,
2747 repo,
2748 task_manager,
2749 config,
2750 store,
2751 &session_map,
2752 router,
2753 dispatching,
2754 auto_merge_in_flight,
2755 )
2756 .await?;
2757 if let Err(err) = crate::engine::dependencies::reconcile_dependencies(
2758 backend.as_ref(),
2759 store,
2760 transport.map(|arc| arc.as_ref()),
2761 )
2762 .await
2763 {
2764 tracing::warn!(err = %err, "dependency reconcile failed");
2765 }
2766 tick_route_tasks(backend, task_manager, router, store, repo, routing_quota).await?;
2767 if let Err(e) = route_new_tasks_global(router, store, active_repos, routing_quota).await {
2771 tracing::warn!(err = %e, "global new-task routing sweep failed");
2772 }
2773 let dispatch_mode = dispatch_mode_from_router(router);
2774 tick_dispatch_tasks(
2775 backend,
2776 tmux,
2777 repo,
2778 runner,
2779 capture,
2780 semaphore,
2781 task_manager,
2782 weight_tx,
2783 dispatch_mode,
2784 dispatching,
2785 store,
2786 &session_map,
2787 )
2788 .await?;
2789 if let Err(e) = dispatch_routed_tasks_global(
2795 tmux,
2796 capture,
2797 semaphore,
2798 weight_tx,
2799 dispatch_mode,
2800 dispatching,
2801 store,
2802 &session_map,
2803 active_repos,
2804 )
2805 .await
2806 {
2807 tracing::warn!(err = %e, "global routed-task dispatch sweep failed");
2808 }
2809 tick_unblock_parents(backend, task_manager, store, repo).await?;
2810 if let Err(e) = tick_job_scheduler(jobs_path, backend, Some(store), repo, transport).await {
2811 tracing::error!(?e, "job scheduler tick failed");
2812 }
2813 Ok(())
2814}
2815
2816#[cfg(test)]
2817mod tests {
2818 use super::*;
2819 use crate::backends::{ExternalId, ExternalTask, Mention, Status};
2820 use crate::channels::transport::Transport;
2821 use crate::engine::tasks::TaskManager;
2822 use crate::store::TaskStore;
2823 use crate::tmux::{SessionGuard, TmuxManager};
2824 use async_trait::async_trait;
2825 use std::sync::{Arc, Mutex};
2826
2827 #[test]
2828 fn skips_agent_for_human_and_no_agent_labels() {
2829 assert!(skips_agent(&["owner:human".to_string()]));
2830 assert!(skips_agent(&["no-agent".to_string()]));
2831 assert!(!skips_agent(&["status:new".to_string()]));
2832 }
2833
2834 fn cooled_review_router_for_test() -> Router {
2838 Router::new_for_test(
2839 crate::engine::router::RouterConfig::default(),
2840 vec!["claude".to_string()],
2841 )
2842 }
2843
2844 struct MockBackend {
2848 blocked_tasks: Vec<ExternalTask>,
2850 sub_issues: std::collections::HashMap<String, Vec<ExternalId>>,
2852 tasks_by_id: std::collections::HashMap<String, ExternalTask>,
2854 get_sub_issues_calls: Arc<Mutex<Vec<String>>>,
2856 status_updates: Arc<Mutex<Vec<(String, Status)>>>,
2858 removed_labels: Arc<Mutex<Vec<(String, String)>>>,
2860 posted_comments: Arc<Mutex<Vec<(String, String)>>>,
2862 }
2863
2864 impl MockBackend {
2865 fn new() -> Self {
2866 Self {
2867 blocked_tasks: vec![],
2868 sub_issues: Default::default(),
2869 tasks_by_id: Default::default(),
2870 get_sub_issues_calls: Arc::new(Mutex::new(vec![])),
2871 status_updates: Arc::new(Mutex::new(vec![])),
2872 removed_labels: Arc::new(Mutex::new(vec![])),
2873 posted_comments: Arc::new(Mutex::new(vec![])),
2874 }
2875 }
2876 }
2877
2878 fn make_task(id: &str, labels: &[&str]) -> ExternalTask {
2879 ExternalTask {
2880 id: ExternalId(id.to_string()),
2881 title: format!("Task {id}"),
2882 body: "".to_string(),
2883 state: "open".to_string(),
2884 labels: labels.iter().map(|s| s.to_string()).collect(),
2885 author: "bot".to_string(),
2886 created_at: "2026-01-01T00:00:00Z".to_string(),
2887 updated_at: "2026-01-01T00:00:00Z".to_string(),
2888 url: format!("https://github.com/test/test/issues/{id}"),
2889 }
2890 }
2891
2892 #[async_trait]
2893 impl ExternalBackend for MockBackend {
2894 fn name(&self) -> &str {
2895 "mock"
2896 }
2897 async fn create_task(
2898 &self,
2899 _title: &str,
2900 _body: &str,
2901 _labels: &[String],
2902 ) -> anyhow::Result<ExternalId> {
2903 Ok(ExternalId("new".to_string()))
2904 }
2905 async fn get_task(&self, id: &ExternalId) -> anyhow::Result<ExternalTask> {
2906 self.tasks_by_id
2907 .get(&id.0)
2908 .cloned()
2909 .ok_or_else(|| anyhow::anyhow!("task not found: {}", id.0))
2910 }
2911 async fn list_by_status(&self, status: Status) -> anyhow::Result<Vec<ExternalTask>> {
2912 if status == Status::Blocked {
2913 Ok(self.blocked_tasks.clone())
2914 } else {
2915 Ok(vec![])
2916 }
2917 }
2918 async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
2919 Ok(vec![])
2920 }
2921 async fn post_comment(&self, _id: &ExternalId, _body: &str) -> anyhow::Result<()> {
2922 self.posted_comments
2923 .lock()
2924 .unwrap()
2925 .push((_id.0.clone(), _body.to_string()));
2926 Ok(())
2927 }
2928 async fn set_labels(&self, _id: &ExternalId, _labels: &[String]) -> anyhow::Result<()> {
2929 Ok(())
2930 }
2931 async fn remove_label(&self, _id: &ExternalId, _label: &str) -> anyhow::Result<()> {
2932 self.removed_labels
2933 .lock()
2934 .unwrap()
2935 .push((_id.0.clone(), _label.to_string()));
2936 Ok(())
2937 }
2938 async fn get_sub_issues(&self, id: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
2939 self.get_sub_issues_calls.lock().unwrap().push(id.0.clone());
2940 Ok(self.sub_issues.get(&id.0).cloned().unwrap_or_default())
2941 }
2942 async fn create_sub_task(
2943 &self,
2944 _parent: &ExternalId,
2945 _title: &str,
2946 _body: &str,
2947 _labels: &[String],
2948 ) -> anyhow::Result<ExternalId> {
2949 Ok(ExternalId("child".to_string()))
2950 }
2951 async fn ensure_status_label(&self, _label: &str) -> anyhow::Result<()> {
2952 Ok(())
2953 }
2954 async fn has_open_issue_with_title(
2955 &self,
2956 _title: &str,
2957 _label: &str,
2958 ) -> anyhow::Result<bool> {
2959 Ok(false)
2960 }
2961 async fn health_check(&self) -> anyhow::Result<()> {
2962 Ok(())
2963 }
2964 async fn is_pr_merged(&self, _branch: &str) -> anyhow::Result<bool> {
2965 Ok(false)
2966 }
2967 async fn get_authenticated_user(&self) -> anyhow::Result<Option<String>> {
2968 Ok(Some("testbot".to_string()))
2969 }
2970 async fn get_mentions(&self, _since: &str) -> anyhow::Result<Vec<Mention>> {
2971 Ok(vec![])
2972 }
2973 async fn update_status(&self, id: &ExternalId, status: Status) -> anyhow::Result<()> {
2974 self.status_updates
2975 .lock()
2976 .unwrap()
2977 .push((id.0.clone(), status));
2978 Ok(())
2979 }
2980 }
2981
2982 fn make_task_manager(backend: Arc<dyn ExternalBackend>) -> Arc<TaskManager> {
2983 Arc::new(TaskManager::new(backend))
2984 }
2985
2986 #[tokio::test]
2989 async fn unblock_parents_unblocks_when_all_children_done() {
2990 let mut mock = MockBackend::new();
2991
2992 let parent = make_task("10", &["status:blocked"]);
2994 mock.blocked_tasks.push(parent.clone());
2995 mock.sub_issues.insert(
2996 "10".to_string(),
2997 vec![ExternalId("11".to_string()), ExternalId("12".to_string())],
2998 );
2999 mock.tasks_by_id
3000 .insert("11".to_string(), make_task("11", &["status:done"]));
3001 mock.tasks_by_id
3002 .insert("12".to_string(), make_task("12", &["status:done"]));
3003
3004 let status_updates = mock.status_updates.clone();
3005 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3006 let task_manager = make_task_manager(backend.clone());
3007 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3008
3009 tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3010 .await
3011 .unwrap();
3012
3013 let updates = status_updates.lock().unwrap();
3014 assert_eq!(updates.len(), 1, "parent should be unblocked");
3015 assert_eq!(updates[0], ("10".to_string(), Status::New));
3016 }
3017
3018 #[tokio::test]
3019 async fn unblock_parents_skips_when_child_not_done() {
3020 let mut mock = MockBackend::new();
3021
3022 let parent = make_task("20", &["status:blocked"]);
3023 mock.blocked_tasks.push(parent.clone());
3024 mock.sub_issues.insert(
3025 "20".to_string(),
3026 vec![ExternalId("21".to_string()), ExternalId("22".to_string())],
3027 );
3028 mock.tasks_by_id
3030 .insert("21".to_string(), make_task("21", &["status:done"]));
3031 mock.tasks_by_id
3032 .insert("22".to_string(), make_task("22", &["status:in_progress"]));
3033
3034 let status_updates = mock.status_updates.clone();
3035 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3036 let task_manager = make_task_manager(backend.clone());
3037 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3038
3039 tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3040 .await
3041 .unwrap();
3042
3043 let updates = status_updates.lock().unwrap();
3044 assert!(
3045 updates.is_empty(),
3046 "should not unblock when a child is still running"
3047 );
3048 }
3049
3050 #[tokio::test]
3051 async fn unblock_parents_skips_task_with_no_children() {
3052 let mut mock = MockBackend::new();
3053
3054 let parent = make_task("30", &["status:blocked"]);
3056 mock.blocked_tasks.push(parent.clone());
3057 let status_updates = mock.status_updates.clone();
3060 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3061 let task_manager = make_task_manager(backend.clone());
3062 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3063
3064 tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3065 .await
3066 .unwrap();
3067
3068 let updates = status_updates.lock().unwrap();
3069 assert!(
3070 updates.is_empty(),
3071 "should not unblock tasks with no sub-issues"
3072 );
3073 }
3074
3075 #[tokio::test]
3076 async fn unblock_parents_skips_task_with_block_reason_in_store() {
3077 let mut mock = MockBackend::new();
3080
3081 let parent = make_task("60", &["status:blocked"]);
3083 mock.blocked_tasks.push(parent.clone());
3084 mock.sub_issues
3087 .insert("60".to_string(), vec![ExternalId("61".to_string())]);
3088 mock.tasks_by_id
3089 .insert("61".to_string(), make_task("61", &["status:done"]));
3090
3091 let status_updates = mock.status_updates.clone();
3092 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3093 let task_manager = make_task_manager(backend.clone());
3094
3095 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3097 let task_id = store
3098 .upsert_external(&crate::store::UpsertExternal {
3099 repo: "test/repo",
3100 ext_id: "60",
3101 title: "Task 60",
3102 body: "",
3103 author: "bot",
3104 url: "https://github.com/test/test/issues/60",
3105 labels: &[],
3106 origin: "github",
3107 })
3108 .await
3109 .unwrap();
3110 store
3111 .update_status(task_id, crate::store::TaskStatus::Blocked)
3112 .await
3113 .unwrap();
3114 store
3115 .set_block_reason(task_id, Some("CI failure limit reached"))
3116 .await
3117 .unwrap();
3118
3119 tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3120 .await
3121 .unwrap();
3122
3123 let updates = status_updates.lock().unwrap();
3124 assert!(
3125 updates.is_empty(),
3126 "task with block_reason should be skipped — no API call, no unblock"
3127 );
3128 }
3129
3130 #[tokio::test]
3131 async fn unblock_parents_unblocks_internal_parent_from_store_children() {
3132 let mock = MockBackend::new();
3133 let get_sub_issues_calls = mock.get_sub_issues_calls.clone();
3134 let status_updates = mock.status_updates.clone();
3135 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3136 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3137 let task_manager = Arc::new(TaskManager::with_store(
3138 backend.clone(),
3139 store.clone(),
3140 "test/repo".to_string(),
3141 ));
3142
3143 let parent_id = store
3144 .create_internal("test/repo", "Parent", "", "job", "daily", None)
3145 .await
3146 .unwrap();
3147 store
3148 .update_status(parent_id, crate::store::TaskStatus::Blocked)
3149 .await
3150 .unwrap();
3151 let child_id = store
3152 .create(&crate::store::NewTask {
3153 external_id: Some("91".to_string()),
3154 repo: "test/repo".to_string(),
3155 origin: "github".to_string(),
3156 title: "Child".to_string(),
3157 body: String::new(),
3158 source: String::new(),
3159 source_id: String::new(),
3160 author: String::new(),
3161 url: String::new(),
3162 labels: vec![],
3163 parent_id: Some(parent_id),
3164 })
3165 .await
3166 .unwrap();
3167 store
3168 .update_status(child_id, crate::store::TaskStatus::Done)
3169 .await
3170 .unwrap();
3171
3172 tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3173 .await
3174 .unwrap();
3175
3176 let parent = store.get(parent_id).await.unwrap();
3177 assert_eq!(parent.status, crate::store::TaskStatus::New);
3178 assert!(
3179 status_updates.lock().unwrap().is_empty(),
3180 "internal parent should be updated in store without backend mirroring"
3181 );
3182 assert!(
3183 get_sub_issues_calls.lock().unwrap().is_empty(),
3184 "internal parent should not query backend sub-issues"
3185 );
3186 }
3187
3188 #[cfg(test)]
3192 async fn set_task_updated_at_past(store: &TaskStore, task_id: i64) {
3193 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
3194 .bind(task_id)
3195 .execute(store.pool())
3196 .await
3197 .unwrap();
3198 }
3199
3200 #[tokio::test]
3201 async fn recover_stuck_tasks_resets_external_in_review_to_needs_review() {
3202 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3203 let mock = MockBackend::new();
3204 let status_updates = mock.status_updates.clone();
3205 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3206 let task_manager = Arc::new(TaskManager::with_store(
3207 backend.clone(),
3208 store.clone(),
3209 "owner/repo".to_string(),
3210 ));
3211 let tmux = Arc::new(TmuxManager::new());
3212 let config = EngineConfig {
3213 no_session_stuck_timeout: 600,
3214 stuck_timeout: 1800,
3215 ..EngineConfig::default()
3216 };
3217
3218 let id = store
3220 .upsert_external(&crate::store::UpsertExternal {
3221 repo: "owner/repo",
3222 ext_id: "99",
3223 title: "Stuck InReview",
3224 body: "",
3225 author: "",
3226 url: "",
3227 labels: &[],
3228 origin: "github",
3229 })
3230 .await
3231 .unwrap();
3232 store
3233 .update_status(id, crate::store::TaskStatus::InReview)
3234 .await
3235 .unwrap();
3236 set_review_session_expected(&store, "owner/repo", "99", true).await;
3237 set_task_updated_at_past(&store, id).await;
3238
3239 tick_recover_stuck_tasks(
3240 &backend,
3241 &tmux,
3242 "owner/repo",
3243 &task_manager,
3244 &config,
3245 &store,
3246 &std::collections::HashMap::new(),
3247 &cooled_review_router_for_test(),
3248 &Arc::new(DashMap::new()),
3249 &Arc::new(DashSet::new()),
3250 )
3251 .await
3252 .unwrap();
3253
3254 let updates = status_updates.lock().unwrap();
3255 assert_eq!(updates.len(), 1, "stuck InReview task should be recovered");
3256 assert_eq!(updates[0].0, "99");
3257 assert_eq!(updates[0].1, Status::NeedsReview);
3258 }
3259
3260 #[tokio::test]
3261 async fn recover_stuck_tasks_skips_recent_in_review() {
3262 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3263 let mock = MockBackend::new();
3264 let status_updates = mock.status_updates.clone();
3265 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3266 let task_manager = Arc::new(TaskManager::with_store(
3267 backend.clone(),
3268 store.clone(),
3269 "owner/repo".to_string(),
3270 ));
3271 let tmux = Arc::new(TmuxManager::new());
3272 let config = EngineConfig {
3273 no_session_stuck_timeout: 600, stuck_timeout: 1800,
3275 ..EngineConfig::default()
3276 };
3277
3278 let id = store
3280 .upsert_external(&crate::store::UpsertExternal {
3281 repo: "owner/repo",
3282 ext_id: "100",
3283 title: "Recent InReview",
3284 body: "",
3285 author: "",
3286 url: "",
3287 labels: &[],
3288 origin: "github",
3289 })
3290 .await
3291 .unwrap();
3292 store
3293 .update_status(id, crate::store::TaskStatus::InReview)
3294 .await
3295 .unwrap();
3296 tick_recover_stuck_tasks(
3299 &backend,
3300 &tmux,
3301 "owner/repo",
3302 &task_manager,
3303 &config,
3304 &store,
3305 &std::collections::HashMap::new(),
3306 &cooled_review_router_for_test(),
3307 &Arc::new(DashMap::new()),
3308 &Arc::new(DashSet::new()),
3309 )
3310 .await
3311 .unwrap();
3312
3313 let updates = status_updates.lock().unwrap();
3314 assert!(
3315 updates.is_empty(),
3316 "recent InReview task should not be recovered"
3317 );
3318 }
3319
3320 #[tokio::test]
3321 async fn recover_stuck_tasks_resets_internal_in_review_to_needs_review() {
3322 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3323 let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
3324 let task_manager = Arc::new(TaskManager::with_store(
3325 backend.clone(),
3326 store.clone(),
3327 "owner/repo".to_string(),
3328 ));
3329 let tmux = Arc::new(TmuxManager::new());
3330 let config = EngineConfig {
3331 no_session_stuck_timeout: 600,
3332 stuck_timeout: 1800,
3333 ..EngineConfig::default()
3334 };
3335
3336 let id = store
3338 .create_internal("owner/repo", "Internal InReview", "", "cron", "1", None)
3339 .await
3340 .unwrap();
3341 store
3342 .update_status(id, crate::store::TaskStatus::InReview)
3343 .await
3344 .unwrap();
3345 set_review_session_expected(&store, "owner/repo", "internal:1", true).await;
3346 set_task_updated_at_past(&store, id).await;
3347
3348 tick_recover_stuck_tasks(
3349 &backend,
3350 &tmux,
3351 "owner/repo",
3352 &task_manager,
3353 &config,
3354 &store,
3355 &std::collections::HashMap::new(),
3356 &cooled_review_router_for_test(),
3357 &Arc::new(DashMap::new()),
3358 &Arc::new(DashSet::new()),
3359 )
3360 .await
3361 .unwrap();
3362
3363 let task = store.get(id).await.unwrap();
3364 assert_eq!(
3365 task.status,
3366 crate::store::TaskStatus::NeedsReview,
3367 "stuck internal InReview task should be reset to NeedsReview"
3368 );
3369 }
3370
3371 #[tokio::test]
3372 async fn recover_stuck_tasks_resets_cross_repo_external_in_progress_to_new() {
3373 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3374 let mock = MockBackend::new();
3375 let status_updates = mock.status_updates.clone();
3376 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3377 let task_manager = Arc::new(TaskManager::with_store(
3378 backend.clone(),
3379 store.clone(),
3380 "owner/repo".to_string(),
3381 ));
3382 let tmux = Arc::new(TmuxManager::new());
3383 let config = EngineConfig {
3384 no_session_stuck_timeout: 600,
3385 stuck_timeout: 1800,
3386 ..EngineConfig::default()
3387 };
3388
3389 let id = store
3391 .upsert_external(&crate::store::UpsertExternal {
3392 repo: "owner/other",
3393 ext_id: "777",
3394 title: "Cross-repo stuck task",
3395 body: "",
3396 author: "",
3397 url: "",
3398 labels: &[],
3399 origin: "github",
3400 })
3401 .await
3402 .unwrap();
3403 store
3404 .update_status(id, crate::store::TaskStatus::InProgress)
3405 .await
3406 .unwrap();
3407 set_task_updated_at_past(&store, id).await;
3408
3409 tick_recover_stuck_tasks(
3410 &backend,
3411 &tmux,
3412 "owner/repo",
3413 &task_manager,
3414 &config,
3415 &store,
3416 &std::collections::HashMap::new(),
3417 &cooled_review_router_for_test(),
3418 &Arc::new(DashMap::new()),
3419 &Arc::new(DashSet::new()),
3420 )
3421 .await
3422 .unwrap();
3423
3424 let task = store.get(id).await.unwrap();
3425 assert_eq!(
3426 task.status,
3427 crate::store::TaskStatus::New,
3428 "cross-repo stale in_progress task should be reset to New"
3429 );
3430 assert_eq!(task.route_attempts, 0, "route attempts should be cleared");
3431 assert!(task.agent.is_none(), "agent should be cleared");
3432 assert!(task.model.is_none(), "model should be cleared");
3433
3434 assert!(
3436 status_updates.lock().unwrap().is_empty(),
3437 "cross-repo recovery should not call backend update_status"
3438 );
3439 }
3440
3441 #[tokio::test]
3442 async fn recover_stuck_tasks_finalizes_open_task_runs_rows() {
3443 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3444 let mock = MockBackend::new();
3445 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3446 let task_manager = Arc::new(TaskManager::with_store(
3447 backend.clone(),
3448 store.clone(),
3449 "owner/repo".to_string(),
3450 ));
3451 let tmux = Arc::new(TmuxManager::new());
3452 let config = EngineConfig {
3453 no_session_stuck_timeout: 600,
3454 stuck_timeout: 1800,
3455 ..EngineConfig::default()
3456 };
3457
3458 let id = store
3459 .create_internal("owner/repo", "Stuck with open run", "", "cron", "1", None)
3460 .await
3461 .unwrap();
3462 store
3463 .set_fields(
3464 id,
3465 &[
3466 ("agent", serde_json::json!("claude")),
3467 ("model", serde_json::json!("sonnet")),
3468 ("route_attempts", serde_json::json!(1)),
3469 ],
3470 )
3471 .await
3472 .unwrap();
3473 store
3474 .update_status(id, crate::store::TaskStatus::InProgress)
3475 .await
3476 .unwrap();
3477 set_task_updated_at_past(&store, id).await;
3478
3479 store
3481 .start_run(&crate::store::StartRun {
3482 task_id: id,
3483 attempt: 1,
3484 run_type: "agent",
3485 agent: "claude",
3486 model: "sonnet",
3487 command: "claude --model sonnet",
3488 prompt: "fix it",
3489 })
3490 .await
3491 .unwrap();
3492
3493 tick_recover_stuck_tasks(
3494 &backend,
3495 &tmux,
3496 "owner/repo",
3497 &task_manager,
3498 &config,
3499 &store,
3500 &std::collections::HashMap::new(),
3501 &cooled_review_router_for_test(),
3502 &Arc::new(DashMap::new()),
3503 &Arc::new(DashSet::new()),
3504 )
3505 .await
3506 .unwrap();
3507
3508 let task = store.get(id).await.unwrap();
3509 assert_eq!(task.status, crate::store::TaskStatus::New);
3510
3511 let runs = store.get_runs(id).await.unwrap();
3512 assert_eq!(runs.len(), 1);
3513 assert_eq!(runs[0].outcome, "aborted");
3514 assert!(runs[0].completed_at.is_some());
3515 assert_eq!(
3516 runs[0].error,
3517 "stuck-task recovery: internal no session found"
3518 );
3519 }
3520
3521 #[tokio::test]
3538 async fn recover_stuck_tasks_skips_reclaim_while_dispatch_guard_held() {
3539 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3540 let mock = MockBackend::new();
3541 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3542 let task_manager = Arc::new(TaskManager::with_store(
3543 backend.clone(),
3544 store.clone(),
3545 "owner/repo".to_string(),
3546 ));
3547 let tmux = Arc::new(TmuxManager::new());
3548 let config = EngineConfig {
3549 no_session_stuck_timeout: 600,
3550 stuck_timeout: 1800,
3551 ..EngineConfig::default()
3552 };
3553
3554 let id = store
3555 .create_internal(
3556 "owner/repo",
3557 "Long-running daily job",
3558 "",
3559 "cron",
3560 "1",
3561 None,
3562 )
3563 .await
3564 .unwrap();
3565 store
3566 .set_fields(
3567 id,
3568 &[
3569 ("agent", serde_json::json!("claude")),
3570 ("model", serde_json::json!("sonnet")),
3571 ("route_attempts", serde_json::json!(1)),
3572 ],
3573 )
3574 .await
3575 .unwrap();
3576 store
3577 .update_status(id, crate::store::TaskStatus::InProgress)
3578 .await
3579 .unwrap();
3580 set_task_updated_at_past(&store, id).await;
3583
3584 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3587 dispatching.insert(
3588 format!("owner/repo/internal:{id}"),
3589 format!("internal:{id}"),
3590 );
3591
3592 tick_recover_stuck_tasks(
3593 &backend,
3594 &tmux,
3595 "owner/repo",
3596 &task_manager,
3597 &config,
3598 &store,
3599 &std::collections::HashMap::new(),
3600 &cooled_review_router_for_test(),
3601 &dispatching,
3602 &Arc::new(DashSet::new()),
3603 )
3604 .await
3605 .unwrap();
3606
3607 let task = store.get(id).await.unwrap();
3608 assert_eq!(
3609 task.status,
3610 crate::store::TaskStatus::InProgress,
3611 "task should not be reclaimed while its dispatch guard is still held"
3612 );
3613
3614 let runs = store.get_runs(id).await.unwrap();
3616 assert!(runs.is_empty());
3617
3618 dispatching.clear();
3621 tick_recover_stuck_tasks(
3622 &backend,
3623 &tmux,
3624 "owner/repo",
3625 &task_manager,
3626 &config,
3627 &store,
3628 &std::collections::HashMap::new(),
3629 &cooled_review_router_for_test(),
3630 &dispatching,
3631 &Arc::new(DashSet::new()),
3632 )
3633 .await
3634 .unwrap();
3635 let task = store.get(id).await.unwrap();
3636 assert_eq!(
3637 task.status,
3638 crate::store::TaskStatus::New,
3639 "task should still be reclaimed once the dispatch guard is released"
3640 );
3641 }
3642
3643 #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
3659 async fn recover_stuck_tasks_never_reclaims_while_guard_held_under_real_concurrency() {
3660 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3661 let mock = MockBackend::new();
3662 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3663 let task_manager = Arc::new(TaskManager::with_store(
3664 backend.clone(),
3665 store.clone(),
3666 "owner/repo".to_string(),
3667 ));
3668 let tmux = Arc::new(TmuxManager::new());
3669 let config = EngineConfig {
3670 no_session_stuck_timeout: 600,
3671 stuck_timeout: 1800,
3672 ..EngineConfig::default()
3673 };
3674
3675 let id = store
3676 .create_internal(
3677 "owner/repo",
3678 "Long-running daily job",
3679 "",
3680 "cron",
3681 "1",
3682 None,
3683 )
3684 .await
3685 .unwrap();
3686 store
3687 .set_fields(
3688 id,
3689 &[
3690 ("agent", serde_json::json!("claude")),
3691 ("model", serde_json::json!("sonnet")),
3692 ("route_attempts", serde_json::json!(1)),
3693 ],
3694 )
3695 .await
3696 .unwrap();
3697 store
3698 .update_status(id, crate::store::TaskStatus::InProgress)
3699 .await
3700 .unwrap();
3701 set_task_updated_at_past(&store, id).await;
3702
3703 let dispatch_key = format!("owner/repo/internal:{id}");
3704 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3705
3706 let claim_task_id = format!("internal:{id}");
3713 {
3714 use dashmap::mapref::entry::Entry;
3715 match dispatching.entry(dispatch_key.clone()) {
3716 Entry::Vacant(slot) => {
3717 slot.insert(claim_task_id);
3718 }
3719 Entry::Occupied(_) => panic!("dispatch key already claimed"),
3720 }
3721 }
3722 let guard = DispatchGuard::new(Arc::clone(&dispatching), dispatch_key.clone());
3723
3724 let (release_tx, release_rx) = tokio::sync::oneshot::channel::<()>();
3735 let holder = tokio::spawn(async move {
3736 let _ = release_rx.await;
3737 drop(guard);
3738 });
3739
3740 const HAMMER_ITERATIONS: u32 = 300;
3748 for i in 0..HAMMER_ITERATIONS {
3749 tick_recover_stuck_tasks(
3750 &backend,
3751 &tmux,
3752 "owner/repo",
3753 &task_manager,
3754 &config,
3755 &store,
3756 &std::collections::HashMap::new(),
3757 &cooled_review_router_for_test(),
3758 &dispatching,
3759 &Arc::new(DashSet::new()),
3760 )
3761 .await
3762 .unwrap();
3763
3764 let task = store.get(id).await.unwrap();
3765 assert_eq!(
3766 task.status,
3767 crate::store::TaskStatus::InProgress,
3768 "task must never be reclaimed while the dispatch guard is deterministically \
3769 still held, even under real cross-thread concurrency (dispatch_key={dispatch_key}, \
3770 iteration={i}, contains_key_now={}, map_len_now={})",
3771 dispatching.contains_key(&dispatch_key),
3772 dispatching.len(),
3773 );
3774 tokio::task::yield_now().await;
3775 }
3776
3777 let _ = release_tx.send(());
3779 holder.await.unwrap();
3780
3781 tick_recover_stuck_tasks(
3783 &backend,
3784 &tmux,
3785 "owner/repo",
3786 &task_manager,
3787 &config,
3788 &store,
3789 &std::collections::HashMap::new(),
3790 &cooled_review_router_for_test(),
3791 &dispatching,
3792 &Arc::new(DashSet::new()),
3793 )
3794 .await
3795 .unwrap();
3796 let task = store.get(id).await.unwrap();
3797 assert_eq!(
3798 task.status,
3799 crate::store::TaskStatus::New,
3800 "task should be reclaimed once the dispatch guard is released"
3801 );
3802 }
3803
3804 #[tokio::test]
3805 async fn recover_stuck_tasks_skips_external_in_review_waiting_for_human_review() {
3806 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3807 let mock = MockBackend::new();
3808 let status_updates = mock.status_updates.clone();
3809 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3810 let task_manager = Arc::new(TaskManager::with_store(
3811 backend.clone(),
3812 store.clone(),
3813 "owner/repo".to_string(),
3814 ));
3815 let tmux = Arc::new(TmuxManager::new());
3816 let config = EngineConfig {
3817 no_session_stuck_timeout: 600,
3818 stuck_timeout: 1800,
3819 ..EngineConfig::default()
3820 };
3821
3822 let id = store
3823 .upsert_external(&crate::store::UpsertExternal {
3824 repo: "owner/repo",
3825 ext_id: "101",
3826 title: "Waiting For Human Review",
3827 body: "",
3828 author: "",
3829 url: "",
3830 labels: &[],
3831 origin: "github",
3832 })
3833 .await
3834 .unwrap();
3835 store
3836 .update_status(id, crate::store::TaskStatus::InReview)
3837 .await
3838 .unwrap();
3839 set_task_updated_at_past(&store, id).await;
3840
3841 tick_recover_stuck_tasks(
3842 &backend,
3843 &tmux,
3844 "owner/repo",
3845 &task_manager,
3846 &config,
3847 &store,
3848 &std::collections::HashMap::new(),
3849 &cooled_review_router_for_test(),
3850 &Arc::new(DashMap::new()),
3851 &Arc::new(DashSet::new()),
3852 )
3853 .await
3854 .unwrap();
3855
3856 let updates = status_updates.lock().unwrap();
3857 assert!(
3858 updates.is_empty(),
3859 "InReview task waiting for human review should not be reset"
3860 );
3861 }
3862
3863 #[tokio::test]
3864 async fn unblock_parents_no_blocked_tasks() {
3865 let mock = MockBackend::new(); let status_updates = mock.status_updates.clone();
3867 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3868 let task_manager = make_task_manager(backend.clone());
3869 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3870
3871 tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3872 .await
3873 .unwrap();
3874
3875 assert!(status_updates.lock().unwrap().is_empty());
3876 }
3877
3878 #[tokio::test]
3879 async fn unblock_parents_handles_multiple_blocked_tasks() {
3880 let mut mock = MockBackend::new();
3881
3882 let t40 = make_task("40", &["status:blocked"]);
3884 mock.blocked_tasks.push(t40);
3885 mock.sub_issues
3886 .insert("40".to_string(), vec![ExternalId("41".to_string())]);
3887 mock.tasks_by_id
3888 .insert("41".to_string(), make_task("41", &["status:done"]));
3889
3890 let t50 = make_task("50", &["status:blocked"]);
3892 mock.blocked_tasks.push(t50);
3893 mock.sub_issues
3894 .insert("50".to_string(), vec![ExternalId("51".to_string())]);
3895 mock.tasks_by_id
3896 .insert("51".to_string(), make_task("51", &["status:in_progress"]));
3897
3898 let status_updates = mock.status_updates.clone();
3899 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3900 let task_manager = make_task_manager(backend.clone());
3901 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3902
3903 tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3904 .await
3905 .unwrap();
3906
3907 let updates = status_updates.lock().unwrap();
3908 assert_eq!(updates.len(), 1, "only one parent should be unblocked");
3909 assert_eq!(updates[0].0, "40", "task 40 should be unblocked");
3910 assert_eq!(updates[0].1, Status::New);
3911 }
3912
3913 #[tokio::test]
3914 async fn detect_silent_agents_skips_backend_for_internal_tasks() {
3915 let transport = Arc::new(Transport::new());
3916 let capture = Arc::new(CaptureService::new(transport));
3917 let mock = Arc::new(MockBackend::new());
3918 let backend: Arc<dyn ExternalBackend> = mock.clone();
3919 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3920 let task_manager = Arc::new(TaskManager::with_store(
3921 backend.clone(),
3922 store.clone(),
3923 "owner/repo".to_string(),
3924 ));
3925 let tmux = Arc::new(TmuxManager::new());
3926 let config = EngineConfig {
3927 silence_grace_period: 0,
3928 ..EngineConfig::default()
3929 };
3930
3931 let internal_id = store
3932 .create_internal("owner/repo", "Silent internal", "", "cron", "1", None)
3933 .await
3934 .unwrap();
3935 store
3936 .update_status(internal_id, crate::store::TaskStatus::InProgress)
3937 .await
3938 .unwrap();
3939
3940 let task_id = format!("internal:{internal_id}");
3941 capture
3942 .register_session("owner/repo", &task_id, "orch-test-internal")
3943 .await;
3944 tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
3945
3946 tick_detect_silent_agents(
3947 &tmux,
3948 "owner/repo",
3949 &capture,
3950 &backend,
3951 &task_manager,
3952 &config,
3953 &store,
3954 )
3955 .await
3956 .unwrap();
3957
3958 assert!(
3959 mock.removed_labels.lock().unwrap().is_empty(),
3960 "internal tasks should skip backend label removal"
3961 );
3962 assert!(
3963 mock.posted_comments.lock().unwrap().is_empty(),
3964 "internal tasks should skip backend comments"
3965 );
3966
3967 let task = store.get(internal_id).await.unwrap();
3968 assert!(
3972 matches!(
3973 task.status,
3974 crate::store::TaskStatus::Routed | crate::store::TaskStatus::New
3975 ),
3976 "internal task should be routed to fallback or reset to new, got {:?}",
3977 task.status
3978 );
3979 }
3980
3981 #[tokio::test]
3982 async fn detect_silent_agents_finalizes_open_task_runs_rows() {
3983 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3984 let mock = MockBackend::new();
3985 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3986 let task_manager = Arc::new(TaskManager::with_store(
3987 backend.clone(),
3988 store.clone(),
3989 "owner/repo".to_string(),
3990 ));
3991 let transport = Arc::new(Transport::new());
3992 let capture = Arc::new(CaptureService::new(transport));
3993 let tmux = Arc::new(TmuxManager::new());
3994 let config = EngineConfig {
3995 silence_grace_period: 60,
3996 silence_cooldown: 300,
3997 ..EngineConfig::default()
3998 };
3999
4000 let internal_id = store
4001 .create_internal("owner/repo", "Silent with open run", "", "cron", "1", None)
4002 .await
4003 .unwrap();
4004 store
4005 .set_fields(
4006 internal_id,
4007 &[
4008 ("agent", serde_json::json!("claude")),
4009 ("model", serde_json::json!("sonnet")),
4010 ],
4011 )
4012 .await
4013 .unwrap();
4014 store
4015 .update_status(internal_id, crate::store::TaskStatus::InProgress)
4016 .await
4017 .unwrap();
4018
4019 let task_id = format!("internal:{internal_id}");
4020 capture
4021 .register_session("owner/repo", &task_id, "orch-test-silent-run")
4022 .await;
4023 capture
4024 .set_buffer_state_for_test(
4025 "owner/repo",
4026 &task_id,
4027 false,
4028 false,
4029 chrono::Utc::now() - chrono::Duration::seconds(120),
4030 )
4031 .await;
4032
4033 store
4035 .start_run(&crate::store::StartRun {
4036 task_id: internal_id,
4037 attempt: 1,
4038 run_type: "agent",
4039 agent: "claude",
4040 model: "sonnet",
4041 command: "claude --model sonnet",
4042 prompt: "fix it",
4043 })
4044 .await
4045 .unwrap();
4046
4047 tick_detect_silent_agents(
4048 &tmux,
4049 "owner/repo",
4050 &capture,
4051 &backend,
4052 &task_manager,
4053 &config,
4054 &store,
4055 )
4056 .await
4057 .unwrap();
4058
4059 let runs = store.get_runs(internal_id).await.unwrap();
4060 assert_eq!(runs.len(), 1);
4061 assert_eq!(runs[0].outcome, "aborted");
4062 assert!(runs[0].completed_at.is_some());
4063 assert_eq!(runs[0].error, "silence detection: session killed");
4064 }
4065
4066 #[tokio::test]
4077 async fn phase1_session_completion_touch_prevents_phase2_reclaim() {
4078 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4079 let mock = MockBackend::new();
4080 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4081 let task_manager = Arc::new(TaskManager::with_store(
4082 backend.clone(),
4083 store.clone(),
4084 "owner/repo".to_string(),
4085 ));
4086 let tmux = Arc::new(TmuxManager::new());
4087 let config = EngineConfig {
4088 no_session_stuck_timeout: 600,
4089 stuck_timeout: 1800,
4090 ..EngineConfig::default()
4091 };
4092
4093 let id = store
4097 .create_internal("owner/repo", "Completed task", "", "cron", "1", None)
4098 .await
4099 .unwrap();
4100 store
4101 .update_status(id, crate::store::TaskStatus::InProgress)
4102 .await
4103 .unwrap();
4104 set_task_updated_at_past(&store, id).await;
4105
4106 let task_id = format!("internal:{id}");
4109 store_touch_updated_at(&Some(Arc::clone(&store)), "owner/repo", &task_id).await;
4110
4111 tick_recover_stuck_tasks(
4113 &backend,
4114 &tmux,
4115 "owner/repo",
4116 &task_manager,
4117 &config,
4118 &store,
4119 &std::collections::HashMap::new(),
4120 &cooled_review_router_for_test(),
4121 &Arc::new(DashMap::new()),
4122 &Arc::new(DashSet::new()),
4123 )
4124 .await
4125 .unwrap();
4126
4127 let task = store.get(id).await.unwrap();
4128 assert_eq!(
4129 task.status,
4130 crate::store::TaskStatus::InProgress,
4131 "Phase 2 must not reclaim a task whose updated_at was just touched by Phase 1"
4132 );
4133 }
4134
4135 #[tokio::test]
4136 async fn stuck_task_timing_treats_dead_session_as_missing() {
4137 let tmux = Arc::new(TmuxManager::new());
4138 let repo = "owner/repo";
4139 let task_id = "internal:33498";
4140 let session_name = tmux.session_name(repo, task_id);
4141
4142 let create_result = tokio::process::Command::new("tmux")
4143 .args(["new-session", "-d", "-s", &session_name, "-c", "/tmp"])
4144 .output()
4145 .await;
4146
4147 match create_result {
4148 Ok(o) if o.status.success() => {}
4149 _ => {
4150 eprintln!("Skipping test: tmux not available or failed to create test session");
4151 return;
4152 }
4153 }
4154
4155 let _guard = SessionGuard(session_name.clone());
4156
4157 let set_option_result = tokio::process::Command::new("tmux")
4158 .args(["set-option", "-t", &session_name, "remain-on-exit", "on"])
4159 .output()
4160 .await;
4161 if !matches!(set_option_result, Ok(ref o) if o.status.success()) {
4162 eprintln!("Skipping test: unable to set tmux remain-on-exit option");
4163 return;
4164 }
4165
4166 let send_exit_result = tokio::process::Command::new("tmux")
4167 .args(["send-keys", "-t", &session_name, "exit", "Enter"])
4168 .output()
4169 .await;
4170 if !matches!(send_exit_result, Ok(ref o) if o.status.success()) {
4171 eprintln!("Skipping test: unable to exit tmux pane");
4172 return;
4173 }
4174
4175 for _ in 0..20 {
4177 tokio::time::sleep(std::time::Duration::from_millis(100)).await;
4178 if !tmux.session_is_running(&session_name).await {
4179 break;
4180 }
4181 }
4182
4183 let mut session_map = tmux.batch_session_active().await;
4188 for _ in 0..20 {
4189 if !session_map.get(&session_name).copied().unwrap_or(false) {
4190 break;
4191 }
4192 tokio::time::sleep(std::time::Duration::from_millis(100)).await;
4193 session_map = tmux.batch_session_active().await;
4194 }
4195 if session_map.get(&session_name).copied().unwrap_or(false) {
4196 eprintln!("Skipping test: session still alive in batch_session_active after retries (CI tmux lag)");
4197 return;
4198 }
4199
4200 let config = EngineConfig {
4201 no_session_stuck_timeout: 600,
4202 stuck_timeout: 1800,
4203 ..EngineConfig::default()
4204 };
4205 let updated_at = (chrono::Utc::now()
4206 - chrono::Duration::seconds(config.no_session_stuck_timeout as i64 + 5))
4207 .to_rfc3339();
4208
4209 let timing = stuck_task_timing_from_map(
4210 &tmux,
4211 repo,
4212 task_id,
4213 &updated_at,
4214 &config,
4215 "parse failure",
4216 &session_map,
4217 )
4218 .expect("dead session should be treated as missing");
4219
4220 assert!(!timing.has_session);
4221 assert_eq!(timing.threshold, config.no_session_stuck_timeout);
4222 }
4223
4224 #[tokio::test]
4225 #[serial_test::serial(suspend_state)]
4226 async fn stuck_task_timing_discounts_host_suspend_gap() {
4227 super::super::suspend::clear_for_test();
4228
4229 let tmux = Arc::new(TmuxManager::new());
4230 let repo = "owner/repo";
4231 let task_id = "internal:99001";
4232 let session_map = std::collections::HashMap::new();
4233
4234 let config = EngineConfig {
4235 no_session_stuck_timeout: 1800,
4236 stuck_timeout: 1800,
4237 ..EngineConfig::default()
4238 };
4239
4240 let updated_at = (chrono::Utc::now() - chrono::Duration::minutes(33)).to_rfc3339();
4244 let suspend_gap_detected_at = chrono::Utc::now() - chrono::Duration::minutes(1);
4245 super::super::suspend::inject_gap_for_test(
4246 suspend_gap_detected_at,
4247 chrono::Duration::minutes(30),
4248 );
4249
4250 let timing = stuck_task_timing_from_map(
4251 &tmux,
4252 repo,
4253 task_id,
4254 &updated_at,
4255 &config,
4256 "parse failure",
4257 &session_map,
4258 );
4259
4260 assert!(
4261 timing.is_none(),
4262 "task age should be discounted below the stuck threshold once suspend time is subtracted, got {timing:?}"
4263 );
4264
4265 super::super::suspend::clear_for_test();
4266 }
4267
4268 #[tokio::test]
4269 async fn recover_stuck_tasks_reclaims_routed_task_blocked_by_stale_session() {
4270 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4271 let mock = MockBackend::new();
4272 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4273 let task_manager = Arc::new(TaskManager::with_store(
4274 backend.clone(),
4275 store.clone(),
4276 "owner/repo".to_string(),
4277 ));
4278 let tmux = Arc::new(TmuxManager::new());
4279 let config = EngineConfig {
4280 no_session_stuck_timeout: 600,
4281 stuck_timeout: 1,
4282 ..EngineConfig::default()
4283 };
4284
4285 let id = store
4286 .create_internal("owner/repo", "Blocked routed task", "", "cron", "1", None)
4287 .await
4288 .unwrap();
4289 store
4290 .update_status(id, crate::store::TaskStatus::Routed)
4291 .await
4292 .unwrap();
4293 set_task_updated_at_past(&store, id).await;
4294
4295 let task_id = format!("internal:{id}");
4296 let session_name = tmux.session_name("owner/repo", &task_id);
4297 let create_result = tokio::process::Command::new("tmux")
4298 .args([
4299 "new-session",
4300 "-d",
4301 "-s",
4302 &session_name,
4303 "-c",
4304 "/tmp",
4305 "sleep 60",
4306 ])
4307 .output()
4308 .await;
4309
4310 match create_result {
4311 Ok(o) if o.status.success() => {}
4312 _ => {
4313 eprintln!("Skipping test: tmux not available or failed to create test session");
4314 return;
4315 }
4316 }
4317
4318 let session_map = tmux.batch_session_active().await;
4319 assert_eq!(session_map.get(&session_name).copied(), Some(true));
4320
4321 tick_recover_stuck_tasks(
4322 &backend,
4323 &tmux,
4324 "owner/repo",
4325 &task_manager,
4326 &config,
4327 &store,
4328 &session_map,
4329 &cooled_review_router_for_test(),
4330 &Arc::new(DashMap::new()),
4331 &Arc::new(DashSet::new()),
4332 )
4333 .await
4334 .unwrap();
4335
4336 let task = store.get(id).await.unwrap();
4337 assert_eq!(
4338 task.status,
4339 crate::store::TaskStatus::New,
4340 "routed task blocked by a stale session should be reset for re-routing"
4341 );
4342 assert!(
4343 !tmux.session_exists(&session_name).await,
4344 "recovery should kill the stale tmux session so dispatch can proceed"
4345 );
4346 }
4347
4348 #[tokio::test]
4362 async fn dispatch_does_not_deadlock_under_write_lock() {
4363 use crate::engine::router::Router;
4364 use tokio::sync::RwLock;
4365
4366 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4367 let mock = MockBackend::new();
4368 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4369 let task_manager = Arc::new(TaskManager::with_store(
4370 backend.clone(),
4371 store.clone(),
4372 "owner/repo".to_string(),
4373 ));
4374 let tmux = Arc::new(TmuxManager::new());
4375 let semaphore = Arc::new(Semaphore::new(4));
4376 let (weight_tx, _weight_rx) = mpsc::channel(16);
4377 let dispatching = Arc::new(DashMap::new());
4378 let transport = Arc::new(Transport::new());
4379 let capture = Arc::new(CaptureService::new(transport));
4380
4381 let router_config = crate::engine::router::RouterConfig::default();
4382 let router = Router::new(router_config);
4383 let router_arc = Arc::new(RwLock::new(router));
4384
4385 let write_guard = router_arc.write().await;
4389 let dispatch_mode = dispatch_mode_from_router(&write_guard);
4390
4391 let result = tokio::time::timeout(
4392 std::time::Duration::from_secs(2),
4393 tick_dispatch_tasks(
4394 &backend,
4395 &tmux,
4396 "owner/repo",
4397 &Arc::new(crate::engine::runner::TaskRunner::new(
4398 "owner/repo".to_string(),
4399 )),
4400 &capture,
4401 &semaphore,
4402 &task_manager,
4403 &weight_tx,
4404 dispatch_mode,
4405 &dispatching,
4406 &store,
4407 &std::collections::HashMap::new(),
4408 ),
4409 )
4410 .await;
4411
4412 assert!(
4413 result.is_ok(),
4414 "tick_dispatch_tasks deadlocked! It tried to acquire a read lock \
4415 on router_arc while a write lock was already held (issue #1361)"
4416 );
4417 }
4418
4419 #[tokio::test]
4427 async fn dispatch_skips_task_whose_routed_model_entered_cooldown() {
4428 use crate::store::{StoreRoute, TaskStatus as DbStatus, UpsertExternal};
4429
4430 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4431 let mock = MockBackend::new();
4432 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4433 let task_manager = Arc::new(TaskManager::with_store(
4434 backend.clone(),
4435 store.clone(),
4436 "owner/repo".to_string(),
4437 ));
4438 let tmux = Arc::new(TmuxManager::new());
4439 let runner = Arc::new(TaskRunner::new("owner/repo".to_string()));
4440 let semaphore = Arc::new(Semaphore::new(4));
4441 let (weight_tx, _weight_rx) = mpsc::channel(16);
4442 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4443 let transport = Arc::new(Transport::new());
4444 let capture = Arc::new(CaptureService::new(transport));
4445
4446 let id = store
4447 .upsert_external(&UpsertExternal {
4448 repo: "owner/repo",
4449 ext_id: "999",
4450 title: "Stale routed task",
4451 body: "",
4452 author: "user",
4453 url: "https://github.com/owner/repo/issues/999",
4454 labels: &[],
4455 origin: "github",
4456 })
4457 .await
4458 .unwrap();
4459 store
4460 .store_route(&StoreRoute {
4461 id,
4462 agent: "minimax",
4463 model: Some("opus"),
4464 complexity: "medium",
4465 estimate: 0,
4466 reason: "test",
4467 profile: "{}",
4468 skills: "[]",
4469 })
4470 .await
4471 .unwrap();
4472 store.update_status(id, DbStatus::Routed).await.unwrap();
4473
4474 crate::engine::cooldown::set_model_cooldown(
4477 "minimax",
4478 "opus",
4479 300,
4480 "rate_limit_usage_window",
4481 )
4482 .await;
4483
4484 let dispatch_mode = DispatchMode {
4485 is_degraded: false,
4486 healthy_agents: 1,
4487 threshold: 1,
4488 };
4489
4490 tick_dispatch_tasks(
4491 &backend,
4492 &tmux,
4493 "owner/repo",
4494 &runner,
4495 &capture,
4496 &semaphore,
4497 &task_manager,
4498 &weight_tx,
4499 dispatch_mode,
4500 &dispatching,
4501 &store,
4502 &std::collections::HashMap::new(),
4503 )
4504 .await
4505 .unwrap();
4506
4507 let task = store.get(id).await.unwrap();
4508 assert_eq!(
4509 task.status,
4510 DbStatus::Routed,
4511 "task whose routed model is now cooled down must stay Routed, not be dispatched"
4512 );
4513 assert!(
4514 dispatching.is_empty(),
4515 "dispatch guard must be released when skipping a stale-cooldown task"
4516 );
4517 }
4518
4519 #[tokio::test]
4520 async fn dispatch_skips_a_task_with_an_open_dependency() {
4521 use crate::store::{
4522 DepSource, NewDependency, StoreRoute, TaskStatus as DbStatus, UpsertExternal,
4523 };
4524
4525 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4526 let mock = MockBackend::new();
4527 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4528 let task_manager = Arc::new(TaskManager::with_store(
4529 backend.clone(),
4530 store.clone(),
4531 "owner/repo".to_string(),
4532 ));
4533 let tmux = Arc::new(TmuxManager::new());
4534 let runner = Arc::new(TaskRunner::new("owner/repo".to_string()));
4535 let semaphore = Arc::new(Semaphore::new(4));
4536 let (weight_tx, _weight_rx) = mpsc::channel(16);
4537 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4538 let transport = Arc::new(Transport::new());
4539 let capture = Arc::new(CaptureService::new(transport));
4540
4541 let id = store
4542 .upsert_external(&UpsertExternal {
4543 repo: "owner/repo",
4544 ext_id: "42",
4545 title: "Waiting routed task",
4546 body: "",
4547 author: "user",
4548 url: "https://github.com/owner/repo/issues/42",
4549 labels: &[],
4550 origin: "github",
4551 })
4552 .await
4553 .unwrap();
4554 store
4555 .replace_dependencies(
4556 id,
4557 &[NewDependency {
4558 repo: "owner/repo".into(),
4559 number: 7,
4560 source: DepSource::Github,
4561 }],
4562 )
4563 .await
4564 .unwrap();
4565 store
4566 .store_route(&StoreRoute {
4567 id,
4568 agent: "claude",
4569 model: Some("haiku"),
4570 complexity: "simple",
4571 estimate: 0,
4572 reason: "test",
4573 profile: "{}",
4574 skills: "[]",
4575 })
4576 .await
4577 .unwrap();
4578 store.update_status(id, DbStatus::Routed).await.unwrap();
4579
4580 let dispatch_mode = DispatchMode {
4581 is_degraded: false,
4582 healthy_agents: 1,
4583 threshold: 1,
4584 };
4585 tick_dispatch_tasks(
4586 &backend,
4587 &tmux,
4588 "owner/repo",
4589 &runner,
4590 &capture,
4591 &semaphore,
4592 &task_manager,
4593 &weight_tx,
4594 dispatch_mode,
4595 &dispatching,
4596 &store,
4597 &std::collections::HashMap::new(),
4598 )
4599 .await
4600 .unwrap();
4601
4602 let task = store.get(id).await.unwrap();
4603 assert_eq!(task.status, DbStatus::Waiting);
4604 assert!(dispatching.is_empty());
4605 }
4606
4607 #[tokio::test]
4620 async fn completed_session_does_not_cause_stuck_reclaim_race() {
4621 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4622 let mock = MockBackend::new();
4623 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4624 let task_manager = Arc::new(TaskManager::with_store(
4625 backend.clone(),
4626 store.clone(),
4627 "owner/repo".to_string(),
4628 ));
4629 let tmux = Arc::new(TmuxManager::new());
4630 let transport = Arc::new(Transport::new());
4631 let capture = Arc::new(CaptureService::new(transport));
4632
4633 let id = store
4635 .upsert_external(&crate::store::UpsertExternal {
4636 repo: "owner/repo",
4637 ext_id: "200",
4638 title: "Race Task",
4639 body: "",
4640 author: "",
4641 url: "",
4642 labels: &[],
4643 origin: "github",
4644 })
4645 .await
4646 .unwrap();
4647 store
4648 .update_status(id, crate::store::TaskStatus::InProgress)
4649 .await
4650 .unwrap();
4651
4652 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4654 .bind(id)
4655 .execute(store.pool())
4656 .await
4657 .unwrap();
4658
4659 let task_id = "200";
4661 let session_name = tmux.session_name("owner/repo", task_id);
4662 let create_result = tokio::process::Command::new("tmux")
4663 .args(["new-session", "-d", "-s", &session_name, "-c", "/tmp"])
4664 .output()
4665 .await;
4666 match create_result {
4667 Ok(o) if o.status.success() => {}
4668 _ => {
4669 eprintln!("Skipping test: tmux not available or failed to create test session");
4670 return;
4671 }
4672 }
4673 let _guard = SessionGuard(session_name.clone());
4674
4675 let _ = tokio::process::Command::new("tmux")
4677 .args(["set-option", "-t", &session_name, "remain-on-exit", "on"])
4678 .output()
4679 .await;
4680 let _ = tokio::process::Command::new("tmux")
4681 .args(["send-keys", "-t", &session_name, "exit", "Enter"])
4682 .output()
4683 .await;
4684
4685 for _ in 0..20 {
4687 tokio::time::sleep(std::time::Duration::from_millis(100)).await;
4688 if !tmux.session_is_running(&session_name).await {
4689 break;
4690 }
4691 }
4692
4693 let task_key = task_id.to_string();
4695 capture
4696 .register_session("owner/repo", &task_key, &session_name)
4697 .await;
4698
4699 let config = EngineConfig {
4701 no_session_stuck_timeout: 600,
4702 stuck_timeout: 1800,
4703 ..EngineConfig::default()
4704 };
4705
4706 tick_check_session_completions(&tmux, "owner/repo", &capture, &store)
4707 .await
4708 .unwrap();
4709
4710 {
4716 let task_state = store.get(id).await.unwrap();
4717 if task_state.updated_at.starts_with("2020") {
4718 eprintln!(
4719 "Skipping test: tick_check_session_completions did not update stored \
4720 updated_at — session was not in snapshot (CI tmux environment issue)"
4721 );
4722 return;
4723 }
4724 }
4725
4726 tick_recover_stuck_tasks(
4728 &backend,
4729 &tmux,
4730 "owner/repo",
4731 &task_manager,
4732 &config,
4733 &store,
4734 &std::collections::HashMap::new(),
4735 &cooled_review_router_for_test(),
4736 &Arc::new(DashMap::new()),
4737 &Arc::new(DashSet::new()),
4738 )
4739 .await
4740 .unwrap();
4741
4742 let task = store.get(id).await.unwrap();
4744 assert_eq!(
4745 task.status,
4746 crate::store::TaskStatus::InProgress,
4747 "task should not be reclaimed to New"
4748 );
4749 }
4750
4751 #[tokio::test]
4761 async fn session_completion_matches_repo_configured_with_git_suffix() {
4762 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4763 let tmux = Arc::new(TmuxManager::new());
4764 let transport = Arc::new(Transport::new());
4765 let capture = Arc::new(CaptureService::new(transport));
4766
4767 let repo = "owner/repo.git";
4768 let id = store
4769 .upsert_external(&crate::store::UpsertExternal {
4770 repo,
4771 ext_id: "201",
4772 title: "Git-suffix repo task",
4773 body: "",
4774 author: "",
4775 url: "",
4776 labels: &[],
4777 origin: "github",
4778 })
4779 .await
4780 .unwrap();
4781 store
4782 .update_status(id, crate::store::TaskStatus::InProgress)
4783 .await
4784 .unwrap();
4785 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4786 .bind(id)
4787 .execute(store.pool())
4788 .await
4789 .unwrap();
4790
4791 let task_id = "201";
4794 let session_name = tmux.session_name(repo, task_id);
4795 let create_result = tokio::process::Command::new("tmux")
4796 .args(["new-session", "-d", "-s", &session_name, "-c", "/tmp"])
4797 .output()
4798 .await;
4799 match create_result {
4800 Ok(o) if o.status.success() => {}
4801 _ => {
4802 eprintln!("Skipping test: tmux not available or failed to create test session");
4803 return;
4804 }
4805 }
4806 let _guard = SessionGuard(session_name.clone());
4807
4808 let _ = tokio::process::Command::new("tmux")
4809 .args(["set-option", "-t", &session_name, "remain-on-exit", "on"])
4810 .output()
4811 .await;
4812 let _ = tokio::process::Command::new("tmux")
4813 .args(["send-keys", "-t", &session_name, "exit", "Enter"])
4814 .output()
4815 .await;
4816
4817 for _ in 0..20 {
4818 tokio::time::sleep(std::time::Duration::from_millis(100)).await;
4819 if !tmux.session_is_running(&session_name).await {
4820 break;
4821 }
4822 }
4823
4824 capture.register_session(repo, task_id, &session_name).await;
4825
4826 tick_check_session_completions(&tmux, repo, &capture, &store)
4827 .await
4828 .unwrap();
4829
4830 let task_state = store.get(id).await.unwrap();
4831 if task_state.updated_at.starts_with("2020") {
4832 eprintln!(
4833 "Skipping test: tick_check_session_completions did not update stored \
4834 updated_at — session was not in snapshot (CI tmux environment issue)"
4835 );
4836 return;
4837 }
4838 assert!(
4839 !task_state.updated_at.starts_with("2020"),
4840 "tick_check_session_completions must match its own repo's live sessions \
4841 even when the repo is configured with a \".git\" suffix"
4842 );
4843 }
4844
4845 #[tokio::test]
4851 async fn job_scheduler_returns_ok_on_duplicate_job_config() {
4852 let tmp = tempfile::tempdir().unwrap();
4853 let cfg = tmp.path().join(".orch.yml");
4854 std::fs::write(
4855 &cfg,
4856 "jobs:\n - id: trading-scan\n schedule: \"0 9 * * *\"\n task:\n title: Scan\n body: do it\n",
4857 )
4858 .unwrap();
4859 let jobs_dir = tmp.path().join("prompts").join("jobs");
4860 std::fs::create_dir_all(&jobs_dir).unwrap();
4861 std::fs::write(
4862 jobs_dir.join("trading-scan.md"),
4863 "---\nid: trading-scan\nschedule: '0 9 * * *'\ntitle: Trading Scan\n---\n\nDo trading scan.\n",
4864 )
4865 .unwrap();
4866
4867 let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
4868 let result = tick_job_scheduler(&cfg, &backend, None, "test/repo", None).await;
4869 assert!(
4870 result.is_ok(),
4871 "duplicate job id must not propagate as Err: {result:?}"
4872 );
4873 }
4874
4875 #[tokio::test]
4879 async fn job_scheduler_processes_valid_project_when_other_has_duplicate() {
4880 let bad_tmp = tempfile::tempdir().unwrap();
4882 let bad_cfg = bad_tmp.path().join(".orch.yml");
4883 std::fs::write(
4884 &bad_cfg,
4885 "jobs:\n - id: dup\n schedule: \"0 9 * * *\"\n task:\n title: Dup\n body: x\n",
4886 )
4887 .unwrap();
4888 let bad_jobs_dir = bad_tmp.path().join("prompts").join("jobs");
4889 std::fs::create_dir_all(&bad_jobs_dir).unwrap();
4890 std::fs::write(
4891 bad_jobs_dir.join("dup.md"),
4892 "---\nid: dup\nschedule: '0 9 * * *'\ntitle: Dup\n---\n\nbody\n",
4893 )
4894 .unwrap();
4895
4896 let good_tmp = tempfile::tempdir().unwrap();
4898 let good_cfg = good_tmp.path().join(".orch.yml");
4899 std::fs::write(
4900 &good_cfg,
4901 "jobs:\n - id: valid-job\n schedule: \"0 9 * * *\"\n task:\n title: Valid\n body: ok\n",
4902 )
4903 .unwrap();
4904
4905 let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
4906
4907 let bad_result = tick_job_scheduler(&bad_cfg, &backend, None, "test/bad-repo", None).await;
4908 assert!(
4909 bad_result.is_ok(),
4910 "bad project must not abort scheduler: {bad_result:?}"
4911 );
4912
4913 let good_result =
4914 tick_job_scheduler(&good_cfg, &backend, None, "test/good-repo", None).await;
4915 assert!(
4916 good_result.is_ok(),
4917 "valid project must still run after bad project: {good_result:?}"
4918 );
4919 }
4920
4921 #[serial_test::serial(cooldown_state)]
4929 #[tokio::test]
4930 async fn global_sweep_routes_inactive_repo_tasks() {
4931 use crate::engine::router::{Router, RouterConfig};
4932 use crate::store::{TaskStatus, UpsertExternal};
4933
4934 crate::engine::cooldown::reset_global_state().await;
4935
4936 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4937
4938 let active_id = store
4940 .upsert_external(&UpsertExternal {
4941 repo: "active/repo",
4942 ext_id: "1",
4943 title: "Active task",
4944 body: "",
4945 author: "user",
4946 url: "https://github.com/active/repo/issues/1",
4947 labels: &[],
4948 origin: "github",
4949 })
4950 .await
4951 .unwrap();
4952
4953 let inactive_id = store
4955 .upsert_external(&UpsertExternal {
4956 repo: "inactive/repo",
4957 ext_id: "42",
4958 title: "Orphaned task",
4959 body: "",
4960 author: "user",
4961 url: "https://github.com/inactive/repo/issues/42",
4962 labels: &[],
4963 origin: "github",
4964 })
4965 .await
4966 .unwrap();
4967
4968 assert_eq!(store.get(active_id).await.unwrap().status, TaskStatus::New);
4969 assert_eq!(
4970 store.get(inactive_id).await.unwrap().status,
4971 TaskStatus::New
4972 );
4973
4974 let mut model_map: std::collections::HashMap<
4975 String,
4976 std::collections::HashMap<String, Vec<String>>,
4977 > = std::collections::HashMap::new();
4978 let mut tier = std::collections::HashMap::new();
4979 tier.insert("claude".to_string(), vec!["haiku".to_string()]);
4980 model_map.insert("medium".to_string(), tier);
4981 let config = RouterConfig {
4982 mode: "round_robin".to_string(),
4983 agents: vec!["claude".to_string()],
4984 model_map,
4985 ..Default::default()
4986 };
4987 let mut router = Router::new_for_test(config, vec!["claude".to_string()]);
4988
4989 let active_repos: std::collections::HashSet<String> =
4990 ["active/repo".to_string()].into_iter().collect();
4991 let mut routing_quota = RoutingTickQuota::new(1);
4992 route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
4993 .await
4994 .unwrap();
4995
4996 assert_eq!(
4998 store.get(active_id).await.unwrap().status,
4999 TaskStatus::New,
5000 "global sweep must not touch tasks for the active repo"
5001 );
5002
5003 let after = store.get(inactive_id).await.unwrap();
5005 assert_eq!(
5006 after.status,
5007 TaskStatus::Routed,
5008 "global sweep must route tasks from inactive repos"
5009 );
5010 assert!(
5011 matches!(&after.agent, Some(a) if !a.is_empty()),
5012 "agent must be set after routing"
5013 );
5014 }
5015
5016 #[serial_test::serial(cooldown_state)]
5024 #[tokio::test]
5025 async fn global_sweep_does_not_touch_other_active_repo_tasks() {
5026 use crate::engine::router::{Router, RouterConfig};
5027 use crate::store::{TaskStatus, UpsertExternal};
5028
5029 crate::engine::cooldown::reset_global_state().await;
5030
5031 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5032
5033 let other_active_id = store
5036 .upsert_external(&UpsertExternal {
5037 repo: "other-active/repo",
5038 ext_id: "7",
5039 title: "Other active project's task",
5040 body: "",
5041 author: "user",
5042 url: "https://github.com/other-active/repo/issues/7",
5043 labels: &[],
5044 origin: "github",
5045 })
5046 .await
5047 .unwrap();
5048
5049 let inactive_id = store
5051 .upsert_external(&UpsertExternal {
5052 repo: "inactive/repo",
5053 ext_id: "42",
5054 title: "Orphaned task",
5055 body: "",
5056 author: "user",
5057 url: "https://github.com/inactive/repo/issues/42",
5058 labels: &[],
5059 origin: "github",
5060 })
5061 .await
5062 .unwrap();
5063
5064 let mut model_map: std::collections::HashMap<
5065 String,
5066 std::collections::HashMap<String, Vec<String>>,
5067 > = std::collections::HashMap::new();
5068 let mut tier = std::collections::HashMap::new();
5069 tier.insert("claude".to_string(), vec!["haiku".to_string()]);
5070 model_map.insert("medium".to_string(), tier);
5071 let config = RouterConfig {
5072 mode: "round_robin".to_string(),
5073 agents: vec!["claude".to_string()],
5074 model_map,
5075 ..Default::default()
5076 };
5077 let mut router = Router::new_for_test(config, vec!["claude".to_string()]);
5078
5079 let active_repos: std::collections::HashSet<String> =
5083 ["active/repo".to_string(), "other-active/repo".to_string()]
5084 .into_iter()
5085 .collect();
5086 let mut routing_quota = RoutingTickQuota::new(1);
5087 route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
5088 .await
5089 .unwrap();
5090
5091 assert_eq!(
5092 store.get(other_active_id).await.unwrap().status,
5093 TaskStatus::New,
5094 "global sweep run from a different active repo's tick must not route \
5095 another active repo's new tasks — that repo's own tick will route them"
5096 );
5097 assert_eq!(
5098 store.get(inactive_id).await.unwrap().status,
5099 TaskStatus::Routed,
5100 "global sweep must still route tasks from genuinely orphaned repos"
5101 );
5102 }
5103
5104 #[serial_test::serial(cooldown_state)]
5106 #[tokio::test]
5107 async fn global_sweep_skips_no_agent_tasks() {
5108 use crate::engine::router::{Router, RouterConfig};
5109 use crate::store::{TaskStatus, UpsertExternal};
5110
5111 crate::engine::cooldown::reset_global_state().await;
5112
5113 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5114
5115 let id = store
5116 .upsert_external(&UpsertExternal {
5117 repo: "inactive/repo",
5118 ext_id: "99",
5119 title: "No-agent task",
5120 body: "",
5121 author: "user",
5122 url: "https://github.com/inactive/repo/issues/99",
5123 labels: &["no-agent".to_string()],
5124 origin: "github",
5125 })
5126 .await
5127 .unwrap();
5128
5129 let mut model_map: std::collections::HashMap<
5130 String,
5131 std::collections::HashMap<String, Vec<String>>,
5132 > = std::collections::HashMap::new();
5133 let mut tier = std::collections::HashMap::new();
5134 tier.insert("claude".to_string(), vec!["haiku".to_string()]);
5135 model_map.insert("medium".to_string(), tier);
5136 let config = RouterConfig {
5137 mode: "round_robin".to_string(),
5138 agents: vec!["claude".to_string()],
5139 model_map,
5140 ..Default::default()
5141 };
5142 let mut router = Router::new_for_test(config, vec!["claude".to_string()]);
5143
5144 let active_repos: std::collections::HashSet<String> =
5145 ["active/repo".to_string()].into_iter().collect();
5146 let mut routing_quota = RoutingTickQuota::new(1);
5147 route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
5148 .await
5149 .unwrap();
5150
5151 assert_eq!(
5152 store.get(id).await.unwrap().status,
5153 TaskStatus::New,
5154 "no-agent tasks must not be routed"
5155 );
5156 }
5157
5158 #[serial_test::serial(cooldown_state)]
5163 #[tokio::test]
5164 async fn global_sweep_honors_shared_routing_quota() {
5165 use crate::engine::router::{Router, RouterConfig};
5166 use crate::store::{TaskStatus, UpsertExternal};
5167
5168 crate::engine::cooldown::reset_global_state().await;
5169
5170 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5171 for ext_id in ["1", "2"] {
5172 store
5173 .upsert_external(&UpsertExternal {
5174 repo: "inactive/repo",
5175 ext_id,
5176 title: "Orphaned task",
5177 body: "",
5178 author: "user",
5179 url: &format!("https://github.com/inactive/repo/issues/{ext_id}"),
5180 labels: &[],
5181 origin: "github",
5182 })
5183 .await
5184 .unwrap();
5185 }
5186
5187 let mut model_map: std::collections::HashMap<
5188 String,
5189 std::collections::HashMap<String, Vec<String>>,
5190 > = std::collections::HashMap::new();
5191 let mut tier = std::collections::HashMap::new();
5192 tier.insert("claude".to_string(), vec!["haiku".to_string()]);
5193 model_map.insert("medium".to_string(), tier);
5194 let config = RouterConfig {
5195 mode: "round_robin".to_string(),
5196 agents: vec!["claude".to_string()],
5197 model_map,
5198 ..Default::default()
5199 };
5200 let mut router = Router::new_for_test(config, vec!["claude".to_string()]);
5201
5202 let active_repos: std::collections::HashSet<String> =
5203 ["active/repo".to_string()].into_iter().collect();
5204 let mut routing_quota = RoutingTickQuota::new(1);
5205 route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
5206 .await
5207 .unwrap();
5208
5209 let all_tasks = store.list_all_global().await.unwrap();
5210 let routed = all_tasks
5211 .iter()
5212 .filter(|task| task.status == TaskStatus::Routed)
5213 .count();
5214 let still_new = all_tasks
5215 .iter()
5216 .filter(|task| task.status == TaskStatus::New)
5217 .count();
5218
5219 assert_eq!(routed, 1, "only one task may consume the shared quota");
5220 assert_eq!(still_new, 1, "remaining tasks wait for the next tick");
5221
5222 route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
5223 .await
5224 .unwrap();
5225 let after_exhausted = store.list_all_global().await.unwrap();
5226 assert_eq!(
5227 after_exhausted
5228 .iter()
5229 .filter(|task| task.status == TaskStatus::Routed)
5230 .count(),
5231 1,
5232 "an exhausted shared quota must not reset within the same tick"
5233 );
5234 }
5235
5236 #[tokio::test]
5253 async fn group_orphaned_routed_tasks_selects_inactive_repo_tasks() {
5254 use crate::store::{StoreRoute, TaskStatus, UpsertExternal};
5255
5256 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5257
5258 async fn make_routed(store: &TaskStore, repo: &str, ext_id: &str) -> i64 {
5259 let id = store
5260 .upsert_external(&UpsertExternal {
5261 repo,
5262 ext_id,
5263 title: "Routed task",
5264 body: "",
5265 author: "user",
5266 url: &format!("https://github.com/{repo}/issues/{ext_id}"),
5267 labels: &[],
5268 origin: "github",
5269 })
5270 .await
5271 .unwrap();
5272 store
5273 .store_route(&StoreRoute {
5274 id,
5275 agent: "claude",
5276 model: Some("sonnet"),
5277 complexity: "medium",
5278 estimate: 0,
5279 reason: "test",
5280 profile: "{}",
5281 skills: "[]",
5282 })
5283 .await
5284 .unwrap();
5285 store.update_status(id, TaskStatus::Routed).await.unwrap();
5286 id
5287 }
5288
5289 let active_id = make_routed(&store, "active/repo", "1").await;
5290 let inactive_id_a = make_routed(&store, "inactive/repo", "2").await;
5291 let other_inactive_id = make_routed(&store, "other-inactive/repo", "3").await;
5292 let inactive_id_b = make_routed(&store, "inactive/repo", "4").await;
5293
5294 let all_routed = store
5295 .list_all_by_status_global(TaskStatus::Routed)
5296 .await
5297 .unwrap();
5298 let active_repos: std::collections::HashSet<String> =
5299 ["active/repo".to_string()].into_iter().collect();
5300 let by_repo = group_orphaned_routed_tasks(all_routed, &active_repos);
5301
5302 assert!(
5303 !by_repo.contains_key("active/repo"),
5304 "global sweep must not select tasks for the active repo"
5305 );
5306 let inactive_ids: std::collections::HashSet<i64> = by_repo
5307 .get("inactive/repo")
5308 .expect("inactive/repo must be selected")
5309 .iter()
5310 .map(|t| t.id)
5311 .collect();
5312 assert_eq!(
5313 inactive_ids,
5314 [inactive_id_a, inactive_id_b].into_iter().collect(),
5315 "global sweep must group all orphaned tasks for a repo together"
5316 );
5317 assert_eq!(
5318 by_repo
5319 .get("other-inactive/repo")
5320 .map(|tasks| tasks.iter().map(|t| t.id).collect::<Vec<_>>()),
5321 Some(vec![other_inactive_id])
5322 );
5323
5324 assert_eq!(
5327 store.get(active_id).await.unwrap().status,
5328 TaskStatus::Routed
5329 );
5330 }
5331
5332 #[tokio::test]
5337 async fn group_orphaned_routed_tasks_excludes_all_active_repos_not_just_current() {
5338 use crate::store::{StoreRoute, TaskStatus, UpsertExternal};
5339
5340 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5341
5342 async fn make_routed(store: &TaskStore, repo: &str, ext_id: &str) -> i64 {
5343 let id = store
5344 .upsert_external(&UpsertExternal {
5345 repo,
5346 ext_id,
5347 title: "Routed task",
5348 body: "",
5349 author: "user",
5350 url: &format!("https://github.com/{repo}/issues/{ext_id}"),
5351 labels: &[],
5352 origin: "github",
5353 })
5354 .await
5355 .unwrap();
5356 store
5357 .store_route(&StoreRoute {
5358 id,
5359 agent: "claude",
5360 model: Some("sonnet"),
5361 complexity: "medium",
5362 estimate: 0,
5363 reason: "test",
5364 profile: "{}",
5365 skills: "[]",
5366 })
5367 .await
5368 .unwrap();
5369 store.update_status(id, TaskStatus::Routed).await.unwrap();
5370 id
5371 }
5372
5373 let repo_a_id = make_routed(&store, "active/repo-a", "1").await;
5374 let repo_b_id = make_routed(&store, "active/repo-b", "2").await;
5375 let orphaned_id = make_routed(&store, "inactive/repo", "3").await;
5376
5377 let all_routed = store
5378 .list_all_by_status_global(TaskStatus::Routed)
5379 .await
5380 .unwrap();
5381 let active_repos: std::collections::HashSet<String> =
5384 ["active/repo-a".to_string(), "active/repo-b".to_string()]
5385 .into_iter()
5386 .collect();
5387 let by_repo = group_orphaned_routed_tasks(all_routed, &active_repos);
5388
5389 assert!(
5390 !by_repo.contains_key("active/repo-a"),
5391 "the current repo's own tasks must not be swept"
5392 );
5393 assert!(
5394 !by_repo.contains_key("active/repo-b"),
5395 "another active repo's tasks must not be swept during repo A's tick"
5396 );
5397 assert_eq!(
5398 by_repo
5399 .get("inactive/repo")
5400 .map(|tasks| tasks.iter().map(|t| t.id).collect::<Vec<_>>()),
5401 Some(vec![orphaned_id]),
5402 "a genuinely inactive repo's tasks must still be swept"
5403 );
5404
5405 assert_eq!(
5407 store.get(repo_a_id).await.unwrap().status,
5408 TaskStatus::Routed
5409 );
5410 assert_eq!(
5411 store.get(repo_b_id).await.unwrap().status,
5412 TaskStatus::Routed
5413 );
5414 }
5415
5416 #[tokio::test]
5418 async fn global_dispatch_sweep_skips_no_agent_tasks() {
5419 use crate::store::{StoreRoute, TaskStatus, UpsertExternal};
5420
5421 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5422
5423 let id = store
5424 .upsert_external(&UpsertExternal {
5425 repo: "inactive/repo",
5426 ext_id: "99",
5427 title: "No-agent routed task",
5428 body: "",
5429 author: "user",
5430 url: "https://github.com/inactive/repo/issues/99",
5431 labels: &["no-agent".to_string()],
5432 origin: "github",
5433 })
5434 .await
5435 .unwrap();
5436 store
5437 .store_route(&StoreRoute {
5438 id,
5439 agent: "claude",
5440 model: Some("sonnet"),
5441 complexity: "medium",
5442 estimate: 0,
5443 reason: "test",
5444 profile: "{}",
5445 skills: "[]",
5446 })
5447 .await
5448 .unwrap();
5449 store.update_status(id, TaskStatus::Routed).await.unwrap();
5450
5451 let tmux = Arc::new(TmuxManager::new());
5456 let transport = Arc::new(Transport::new());
5457 let capture = Arc::new(CaptureService::new(transport));
5458 let semaphore = Arc::new(Semaphore::new(4));
5459 let (weight_tx, _weight_rx) = mpsc::channel(16);
5460 let dispatching = Arc::new(DashMap::new());
5461 let dispatch_mode = DispatchMode {
5462 is_degraded: false,
5463 healthy_agents: 1,
5464 threshold: 1,
5465 };
5466
5467 let active_repos: std::collections::HashSet<String> =
5468 ["active/repo".to_string()].into_iter().collect();
5469 dispatch_routed_tasks_global(
5470 &tmux,
5471 &capture,
5472 &semaphore,
5473 &weight_tx,
5474 dispatch_mode,
5475 &dispatching,
5476 &store,
5477 &std::collections::HashMap::new(),
5478 &active_repos,
5479 )
5480 .await
5481 .unwrap();
5482
5483 assert_eq!(
5484 store.get(id).await.unwrap().status,
5485 TaskStatus::Routed,
5486 "no-agent tasks must not be dispatched"
5487 );
5488 }
5489}