1use crate::backends::{ExternalBackend, ExternalId, ExternalTask, Status};
13use crate::channels::capture::CaptureService;
14use crate::config;
15use crate::engine::cooldown::{
16 github_circuit_remaining_secs, is_agent_in_cooldown, is_github_circuit_open,
17 is_model_in_cooldown, record_agent_failure_with_message, record_silence_detection,
18 set_agent_cooldown, set_model_cooldown, SILENCE_AGENT_COOLDOWN_SECS,
19 SILENCE_EXTENDED_COOLDOWN_SECS,
20};
21use crate::engine::dispatch_guard::DispatchGuard;
22use crate::engine::jobs;
23use crate::engine::router::{get_route_result, AllAgentsCooledError, RouteResultError, Router};
24use crate::engine::runner::{TaskRunner, WeightSignal};
25use crate::engine::tasks::{is_internal_id, TaskManager};
26use crate::repo_context::REPO_CONTEXT;
27use crate::store::TaskStore;
28use crate::tmux::TmuxManager;
29use dashmap::{DashMap, DashSet};
30use std::sync::atomic::{AtomicBool, Ordering};
31use std::sync::Arc;
32use std::sync::LazyLock;
33use std::time::{Instant, SystemTime};
34use tokio::sync::{mpsc, Semaphore};
35
36static ENGINE_START_TIME: LazyLock<u64> = LazyLock::new(|| {
39 SystemTime::now()
40 .duration_since(SystemTime::UNIX_EPOCH)
41 .unwrap_or_default()
42 .as_secs()
43});
44
45static STARTUP_CLEANUP_DONE: AtomicBool = AtomicBool::new(false);
47
48pub(crate) struct RoutingTickQuota {
55 remaining: usize,
56}
57
58impl RoutingTickQuota {
59 pub(crate) fn new(max: usize) -> Self {
60 Self { remaining: max }
61 }
62
63 fn is_exhausted(&self) -> bool {
64 self.remaining == 0
65 }
66
67 fn remaining(&self) -> usize {
68 self.remaining
69 }
70
71 fn consume_one(&mut self) -> bool {
72 if self.remaining == 0 {
73 return false;
74 }
75 self.remaining -= 1;
76 true
77 }
78}
79
80async fn startup_cleanup(tmux: &TmuxManager) {
83 if STARTUP_CLEANUP_DONE
85 .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
86 .is_err()
87 {
88 return;
89 }
90
91 let start_time = *ENGINE_START_TIME;
92 tracing::info!(
93 start_time,
94 "running startup cleanup for stale tmux sessions"
95 );
96
97 match tmux.kill_stale_sessions(start_time).await {
98 Ok(killed) => {
99 if killed > 0 {
100 tracing::info!(killed, "startup cleanup killed stale sessions");
101 } else {
102 tracing::debug!("no stale tmux sessions found on startup");
103 }
104 }
105 Err(e) => {
106 tracing::warn!(error = %e, "failed to run startup cleanup");
107 }
108 }
109}
110
111use super::EngineConfig;
112use crate::store::{set_review_session_expected, store_set_by_id, store_touch_updated_at};
113
114async fn recover_routed_blocked_dispatch(
115 backend: &Arc<dyn ExternalBackend>,
116 store: &Arc<TaskStore>,
117 task_manager: &Arc<TaskManager>,
118 repo: &str,
119 task: &ExternalTask,
120 timing: &StuckTaskTiming,
121) {
122 tracing::warn!(
123 task_id = task.id.0,
124 age_mins = timing.age.num_minutes(),
125 threshold_mins = timing.threshold / 60,
126 "recovering routed task: dispatch blocked by stale tmux session → new"
127 );
128
129 let resolved_store_id = match store.resolve_task_id(repo, &task.id.0).await {
130 Ok(Some(id)) => Some(id),
131 Ok(None) => {
132 tracing::warn!(
133 task_id = %task.id.0,
134 repo,
135 "resolve_task_id returned None during routed-task recovery"
136 );
137 None
138 }
139 Err(e) => {
140 tracing::error!(
141 task_id = %task.id.0,
142 repo,
143 error = %e,
144 "resolve_task_id failed during routed-task recovery"
145 );
146 None
147 }
148 };
149
150 if let Some(store_id) = resolved_store_id {
151 if let Err(e) = store_set_by_id(
152 &Some(store),
153 store_id,
154 &[
155 ("agent", serde_json::Value::Null),
156 ("model", serde_json::Value::Null),
157 ("route_attempts", serde_json::json!(0)),
158 ],
159 )
160 .await
161 {
162 tracing::warn!(
163 task_id = task.id.0,
164 ?e,
165 "failed to clear routing fields for routed stuck task — will retry next tick"
166 );
167 return;
168 }
169 finalize_open_runs_for_recovery(
170 store,
171 store_id,
172 "stuck-task recovery: stale session blocking dispatch killed",
173 )
174 .await;
175 }
176
177 if let Err(e) = task_manager.update_task_status(&task.id, Status::New).await {
178 tracing::warn!(
179 task_id = task.id.0,
180 ?e,
181 "failed to reset routed stuck task status"
182 );
183 return;
184 }
185
186 crate::store::store_log_activity(
187 &Some(Arc::clone(store)),
188 repo,
189 &task.id.0,
190 "rerouted",
191 Some("routed"),
192 Some("new"),
193 None::<&str>,
194 None::<&str>,
195 Some(&serde_json::json!({
196 "failure_reason": "routed_blocked_by_existing_session",
197 "age_minutes": timing.age.num_minutes(),
198 "had_session": true,
199 })),
200 )
201 .await;
202
203 if !is_internal_id(&task.id.0) {
204 let backend_clone = Arc::clone(backend);
205 let task_id_clone = task.id.clone();
206 let body = format!(
207 "[{}] recovered: stuck routed task — existing tmux session blocked dispatch for {}m; cleared route for retry{}",
208 chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
209 timing.age.num_minutes(),
210 crate::engine::orch_footer()
211 );
212 tokio::spawn(async move {
213 if let Err(e) = backend_clone.post_comment(&task_id_clone, &body).await {
214 tracing::warn!(
215 task_id = task_id_clone.0,
216 ?e,
217 "failed to post routed-task recovery comment"
218 );
219 }
220 });
221 }
222}
223
224pub(crate) async fn tick_check_session_completions(
226 tmux: &Arc<TmuxManager>,
227 repo: &str,
228 capture: &Arc<CaptureService>,
229 store: &Arc<TaskStore>,
230) -> anyhow::Result<()> {
231 let _span = tracing::info_span!("engine.tick.phase1.sessions").entered();
232 let session_snapshot = tmux.snapshot().await;
233 let repo_name = repo
239 .rsplit('/')
240 .next()
241 .unwrap_or(repo)
242 .trim_end_matches(".git");
243 for (session, active) in session_snapshot {
244 if session.project != repo_name {
246 continue;
247 }
248
249 if !active {
250 tracing::info!(
251 session = %session.name,
252 task_id = %session.task_id,
253 "session completed, collecting results"
254 );
255 let store_task_id = {
266 let base = session
267 .task_id
268 .strip_suffix("-review")
269 .unwrap_or(&session.task_id);
270 if let Some(n) = base.strip_prefix("internal-") {
271 format!("internal:{n}")
272 } else {
273 base.to_string()
274 }
275 };
276 store_touch_updated_at(&Some(Arc::clone(store)), repo, &store_task_id).await;
277 capture.unregister_session(repo, &session.task_id).await;
280 if let Err(e) = tmux.kill_session(&session.name).await {
282 tracing::debug!(
283 session = %session.name,
284 ?e,
285 "kill_session failed (session may already be gone)"
286 );
287 }
288 }
289 }
290 Ok(())
291}
292
293#[derive(Debug)]
294struct StuckTaskTiming {
295 has_session: bool,
296 age: chrono::Duration,
297 threshold: u64,
298}
299
300#[allow(clippy::too_many_arguments)]
301fn stuck_task_timing_from_map(
302 tmux: &Arc<TmuxManager>,
303 repo: &str,
304 session_task_id: &str,
305 updated_at: &str,
306 config: &EngineConfig,
307 parse_error_message: &'static str,
308 session_map: &std::collections::HashMap<String, bool>,
309) -> Option<StuckTaskTiming> {
310 let session_name = tmux.session_name(repo, session_task_id);
311 let has_session = session_map.get(&session_name).copied().unwrap_or(false);
313 let threshold = if has_session {
314 config.stuck_timeout
315 } else {
316 config.no_session_stuck_timeout
317 };
318
319 let updated = match chrono::DateTime::parse_from_rfc3339(updated_at) {
320 Ok(dt) => dt.with_timezone(&chrono::Utc),
321 Err(e) => {
322 tracing::warn!(
323 task_id = session_task_id,
324 updated_at,
325 ?e,
326 error_message = parse_error_message,
327 "cannot parse updated_at"
328 );
329 return None;
330 }
331 };
332
333 let raw_age = chrono::Utc::now() - updated;
334 let suspended = super::suspend::suspended_duration_since(updated);
338 let age = (raw_age - suspended).max(chrono::Duration::zero());
339 if age.num_seconds() <= threshold as i64 {
340 return None;
341 }
342
343 Some(StuckTaskTiming {
344 has_session,
345 age,
346 threshold,
347 })
348}
349
350async fn finalize_open_runs_for_recovery(store: &TaskStore, store_id: i64, reason: &str) {
353 if let Err(e) = store
354 .finalize_incomplete_runs(store_id, "aborted", reason)
355 .await
356 {
357 tracing::warn!(
358 store_id,
359 error = %e,
360 reason,
361 "failed to finalize incomplete runs during recovery"
362 );
363 }
364}
365
366pub(crate) async fn tick_detect_silent_agents(
375 tmux: &Arc<TmuxManager>,
376 repo: &str,
377 capture: &Arc<CaptureService>,
378 backend: &Arc<dyn ExternalBackend>,
379 task_manager: &Arc<TaskManager>,
380 config: &EngineConfig,
381 store: &Arc<TaskStore>,
382) -> anyhow::Result<()> {
383 let _span = tracing::info_span!("engine.tick.phase1b.silence").entered();
384 let grace = std::time::Duration::from_secs(config.silence_grace_period);
385 let silent_sessions = capture.get_silent_sessions_for_repo(repo, grace).await;
386
387 for (task_id, session_name, silence_age_secs) in silent_sessions {
388 let use_backend = should_use_backend(&task_id);
389 let store_task = match store.resolve_task_id(repo, &task_id).await {
391 Ok(Some(store_id)) => match store.get(store_id).await {
392 Ok(task) => Some(task),
393 Err(e) => {
394 tracing::warn!(
395 task_id,
396 repo,
397 store_id,
398 operation = "store.get",
399 error = %e,
400 "silence detection: db error fetching task — agent/model cooldown will be skipped"
401 );
402 None
403 }
404 },
405 Ok(None) => {
406 tracing::debug!(
407 task_id,
408 repo,
409 operation = "resolve_task_id",
410 "silence detection: task not found in store (may have been cleaned up)"
411 );
412 None
413 }
414 Err(e) => {
415 tracing::warn!(
416 task_id,
417 repo,
418 operation = "resolve_task_id",
419 error = %e,
420 "silence detection: db error resolving task id — agent/model cooldown will be skipped"
421 );
422 None
423 }
424 };
425 let agent_name = store_task
426 .as_ref()
427 .and_then(|t| t.agent.clone())
428 .unwrap_or_default();
429 let model_name = store_task
430 .as_ref()
431 .and_then(|t| t.model.clone())
432 .unwrap_or_default();
433
434 tracing::warn!(
435 task_id,
436 agent = %agent_name,
437 model = %model_name,
438 silence_age_secs,
439 grace_secs = config.silence_grace_period,
440 cooldown_secs = config.silence_cooldown,
441 "agent silent since session start — killing session, cooling down model + agent, failing over"
442 );
443
444 if let Err(e) = tmux.kill_session(&session_name).await {
446 tracing::debug!(
447 task_id,
448 ?e,
449 "kill_session failed for silent agent (may already be gone)"
450 );
451 }
452
453 capture.unregister_session(repo, &task_id).await;
455
456 if let Some(store_id) = store_task.as_ref().map(|t| t.id) {
458 finalize_open_runs_for_recovery(store, store_id, "silence detection: session killed")
459 .await;
460 }
461
462 let mut extended_note = String::new();
463
464 if !agent_name.is_empty() && !model_name.is_empty() {
466 set_model_cooldown(
467 &agent_name,
468 &model_name,
469 config.silence_cooldown,
470 "silence_detected",
471 )
472 .await;
473 if let Some(result) = record_silence_detection(&agent_name, &model_name).await {
474 if result.extended_cooldown_applied {
475 extended_note = format!(
476 " ({} silences in 24h -> extended cooldown {}s)",
477 result.count, SILENCE_EXTENDED_COOLDOWN_SECS
478 );
479 }
480 }
481 }
482
483 if !agent_name.is_empty() {
487 set_agent_cooldown(&agent_name, SILENCE_AGENT_COOLDOWN_SECS).await;
488 }
489
490 let task_eid = ExternalId(task_id.clone());
495
496 let available: Vec<String> = crate::engine::configured_agents()
498 .into_iter()
499 .filter(|a| crate::cmd_cache::command_exists(a))
500 .collect();
501 let chain = crate::engine::runner::response::get_reroute_chain(
502 &task_id,
503 &Some(Arc::clone(store)),
504 repo,
505 )
506 .await;
507 let chain = crate::engine::runner::response::update_reroute_chain(
508 &task_id,
509 &agent_name,
510 &chain,
511 &Some(Arc::clone(store)),
512 repo,
513 )
514 .await;
515
516 let (next_status, next_agent) = if let Some(fallback) =
517 crate::engine::runner::response::pick_fallback_agent(&agent_name, &chain, &available)
518 {
519 tracing::info!(
520 task_id,
521 from = %agent_name,
522 to = %fallback,
523 "silence detection: failover to different agent, setting routed"
524 );
525 (Status::Routed, Some(fallback))
526 } else {
527 tracing::warn!(
528 task_id,
529 "silence detection: no fallback agents available, resetting to new for re-routing"
530 );
531 (Status::New, None)
532 };
533
534 if use_backend {
535 if let Some(ref st) = store_task {
536 let stale_labels: Vec<String> = st
538 .labels
539 .iter()
540 .filter(|l| {
541 l.starts_with("agent:")
542 || l.starts_with("complexity:")
543 || l.starts_with("model:")
544 })
545 .cloned()
546 .collect();
547 if !stale_labels.is_empty() {
548 let backend_clone = Arc::clone(backend);
549 let task_eid_clone = task_eid.clone();
550 tokio::spawn(async move {
551 for label in &stale_labels {
552 if let Err(e) = backend_clone.remove_label(&task_eid_clone, label).await
553 {
554 tracing::warn!(task_id = task_eid_clone.0, label, error = %e, "failed to remove label during silence detection re-route");
555 }
556 }
557 });
558 }
559 }
560 }
561
562 if let Some(store_id) = store_task.as_ref().map(|t| t.id) {
563 if let Some(ref fallback) = next_agent {
564 let _ = store_set_by_id(
568 &Some(store),
569 store_id,
570 &[
571 ("agent", serde_json::json!(fallback)),
572 ("model", serde_json::json!("")),
573 (
574 "last_error",
575 serde_json::json!(format!(
576 "silence detected after {}s, failing over to {}",
577 config.silence_grace_period, fallback
578 )),
579 ),
580 ],
581 )
582 .await;
583 } else {
584 let _ = store_set_by_id(
585 &Some(store),
586 store_id,
587 &[
588 ("agent", serde_json::Value::Null),
589 ("model", serde_json::Value::Null),
590 (
591 "last_error",
592 serde_json::json!(format!(
593 "silence detected after {}s, no fallback agents available",
594 config.silence_grace_period
595 )),
596 ),
597 ],
598 )
599 .await;
600 }
601 }
602
603 if let Err(e) = task_manager
604 .update_task_status(&task_eid, next_status)
605 .await
606 {
607 tracing::warn!(task_id, ?e, "failed to update silent task status");
608 continue;
609 }
610
611 let details = serde_json::json!({
613 "failure_reason": "agent_silence",
614 "silence_duration_secs": config.silence_grace_period,
615 "cooldown_applied": !agent_name.is_empty() && !model_name.is_empty(),
616 "fallback_available": next_agent.is_some(),
617 });
618 crate::store::store_log_activity(
619 &Some(Arc::clone(store)),
620 repo,
621 &task_id,
622 "rerouted",
623 Some("in_progress"),
624 Some(next_status.as_str()),
625 next_agent.as_deref(),
626 None,
627 Some(&details),
628 )
629 .await;
630
631 let action = if let Some(ref fallback) = next_agent {
633 format!("failing over to {fallback}")
634 } else {
635 "marking needs_review (no fallback agents)".to_string()
636 };
637 let comment = format!(
638 "[{}] agent silent for {}s since session start — killed session, cooled down model `{}:{}` for {}s{}, agent `{}` for {}s, {}{}",
639 chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
640 config.silence_grace_period,
641 agent_name,
642 model_name,
643 config.silence_cooldown,
644 extended_note,
645 agent_name,
646 SILENCE_AGENT_COOLDOWN_SECS,
647 action,
648 crate::engine::orch_footer(),
649 );
650 if use_backend {
651 let backend_clone = Arc::clone(backend);
653 let task_eid_clone = task_eid.clone();
654 tokio::spawn(async move {
655 if let Err(e) = backend_clone.post_comment(&task_eid_clone, &comment).await {
656 tracing::warn!(
657 task_id = task_eid_clone.0,
658 ?e,
659 "failed to post silence detection comment"
660 );
661 }
662 });
663 }
664 }
665
666 Ok(())
667}
668
669fn should_use_backend(task_id: &str) -> bool {
670 !is_internal_id(task_id)
671}
672
673#[allow(clippy::too_many_arguments)]
678pub(crate) async fn tick_recover_stuck_tasks(
679 backend: &Arc<dyn ExternalBackend>,
680 tmux: &Arc<TmuxManager>,
681 repo: &str,
682 task_manager: &Arc<TaskManager>,
683 config: &EngineConfig,
684 store: &Arc<TaskStore>,
685 session_map: &std::collections::HashMap<String, bool>,
686 router: &Router,
687 dispatching: &Arc<DashMap<String, String>>,
688 auto_merge_in_flight: &Arc<DashSet<String>>,
689) -> anyhow::Result<()> {
690 let _span = tracing::info_span!("engine.tick.phase2.stuck_tasks").entered();
691
692 let in_progress = match task_manager
693 .list_external_by_status(Status::InProgress)
694 .await
695 {
696 Ok(tasks) => tasks,
697 Err(e) => {
698 tracing::warn!(?e, "failed to list in_progress tasks for stuck recovery");
699 vec![]
700 }
701 };
702 for task in &in_progress {
703 let Some(timing) = stuck_task_timing_from_map(
704 tmux,
705 repo,
706 &task.id.0,
707 &task.updated_at,
708 config,
709 "cannot parse updated_at, skipping stuck-task check",
710 session_map,
711 ) else {
712 continue;
713 };
714
715 if !timing.has_session {
722 let dispatch_key = format!("{repo}/{}", task.id.0);
723 if dispatching.contains_key(&dispatch_key) {
724 tracing::info!(
732 task_id = task.id.0,
733 dispatch_key,
734 age_mins = timing.age.num_minutes(),
735 "skipping stuck-task reclaim: runner still actively processing this task"
736 );
737 continue;
738 }
739 }
740
741 if timing.has_session {
742 tracing::warn!(
743 task_id = task.id.0,
744 age_mins = timing.age.num_minutes(),
745 threshold_mins = timing.threshold / 60,
746 "recovering stuck task: timed out with active session → new"
747 );
748 } else {
749 tracing::warn!(
750 task_id = task.id.0,
751 dispatch_key = format!("{repo}/{}", task.id.0),
752 age_mins = timing.age.num_minutes(),
753 threshold_mins = timing.threshold / 60,
754 "recovering stuck task: no session found — reclaiming early → new"
755 );
756 }
757 let mut cached_store_id: Option<i64> = None;
762 if timing.has_session {
763 let store_task = match store.resolve_task_id(repo, &task.id.0).await {
764 Ok(Some(store_id)) => {
765 cached_store_id = Some(store_id);
766 match store.get(store_id).await {
767 Ok(t) => Some(t),
768 Err(e) => {
769 tracing::warn!(task_id = task.id.0, error = %e, "failed to fetch task from store for stuck-task cooldown");
770 None
771 }
772 }
773 }
774 Ok(None) => None,
775 Err(e) => {
776 tracing::warn!(task_id = task.id.0, error = %e, "failed to resolve task id for stuck-task cooldown");
777 None
778 }
779 };
780 let agent_name = store_task
781 .as_ref()
782 .and_then(|t| t.agent.clone())
783 .unwrap_or_default();
784 let model_name = store_task
785 .as_ref()
786 .and_then(|t| t.model.clone())
787 .unwrap_or_default();
788
789 if !agent_name.is_empty() && !model_name.is_empty() {
790 set_model_cooldown(
791 &agent_name,
792 &model_name,
793 config.silence_cooldown,
794 "silence_detected",
795 )
796 .await;
797 record_agent_failure_with_message(
798 &agent_name,
799 &format!(
800 "stuck with active session after {}m",
801 timing.age.num_minutes()
802 ),
803 )
804 .await;
805 }
806 if !agent_name.is_empty() {
807 set_agent_cooldown(&agent_name, SILENCE_AGENT_COOLDOWN_SECS).await;
808 }
809 tracing::warn!(
810 task_id = task.id.0,
811 agent = %agent_name,
812 model = %model_name,
813 cooldown_secs = config.silence_cooldown,
814 "stuck-task cooldown applied — router will avoid this agent/model on next attempt"
815 );
816 }
817
818 {
821 let stale_labels: Vec<String> = task
822 .labels
823 .iter()
824 .filter(|l| l.starts_with("agent:") || l.starts_with("model:"))
825 .cloned()
826 .collect();
827 if !stale_labels.is_empty() {
828 let backend_clone = Arc::clone(backend);
829 let task_id_clone = task.id.clone();
830 tokio::spawn(async move {
831 for label in &stale_labels {
832 if let Err(e) = backend_clone.remove_label(&task_id_clone, label).await {
833 tracing::warn!(task_id = task_id_clone.0, label, error = %e, "failed to remove stale routing label during stuck-task recovery");
834 }
835 }
836 });
837 }
838 }
839 let resolved_store_id = match cached_store_id {
840 Some(id) => Some(id),
841 None => match store.resolve_task_id(repo, &task.id.0).await {
842 Ok(Some(id)) => Some(id),
843 Ok(None) => {
844 tracing::warn!(
845 task_id = %task.id.0,
846 repo,
847 "resolve_task_id returned None during stuck-task recovery"
848 );
849 None
850 }
851 Err(e) => {
852 tracing::error!(
853 task_id = %task.id.0,
854 repo,
855 error = %e,
856 "resolve_task_id failed during stuck-task recovery"
857 );
858 None
859 }
860 },
861 };
862 if let Some(store_id) = resolved_store_id {
863 if let Err(e) = store_set_by_id(
864 &Some(store),
865 store_id,
866 &[
867 ("agent", serde_json::Value::Null),
868 ("model", serde_json::Value::Null),
869 ("route_attempts", serde_json::json!(0)),
870 ],
871 )
872 .await
873 {
874 tracing::warn!(
875 task_id = task.id.0,
876 ?e,
877 "failed to clear routing fields for stuck task — will retry next tick"
878 );
879 continue;
880 }
881 finalize_open_runs_for_recovery(
882 store,
883 store_id,
884 if timing.has_session {
885 "stuck-task recovery: active session killed"
886 } else {
887 "stuck-task recovery: no session found"
888 },
889 )
890 .await;
891 }
892 if let Err(e) = task_manager.update_task_status(&task.id, Status::New).await {
893 tracing::warn!(task_id = task.id.0, ?e, "failed to reset stuck task status");
894 continue;
895 }
896
897 let details = serde_json::json!({
899 "failure_reason": if timing.has_session { "stuck_with_session" } else { "stuck_no_session" },
900 "age_minutes": timing.age.num_minutes(),
901 "had_session": timing.has_session,
902 });
903 crate::store::store_log_activity(
904 &Some(Arc::clone(store)),
905 repo,
906 &task.id.0,
907 "rerouted",
908 Some("in_progress"),
909 Some("new"),
910 None::<&str>,
911 None::<&str>,
912 Some(&details),
913 )
914 .await;
915
916 let reason = if timing.has_session {
917 format!(
918 "timed out after {}m with active session (cleared agent for re-routing)",
919 timing.age.num_minutes()
920 )
921 } else {
922 format!(
923 "no session found — reclaiming early after {}m (cleared agent for re-routing)",
924 timing.age.num_minutes()
925 )
926 };
927 {
929 let backend_clone = Arc::clone(backend);
930 let task_id_clone = task.id.clone();
931 let body = format!(
932 "[{}] recovered: stuck in_progress — {}{}",
933 chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
934 reason,
935 crate::engine::orch_footer()
936 );
937 tokio::spawn(async move {
938 if let Err(e) = backend_clone.post_comment(&task_id_clone, &body).await {
939 tracing::warn!(
940 task_id = task_id_clone.0,
941 ?e,
942 "failed to post stuck-task recovery comment"
943 );
944 }
945 });
946 }
947 }
948
949 let mut routed = match task_manager.list_external_by_status(Status::Routed).await {
950 Ok(tasks) => tasks,
951 Err(e) => {
952 tracing::warn!(?e, "failed to list routed tasks for stuck recovery");
953 vec![]
954 }
955 };
956 match task_manager
957 .list_internal_by_status(crate::store::TaskStatus::Routed)
958 .await
959 {
960 Ok(tasks) => routed.extend(tasks),
961 Err(e) => tracing::warn!(
962 ?e,
963 "failed to list internal routed tasks for stuck recovery"
964 ),
965 }
966 for task in &routed {
967 let Some(timing) = stuck_task_timing_from_map(
968 tmux,
969 repo,
970 &task.id.0,
971 &task.updated_at,
972 config,
973 "cannot parse updated_at, skipping routed stuck-task check",
974 session_map,
975 ) else {
976 continue;
977 };
978
979 if !timing.has_session {
980 continue;
981 }
982
983 let session_name = tmux.session_name(repo, &task.id.0);
984 if let Err(e) = tmux.kill_session(&session_name).await {
985 tracing::warn!(
986 task_id = task.id.0,
987 session = %session_name,
988 error = %e,
989 "failed to kill stale tmux session blocking routed task"
990 );
991 continue;
992 }
993
994 recover_routed_blocked_dispatch(backend, store, task_manager, repo, task, &timing).await;
995 }
996
997 let all_external_in_progress = match store
1001 .list_all_by_status_global(crate::store::TaskStatus::InProgress)
1002 .await
1003 {
1004 Ok(tasks) => tasks,
1005 Err(e) => {
1006 tracing::warn!(
1007 ?e,
1008 "failed to list global in_progress tasks for cross-repo stuck recovery"
1009 );
1010 vec![]
1011 }
1012 };
1013 for task in all_external_in_progress
1014 .iter()
1015 .filter(|t| t.origin != "internal" && t.repo != repo)
1016 {
1017 let external_id = task
1018 .external_id
1019 .clone()
1020 .unwrap_or_else(|| task.id.to_string());
1021 let Some(timing) = stuck_task_timing_from_map(
1022 tmux,
1023 &task.repo,
1024 &external_id,
1025 &task.updated_at,
1026 config,
1027 "cannot parse updated_at, skipping cross-repo stuck-task check",
1028 session_map,
1029 ) else {
1030 continue;
1031 };
1032
1033 if !timing.has_session {
1037 let dispatch_key = format!("{}/{external_id}", task.repo);
1038 if dispatching.contains_key(&dispatch_key) {
1039 tracing::info!(
1043 task_id = task.id,
1044 repo = %task.repo,
1045 dispatch_key,
1046 age_mins = timing.age.num_minutes(),
1047 "skipping cross-repo stuck-task reclaim: runner still actively processing this task"
1048 );
1049 continue;
1050 }
1051 }
1052
1053 if timing.has_session {
1054 tracing::warn!(
1055 task_id = task.id,
1056 repo = %task.repo,
1057 age_mins = timing.age.num_minutes(),
1058 threshold_mins = timing.threshold / 60,
1059 "recovering cross-repo stuck task: timed out with active session → new"
1060 );
1061 } else {
1062 tracing::warn!(
1063 task_id = task.id,
1064 repo = %task.repo,
1065 dispatch_key = format!("{}/{external_id}", task.repo),
1066 age_mins = timing.age.num_minutes(),
1067 threshold_mins = timing.threshold / 60,
1068 "recovering cross-repo stuck task: no session found — reclaiming early → new"
1069 );
1070 }
1071
1072 if let Err(e) = store_set_by_id(
1073 &Some(store),
1074 task.id,
1075 &[
1076 ("agent", serde_json::Value::Null),
1077 ("model", serde_json::Value::Null),
1078 ("route_attempts", serde_json::json!(0)),
1079 ],
1080 )
1081 .await
1082 {
1083 tracing::warn!(
1084 task_id = task.id,
1085 repo = %task.repo,
1086 ?e,
1087 "failed to clear routing fields for cross-repo stuck task — will retry next tick"
1088 );
1089 continue;
1090 }
1091
1092 finalize_open_runs_for_recovery(
1093 store,
1094 task.id,
1095 if timing.has_session {
1096 "stuck-task recovery: cross-repo active session killed"
1097 } else {
1098 "stuck-task recovery: cross-repo no session found"
1099 },
1100 )
1101 .await;
1102
1103 if let Err(e) = store
1104 .update_status(task.id, crate::store::TaskStatus::New)
1105 .await
1106 {
1107 tracing::warn!(
1108 task_id = task.id,
1109 repo = %task.repo,
1110 ?e,
1111 "failed to reset cross-repo stuck task status"
1112 );
1113 continue;
1114 }
1115
1116 let details = serde_json::json!({
1117 "failure_reason": if timing.has_session { "stuck_with_session" } else { "stuck_no_session" },
1118 "age_minutes": timing.age.num_minutes(),
1119 "had_session": timing.has_session,
1120 "scope": "cross_repo_global_sweep",
1121 });
1122 crate::store::store_log_activity(
1123 &Some(Arc::clone(store)),
1124 &task.repo,
1125 &external_id,
1126 "rerouted",
1127 Some("in_progress"),
1128 Some("new"),
1129 None::<&str>,
1130 None::<&str>,
1131 Some(&details),
1132 )
1133 .await;
1134 }
1135
1136 use crate::store::TaskStatus as DbStatus;
1139 let internal_in_progress = match task_manager
1140 .list_internal_by_status(DbStatus::InProgress)
1141 .await
1142 {
1143 Ok(tasks) => tasks,
1144 Err(e) => {
1145 tracing::warn!(
1146 ?e,
1147 "failed to list internal in_progress tasks for stuck recovery"
1148 );
1149 vec![]
1150 }
1151 };
1152 for task in &internal_in_progress {
1153 let task_id = task.id.0.clone();
1154 let Some(timing) = stuck_task_timing_from_map(
1155 tmux,
1156 repo,
1157 &task_id,
1158 &task.updated_at,
1159 config,
1160 "cannot parse updated_at, skipping stuck internal-task check",
1161 session_map,
1162 ) else {
1163 continue;
1164 };
1165
1166 if !timing.has_session {
1172 let dispatch_key = format!("{repo}/{task_id}");
1173 if dispatching.contains_key(&dispatch_key) {
1174 tracing::info!(
1184 task_id,
1185 dispatch_key,
1186 age_mins = timing.age.num_minutes(),
1187 "skipping stuck-task reclaim: runner still actively processing this task"
1188 );
1189 continue;
1190 }
1191 }
1192
1193 if timing.has_session {
1194 tracing::warn!(
1195 task_id,
1196 age_mins = timing.age.num_minutes(),
1197 threshold_mins = timing.threshold / 60,
1198 "recovering stuck task: timed out with active session → new"
1199 );
1200 } else {
1201 tracing::warn!(
1202 task_id,
1203 dispatch_key = format!("{repo}/{task_id}"),
1204 age_mins = timing.age.num_minutes(),
1205 threshold_mins = timing.threshold / 60,
1206 "recovering stuck task: no session found — reclaiming early → new"
1207 );
1208 }
1209 let mut cached_store_id: Option<i64> = None;
1213 if timing.has_session {
1214 let store_task = match store.resolve_task_id(repo, &task_id).await {
1215 Ok(Some(store_id)) => {
1216 cached_store_id = Some(store_id);
1217 match store.get(store_id).await {
1218 Ok(t) => Some(t),
1219 Err(e) => {
1220 tracing::warn!(task_id, error = %e, "failed to fetch task from store for stuck internal-task cooldown");
1221 None
1222 }
1223 }
1224 }
1225 Ok(None) => None,
1226 Err(e) => {
1227 tracing::warn!(task_id, error = %e, "failed to resolve task id for stuck internal-task cooldown");
1228 None
1229 }
1230 };
1231 let agent_name = store_task
1232 .as_ref()
1233 .and_then(|t| t.agent.clone())
1234 .unwrap_or_default();
1235 let model_name = store_task
1236 .as_ref()
1237 .and_then(|t| t.model.clone())
1238 .unwrap_or_default();
1239
1240 if !agent_name.is_empty() && !model_name.is_empty() {
1241 set_model_cooldown(
1242 &agent_name,
1243 &model_name,
1244 config.silence_cooldown,
1245 "silence_detected",
1246 )
1247 .await;
1248 record_agent_failure_with_message(
1249 &agent_name,
1250 &format!(
1251 "internal task stuck with active session after {}m",
1252 timing.age.num_minutes()
1253 ),
1254 )
1255 .await;
1256 }
1257 if !agent_name.is_empty() {
1258 set_agent_cooldown(&agent_name, SILENCE_AGENT_COOLDOWN_SECS).await;
1259 }
1260 tracing::warn!(
1261 task_id,
1262 agent = %agent_name,
1263 model = %model_name,
1264 cooldown_secs = config.silence_cooldown,
1265 "internal stuck-task cooldown applied — router will avoid this agent/model on next attempt"
1266 );
1267 }
1268 let resolved_store_id = match cached_store_id {
1271 Some(id) => Some(id),
1272 None => match store.resolve_task_id(repo, &task_id).await {
1273 Ok(Some(id)) => Some(id),
1274 Ok(None) => {
1275 tracing::warn!(
1276 task_id = %task_id,
1277 repo,
1278 "resolve_task_id returned None during stuck internal-task recovery"
1279 );
1280 None
1281 }
1282 Err(e) => {
1283 tracing::error!(
1284 task_id = %task_id,
1285 repo,
1286 error = %e,
1287 "resolve_task_id failed during stuck internal-task recovery"
1288 );
1289 None
1290 }
1291 },
1292 };
1293 if let Some(store_id) = resolved_store_id {
1294 if let Err(e) = store_set_by_id(
1295 &Some(store),
1296 store_id,
1297 &[
1298 ("agent", serde_json::Value::Null),
1299 ("model", serde_json::Value::Null),
1300 ("route_attempts", serde_json::json!(0)),
1301 ],
1302 )
1303 .await
1304 {
1305 tracing::warn!(
1306 task_id,
1307 ?e,
1308 "failed to clear routing fields for stuck internal task — will retry next tick"
1309 );
1310 continue;
1311 }
1312 finalize_open_runs_for_recovery(
1313 store,
1314 store_id,
1315 if timing.has_session {
1316 "stuck-task recovery: internal active session killed"
1317 } else {
1318 "stuck-task recovery: internal no session found"
1319 },
1320 )
1321 .await;
1322 }
1323 if let Err(e) = task_manager
1324 .update_task_status(&ExternalId(task_id.clone()), Status::New)
1325 .await
1326 {
1327 tracing::warn!(task_id, ?e, "failed to reset stuck internal task status");
1328 }
1329 }
1330
1331 let in_review = match task_manager.list_external_by_status(Status::InReview).await {
1335 Ok(tasks) => tasks,
1336 Err(e) => {
1337 tracing::warn!(?e, "failed to list in_review tasks for stuck recovery");
1338 vec![]
1339 }
1340 };
1341 let store_in_review = match store
1345 .list_by_status(repo, crate::store::TaskStatus::InReview)
1346 .await
1347 {
1348 Ok(v) => v,
1349 Err(e) => {
1350 tracing::warn!(
1351 ?e,
1352 "failed to list store in_review tasks for review_session_expected prefetch"
1353 );
1354 vec![]
1355 }
1356 };
1357 let review_expected_set: std::collections::HashSet<String> = store_in_review
1358 .into_iter()
1359 .filter(|t| t.review_session_expected)
1360 .filter_map(|t| t.external_id)
1361 .collect();
1362 for task in &in_review {
1363 if !review_expected_set.contains(&task.id.0) {
1364 tracing::debug!(
1365 task_id = task.id.0,
1366 "in_review task is waiting on PR review, skipping stuck-session recovery"
1367 );
1368 continue;
1369 }
1370
1371 let review_task_id = format!("{}-review", task.id.0);
1372 let in_review_config = crate::engine::EngineConfig {
1380 no_session_stuck_timeout: config.in_review_no_session_stuck_timeout,
1381 ..config.clone()
1382 };
1383 let Some(timing) = stuck_task_timing_from_map(
1384 tmux,
1385 repo,
1386 &review_task_id,
1387 &task.updated_at,
1388 &in_review_config,
1389 "cannot parse updated_at, skipping stuck in_review check",
1390 session_map,
1391 ) else {
1392 continue;
1393 };
1394
1395 tracing::warn!(
1396 task_id = task.id.0,
1397 age_mins = timing.age.num_minutes(),
1398 threshold_mins = timing.threshold / 60,
1399 "recovering stuck in_review task: no session found — resetting to needs_review"
1400 );
1401 if let Err(e) = task_manager
1402 .update_task_status(&task.id, Status::NeedsReview)
1403 .await
1404 {
1405 tracing::warn!(
1406 task_id = task.id.0,
1407 ?e,
1408 "failed to reset stuck in_review task status"
1409 );
1410 continue;
1411 } else {
1412 set_review_session_expected(store, repo, &task.id.0, false).await;
1413 }
1414
1415 if let Ok(Some(store_id)) = store.resolve_task_id(repo, &task.id.0).await {
1417 finalize_open_runs_for_recovery(
1418 store,
1419 store_id,
1420 "stuck-task recovery: in_review session killed",
1421 )
1422 .await;
1423 }
1424
1425 let details = serde_json::json!({
1427 "failure_reason": "stuck_in_review",
1428 "age_minutes": timing.age.num_minutes(),
1429 "had_session": timing.has_session,
1430 });
1431 crate::store::store_log_activity(
1432 &Some(Arc::clone(store)),
1433 repo,
1434 &task.id.0,
1435 "rerouted",
1436 Some("in_review"),
1437 Some("needs_review"),
1438 None::<&str>,
1439 None::<&str>,
1440 Some(&details),
1441 )
1442 .await;
1443 if let Err(e) = backend
1444 .post_comment(
1445 &task.id,
1446 &format!(
1447 "[{}] recovered: stuck in_review — no session found after {}m, resetting to needs_review{}",
1448 chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
1449 timing.age.num_minutes(),
1450 crate::engine::orch_footer()
1451 ),
1452 )
1453 .await
1454 {
1455 tracing::warn!(
1456 task_id = task.id.0,
1457 ?e,
1458 "failed to post stuck in_review recovery comment"
1459 );
1460 }
1461 }
1462
1463 let internal_in_review = match task_manager
1465 .list_internal_by_status(DbStatus::InReview)
1466 .await
1467 {
1468 Ok(tasks) => tasks,
1469 Err(e) => {
1470 tracing::warn!(
1471 ?e,
1472 "failed to list internal in_review tasks for stuck recovery"
1473 );
1474 vec![]
1475 }
1476 };
1477 for task in &internal_in_review {
1478 let task_id = task.id.0.clone();
1479 if !review_expected_set.contains(&task_id) {
1480 tracing::debug!(
1481 task_id,
1482 "internal in_review task is waiting on PR review, skipping stuck-session recovery"
1483 );
1484 continue;
1485 }
1486
1487 let review_task_id = format!("{}-review", task_id);
1488 let in_review_config = crate::engine::EngineConfig {
1489 no_session_stuck_timeout: config.in_review_no_session_stuck_timeout,
1490 ..config.clone()
1491 };
1492 let Some(timing) = stuck_task_timing_from_map(
1493 tmux,
1494 repo,
1495 &review_task_id,
1496 &task.updated_at,
1497 &in_review_config,
1498 "cannot parse updated_at, skipping stuck internal in_review check",
1499 session_map,
1500 ) else {
1501 continue;
1502 };
1503
1504 tracing::warn!(
1505 task_id,
1506 age_mins = timing.age.num_minutes(),
1507 threshold_mins = timing.threshold / 60,
1508 "recovering stuck internal in_review task: no session found — resetting to needs_review"
1509 );
1510 if let Err(e) = task_manager
1511 .update_task_status(&ExternalId(task_id.clone()), Status::NeedsReview)
1512 .await
1513 {
1514 tracing::warn!(
1515 task_id,
1516 ?e,
1517 "failed to reset stuck internal in_review task status"
1518 );
1519 } else {
1520 set_review_session_expected(store, repo, &task_id, false).await;
1521 }
1522
1523 if let Ok(Some(store_id)) = store.resolve_task_id(repo, &task_id).await {
1525 finalize_open_runs_for_recovery(
1526 store,
1527 store_id,
1528 "stuck-task recovery: internal in_review session killed",
1529 )
1530 .await;
1531 }
1532 }
1533
1534 if let Err(e) = crate::engine::sync::auto_unblock_ci_failure_blocked_tasks_global(
1539 task_manager,
1540 store,
1541 auto_merge_in_flight,
1542 )
1543 .await
1544 {
1545 tracing::warn!(err = %e, "global CI-failure blocked sweep failed");
1546 }
1547
1548 if let Err(e) = crate::engine::sync::refire_and_escalate_stale_needs_review_global(
1555 task_manager,
1556 store,
1557 router,
1558 dispatching,
1559 )
1560 .await
1561 {
1562 tracing::warn!(err = %e, "global NeedsReview refire/escalation sweep failed");
1563 }
1564
1565 if let Err(e) = crate::engine::sync::auto_recover_rebroadcast_blocked_tasks_global(
1573 router,
1574 task_manager,
1575 store,
1576 )
1577 .await
1578 {
1579 tracing::warn!(err = %e, "global rebroadcast-blocked recovery sweep failed");
1580 }
1581
1582 Ok(())
1583}
1584
1585pub(crate) async fn tick_route_tasks(
1587 backend: &Arc<dyn ExternalBackend>,
1588 task_manager: &Arc<TaskManager>,
1589 router: &mut Router,
1590 store: &Arc<TaskStore>,
1591 repo: &str,
1592 routing_quota: &mut RoutingTickQuota,
1593) -> anyhow::Result<()> {
1594 let _span = tracing::info_span!("engine.tick.phase3a.route").entered();
1595 if let Err(err) = crate::engine::dependencies::apply_claim_gates(store).await {
1596 tracing::warn!(err = %err, "dependency claim gate failed");
1597 }
1598 if routing_quota.is_exhausted() {
1599 tracing::debug!("routing quota exhausted for this tick — skipping repo routing phase");
1600 return Ok(());
1601 }
1602
1603 if is_github_circuit_open() {
1607 let remaining = github_circuit_remaining_secs();
1608 tracing::info!(
1609 remaining_secs = remaining,
1610 "GitHub 5xx circuit breaker open — skipping routing phase"
1611 );
1612 return Ok(());
1613 }
1614 let new_tasks = task_manager.list_routable().await?;
1615 let routable: Vec<&ExternalTask> = new_tasks
1616 .iter()
1617 .filter(|t| !skips_agent(&t.labels))
1618 .collect();
1619
1620 if !routable.is_empty() {
1624 let start = std::time::Instant::now();
1625 router.refresh_health(store).await;
1626 tracing::debug!(
1627 duration_ms = start.elapsed().as_millis(),
1628 "router.refresh_health completed"
1629 );
1630 }
1631
1632 for task in routable.into_iter().take(routing_quota.remaining()) {
1634 if !routing_quota.consume_one() {
1635 break;
1636 }
1637 let _task_span = tracing::info_span!("engine.route", task_id = %task.id.0).entered();
1638
1639 let task_start = Instant::now();
1640 match router.route(task, store, repo).await {
1641 Ok(result) => {
1642 tracing::debug!(task_id = %task.id.0, duration_ms = task_start.elapsed().as_millis(), "route completed");
1643 if let Err(e) = router
1645 .store_route_result(&task.id.0, &result, store, repo)
1646 .await
1647 {
1648 tracing::warn!(
1649 task_id = task.id.0,
1650 ?e,
1651 "failed to store route result; skipping Routed transition"
1652 );
1653 continue;
1654 }
1655
1656 if is_internal_id(&task.id.0) {
1657 if let Err(e) = task_manager
1659 .update_task_status(&task.id, Status::Routed)
1660 .await
1661 {
1662 tracing::warn!(
1663 task_id = task.id.0,
1664 ?e,
1665 "failed to set internal status:routed"
1666 );
1667 continue;
1668 }
1669 } else {
1670 let backend_clone = Arc::clone(backend);
1674 let task_id_clone = task.id.clone();
1675 let task_labels: Vec<String> = task.labels.clone();
1676 let new_labels = {
1677 let mut labels = vec![
1678 format!("agent:{}", result.agent),
1679 format!("complexity:{}", result.complexity),
1680 ];
1681 if let Some(ref model) = result.model {
1682 labels.push(format!("model:{model}"));
1683 }
1684 labels
1685 };
1686 tokio::spawn(async move {
1687 for label in &task_labels {
1689 if label.starts_with("agent:")
1690 || label.starts_with("complexity:")
1691 || label.starts_with("model:")
1692 {
1693 if let Err(e) =
1694 backend_clone.remove_label(&task_id_clone, label).await
1695 {
1696 tracing::warn!(task_id = task_id_clone.0, label, error = %e, "failed to remove stale routing label during re-route");
1697 }
1698 }
1699 }
1700 if let Err(e) = backend_clone.set_labels(&task_id_clone, &new_labels).await
1702 {
1703 tracing::warn!(
1704 task_id = task_id_clone.0,
1705 ?e,
1706 "failed to set routing labels"
1707 );
1708 }
1709 });
1710
1711 if let Err(e) = task_manager
1713 .update_task_status(&task.id, Status::Routed)
1714 .await
1715 {
1716 tracing::warn!(task_id = task.id.0, ?e, "failed to set status:routed");
1717 continue;
1718 }
1719 }
1720
1721 match store.ensure_external_task(repo, task).await {
1723 Ok(store_id) => {
1724 let profile_json =
1725 serde_json::to_string(&result.profile).unwrap_or_default();
1726 let skills_json =
1727 serde_json::to_string(&result.selected_skills).unwrap_or_default();
1728 if let Err(e) = store
1729 .store_route(&crate::store::StoreRoute {
1730 id: store_id,
1731 agent: &result.agent,
1732 model: result.model.as_deref(),
1733 complexity: &result.complexity,
1734 estimate: result.estimate,
1735 reason: &result.reason,
1736 profile: &profile_json,
1737 skills: &skills_json,
1738 })
1739 .await
1740 {
1741 tracing::debug!(
1742 task_id = task.id.0,
1743 ?e,
1744 "dual-write: store_route failed"
1745 );
1746 }
1747 if let Err(e) = store
1748 .update_status(store_id, crate::store::TaskStatus::Routed)
1749 .await
1750 {
1751 tracing::debug!(
1752 task_id = task.id.0,
1753 ?e,
1754 "dual-write: update_status failed"
1755 );
1756 }
1757 if result.estimate > 0 {
1761 let backend_clone = Arc::clone(backend);
1762 let task_id = task.id.clone();
1763 let estimate = result.estimate;
1764 tokio::spawn(async move {
1765 if let Err(e) = backend_clone
1766 .sync_estimate_to_project(&task_id, estimate)
1767 .await
1768 {
1769 tracing::debug!(
1770 task_id = task_id.0,
1771 estimate,
1772 err = %e,
1773 "dual-write: estimate project sync failed"
1774 );
1775 }
1776 });
1777 }
1778 }
1779 Err(e) => {
1780 tracing::debug!(
1781 task_id = task.id.0,
1782 ?e,
1783 "dual-write: ensure_external_task failed"
1784 );
1785 }
1786 }
1787
1788 if let Some(ref warning) = result.warning {
1789 tracing::warn!(task_id = task.id.0, warning, "routing sanity warning");
1790 }
1791
1792 tracing::info!(
1793 task_id = task.id.0,
1794 agent = %result.agent,
1795 complexity = %result.complexity,
1796 reason = %result.reason,
1797 "task routed"
1798 );
1799 }
1800 Err(e) => {
1801 if let Some(cooled) = e.downcast_ref::<AllAgentsCooledError>() {
1802 tracing::error!(
1803 task_id = %task.id.0,
1804 scope = %cooled.scope(),
1805 remaining_secs = ?cooled.remaining_secs(),
1806 "ALL AGENTS COOLED — no agent available to route this task. \
1807 Task stays in 'new' and will be retried on every tick. \
1808 Check `orch cooldown list` to see what is blocking dispatch."
1809 );
1810 continue;
1811 }
1812 if let Some(waiting) = e.downcast_ref::<crate::store::WaitingOnDependency>() {
1813 tracing::info!(
1814 task_id = %task.id.0,
1815 blockers = %waiting.blockers.join(", "),
1816 "task is waiting on a dependency"
1817 );
1818 continue;
1819 }
1820 tracing::error!(task_id = task.id.0, ?e, "routing failed, skipping task");
1821 }
1822 }
1823 }
1824 Ok(())
1825}
1826
1827pub(crate) async fn route_new_tasks_global(
1843 router: &mut Router,
1844 store: &Arc<TaskStore>,
1845 active_repos: &std::collections::HashSet<String>,
1846 routing_quota: &mut RoutingTickQuota,
1847) -> anyhow::Result<()> {
1848 use crate::store::{StoreRoute, TaskStatus};
1849
1850 if routing_quota.is_exhausted() {
1851 tracing::debug!("routing quota exhausted for this tick — skipping global routing sweep");
1852 return Ok(());
1853 }
1854
1855 let all_new = store.list_all_by_status_global(TaskStatus::New).await?;
1856 let orphaned: Vec<&crate::store::Task> = all_new
1857 .iter()
1858 .filter(|t| !active_repos.contains(&t.repo))
1859 .filter(|t| !skips_agent(&t.labels))
1860 .collect();
1861
1862 if orphaned.is_empty() {
1863 return Ok(());
1864 }
1865
1866 tracing::info!(
1867 count = orphaned.len(),
1868 "global routing sweep: found new tasks from inactive repos"
1869 );
1870
1871 router.refresh_health(store).await;
1872
1873 for task in orphaned.into_iter().take(routing_quota.remaining()) {
1874 if !routing_quota.consume_one() {
1875 break;
1876 }
1877 let ext_task = crate::engine::tasks::store_task_to_external(task);
1878 let task_start = Instant::now();
1879 match router.route(&ext_task, store, &task.repo).await {
1880 Ok(result) => {
1881 tracing::debug!(
1882 task_id = %ext_task.id.0,
1883 repo = %task.repo,
1884 duration_ms = task_start.elapsed().as_millis(),
1885 "global sweep: route completed"
1886 );
1887 let profile_json = serde_json::to_string(&result.profile).unwrap_or_default();
1888 let skills_json =
1889 serde_json::to_string(&result.selected_skills).unwrap_or_default();
1890 if let Err(e) = store
1891 .store_route(&StoreRoute {
1892 id: task.id,
1893 agent: &result.agent,
1894 model: result.model.as_deref(),
1895 complexity: &result.complexity,
1896 estimate: result.estimate,
1897 reason: &result.reason,
1898 profile: &profile_json,
1899 skills: &skills_json,
1900 })
1901 .await
1902 {
1903 tracing::warn!(
1904 task_id = %ext_task.id.0,
1905 repo = %task.repo,
1906 err = %e,
1907 "global sweep: failed to store route result; skipping status update"
1908 );
1909 continue;
1910 }
1911 if let Err(e) = store.update_status(task.id, TaskStatus::Routed).await {
1912 tracing::warn!(
1913 task_id = %ext_task.id.0,
1914 repo = %task.repo,
1915 err = %e,
1916 "global sweep: failed to set status:routed"
1917 );
1918 } else {
1919 tracing::info!(
1920 task_id = %ext_task.id.0,
1921 repo = %task.repo,
1922 agent = %result.agent,
1923 complexity = %result.complexity,
1924 "global sweep: task routed"
1925 );
1926 }
1927 if let Some(ref warning) = result.warning {
1928 tracing::warn!(task_id = %ext_task.id.0, warning, "global sweep: routing sanity warning");
1929 }
1930 }
1931 Err(e) => {
1932 if let Some(cooled) = e.downcast_ref::<AllAgentsCooledError>() {
1933 tracing::error!(
1934 task_id = %ext_task.id.0,
1935 repo = %task.repo,
1936 scope = %cooled.scope(),
1937 remaining_secs = ?cooled.remaining_secs(),
1938 "global sweep: ALL AGENTS COOLED — task stays in 'new'"
1939 );
1940 continue;
1941 }
1942 if let Some(waiting) = e.downcast_ref::<crate::store::WaitingOnDependency>() {
1943 tracing::info!(
1944 task_id = %ext_task.id.0,
1945 repo = %task.repo,
1946 blockers = %waiting.blockers.join(", "),
1947 "global sweep: task is waiting on a dependency"
1948 );
1949 continue;
1950 }
1951 tracing::error!(
1952 task_id = %ext_task.id.0,
1953 repo = %task.repo,
1954 err = ?e,
1955 "global sweep: routing failed, skipping task"
1956 );
1957 }
1958 }
1959 }
1960
1961 Ok(())
1962}
1963
1964#[derive(Debug, Clone, Copy)]
1966pub(crate) struct DispatchMode {
1967 pub(crate) is_degraded: bool,
1968 pub(crate) healthy_agents: usize,
1969 pub(crate) threshold: usize,
1970}
1971
1972pub(crate) fn dispatch_mode_from_router(router: &Router) -> DispatchMode {
1973 let threshold = crate::engine::router::config::min_healthy_agents_threshold();
1974 let healthy_agents = router.healthy_agent_count("simple");
1975 let is_degraded = router.is_degraded(threshold);
1976 DispatchMode {
1977 is_degraded,
1978 healthy_agents,
1979 threshold,
1980 }
1981}
1982
1983#[allow(clippy::too_many_arguments)]
1985#[tracing::instrument(skip_all, name = "engine.tick.phase3b.dispatch")]
1986pub(crate) async fn tick_dispatch_tasks(
1987 backend: &Arc<dyn ExternalBackend>,
1988 tmux: &Arc<TmuxManager>,
1989 repo: &str,
1990 runner: &Arc<TaskRunner>,
1991 capture: &Arc<CaptureService>,
1992 semaphore: &Arc<Semaphore>,
1993 task_manager: &Arc<TaskManager>,
1994 weight_tx: &mpsc::Sender<WeightSignal>,
1995 dispatch_mode: DispatchMode,
1996 dispatching: &Arc<DashMap<String, String>>,
1997 store: &Arc<TaskStore>,
1998 session_map: &std::collections::HashMap<String, bool>,
1999) -> anyhow::Result<()> {
2000 let mut routed_tasks = task_manager.list_external_by_status(Status::Routed).await?;
2002
2003 use crate::store::TaskStatus as DbStatus;
2005 let internal_routed = task_manager
2006 .list_internal_by_status(DbStatus::Routed)
2007 .await?;
2008 routed_tasks.extend(internal_routed);
2009
2010 let dispatchable: Vec<ExternalTask> = routed_tasks
2012 .into_iter()
2013 .filter(|t| !skips_agent(&t.labels))
2014 .collect();
2015
2016 dispatch_tasks_for_repo(
2017 backend,
2018 tmux,
2019 repo,
2020 runner,
2021 capture,
2022 semaphore,
2023 task_manager,
2024 weight_tx,
2025 dispatch_mode,
2026 dispatching,
2027 store,
2028 session_map,
2029 dispatchable,
2030 )
2031 .await
2032}
2033
2034#[allow(clippy::too_many_arguments)]
2040async fn dispatch_tasks_for_repo(
2041 backend: &Arc<dyn ExternalBackend>,
2042 tmux: &Arc<TmuxManager>,
2043 repo: &str,
2044 runner: &Arc<TaskRunner>,
2045 capture: &Arc<CaptureService>,
2046 semaphore: &Arc<Semaphore>,
2047 task_manager: &Arc<TaskManager>,
2048 weight_tx: &mpsc::Sender<WeightSignal>,
2049 dispatch_mode: DispatchMode,
2050 dispatching: &Arc<DashMap<String, String>>,
2051 store: &Arc<TaskStore>,
2052 session_map: &std::collections::HashMap<String, bool>,
2053 dispatchable: Vec<ExternalTask>,
2054) -> anyhow::Result<()> {
2055 if dispatchable.is_empty() {
2056 tracing::debug!(count = 0, "dispatchable tasks found");
2057 return Ok(());
2058 }
2059
2060 tracing::info!(count = dispatchable.len(), "dispatchable tasks found");
2061 if dispatch_mode.is_degraded {
2062 tracing::warn!(
2063 healthy_agents = dispatch_mode.healthy_agents,
2064 threshold = dispatch_mode.threshold,
2065 "degraded mode: using sequential dispatch"
2066 );
2067 }
2068
2069 let sequential_delay = if dispatch_mode.is_degraded {
2070 crate::engine::router::config::sequential_dispatch_delay_ms()
2071 } else {
2072 0
2073 };
2074
2075 for (idx, task) in dispatchable.into_iter().enumerate() {
2076 if crate::engine::dependencies::dispatch_is_waiting(store, repo, &task.id.0).await {
2077 continue;
2078 }
2079 if idx > 0 && dispatch_mode.is_degraded {
2081 let delay_ms = sequential_delay;
2082 tracing::debug!(
2083 task_id = task.id.0,
2084 delay_ms = delay_ms,
2085 "sequential dispatch: waiting before dispatch"
2086 );
2087 tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
2088 }
2089 let dispatch_key = format!("{}/{}", repo, task.id.0);
2096 {
2101 use dashmap::mapref::entry::Entry;
2102 match dispatching.entry(dispatch_key.clone()) {
2103 Entry::Occupied(existing) => {
2104 let existing_id = existing.get().clone();
2105 drop(existing); if existing_id == task.id.0 {
2107 tracing::debug!(
2108 task_id = task.id.0,
2109 "task already dispatching, skipping duplicate"
2110 );
2111 } else {
2112 tracing::warn!(
2113 task_id = task.id.0,
2114 existing_task_id = existing_id,
2115 dispatch_key,
2116 "dispatch key collision: unexpected task already holds this key"
2117 );
2118 }
2119 continue;
2120 }
2121 Entry::Vacant(slot) => {
2122 slot.insert(task.id.0.clone());
2123 tracing::info!(task_id = task.id.0, dispatch_key, "dispatch guard claimed");
2130 }
2131 }
2132 }
2133 let dispatch_guard = DispatchGuard::new(dispatching.clone(), dispatch_key.clone());
2135 let task_id = task.id.0.clone();
2136
2137 let route_result = match get_route_result(store, repo, &task_id).await {
2140 Ok(r) => Some(r),
2141 Err(RouteResultError::NoAgent { .. }) => {
2142 tracing::warn!(
2143 task_id,
2144 "routed task missing agent — resetting to new for re-routing (#1604)"
2145 );
2146 if let Err(e2) = task_manager.update_task_status(&task.id, Status::New).await {
2147 tracing::error!(task_id, error = %e2, "failed to reset task to new");
2148 }
2149 continue; }
2151 Err(e) => {
2152 tracing::warn!(task_id, error = %e, "get_route_result failed — dispatching without route info");
2153 None
2154 }
2155 };
2156
2157 if let Some(route) = route_result.as_ref() {
2163 let agent_cooled = is_agent_in_cooldown(&route.agent);
2164 let model_cooled = route
2165 .model
2166 .as_deref()
2167 .is_some_and(|model| is_model_in_cooldown(&route.agent, model));
2168 if agent_cooled || model_cooled {
2169 tracing::warn!(
2170 task_id,
2171 agent = route.agent,
2172 model = route.model.as_deref().unwrap_or(""),
2173 "routed agent/model entered cooldown after routing — deferring dispatch to next tick"
2174 );
2175 continue; }
2177 }
2178
2179 let session_name = tmux.session_name(repo, &task_id);
2181 if tmux
2182 .session_blocks_dispatch_from_map(&session_name, session_map)
2183 .await
2184 {
2185 tracing::warn!(
2186 task_id,
2187 session_name,
2188 "task has existing tmux session, skipping dispatch"
2189 );
2190 continue; }
2192
2193 let permit = match semaphore.clone().try_acquire_owned() {
2195 Ok(p) => p,
2196 Err(_) => {
2197 tracing::debug!("all parallel slots busy, skipping remaining tasks");
2198 break; }
2200 };
2201
2202 let set_in_progress_result = task_manager
2204 .update_task_status(&task.id, Status::InProgress)
2205 .await;
2206 if let Err(e) = set_in_progress_result {
2207 tracing::error!(task_id, ?e, "failed to set in_progress, skipping dispatch");
2208 drop(permit);
2209 continue; }
2211 tracing::info!(task_id, "dispatching task");
2212
2213 capture
2215 .register_session(repo, &task_id, &session_name)
2216 .await;
2217
2218 let runner = runner.clone();
2220 let backend = backend.clone();
2221 let tmux = tmux.clone();
2222 let capture = capture.clone();
2223 let task_id_for_cleanup = task_id.clone();
2224 let task_owned = task;
2225 let weight_tx = weight_tx.clone();
2226 let repo_owned = repo.to_string();
2227 let task_manager_for_spawn = task_manager.clone();
2228 let store_for_spawn = store.clone();
2229
2230 let repo_ctx = repo_owned.clone();
2231 tokio::spawn(REPO_CONTEXT.scope(repo_ctx, async move {
2232 let _dispatch_guard = dispatch_guard; let dispatch_start = std::time::Instant::now();
2234 match runner
2235 .run_with_context(&task_owned, &backend, &tmux, route_result.as_ref())
2236 .await
2237 {
2238 Ok(signal) => {
2239 tracing::info!(task_id, "task runner completed");
2240
2241 let duration = dispatch_start.elapsed().as_secs_f64();
2242
2243 let enable_review = config::get("workflow.enable_review_agent")
2249 .map(|v| v != "false")
2250 .unwrap_or(true);
2251 let display_status = match &signal {
2252 WeightSignal::Success { .. } if enable_review => {
2253 let has_pr = match store_for_spawn
2257 .resolve_task_id(&repo_owned, &task_id)
2258 .await
2259 {
2260 Ok(Some(store_id)) => match store_for_spawn
2261 .get(store_id)
2262 .await
2263 {
2264 Ok(t) => t.pr_number.is_some(),
2265 Err(e) => {
2266 tracing::warn!(task_id, err = %e, "store.get() failed — defaulting to needs_review");
2267 true
2268 }
2269 },
2270 Ok(None) => false,
2271 Err(e) => {
2272 tracing::warn!(task_id, err = %e, "failed to check PR status — defaulting to needs_review");
2273 true
2274 }
2275 };
2276 if has_pr {
2277 "needs_review"
2278 } else {
2279 "done"
2280 }
2281 }
2282 WeightSignal::Success { .. } => "done",
2283 WeightSignal::RateLimited { .. } => "new",
2284 WeightSignal::Blocked => "blocked",
2285 WeightSignal::Rerouted => "new",
2290 WeightSignal::None if enable_review => "needs_review",
2295 WeightSignal::None => "done",
2296 };
2297
2298 let _ = weight_tx.send(signal).await;
2300
2301 if display_status == "needs_review" {
2305 if let Err(e) = task_manager_for_spawn
2308 .update_task_status_with_duration(
2309 &ExternalId(task_id.clone()),
2310 Status::NeedsReview,
2311 Some(duration),
2312 )
2313 .await
2314 {
2315 tracing::error!(task_id, err = %e, "update_task_status(NeedsReview) failed — task may be stuck");
2316 }
2317 } else {
2319 let final_status = match display_status {
2321 "done" => Status::Done,
2322 "blocked" => Status::Blocked,
2323 _ => Status::New,
2324 };
2325 if let Err(e) = task_manager_for_spawn
2326 .update_task_status_with_duration(
2327 &ExternalId(task_id.clone()),
2328 final_status,
2329 Some(duration),
2330 )
2331 .await
2332 {
2333 tracing::warn!(task_id, ?e, "failed to update task status after completion");
2334 }
2335 }
2336 }
2337 Err(e) => {
2338 tracing::error!(task_id, ?e, "task runner failed");
2339 let duration = dispatch_start.elapsed().as_secs_f64();
2340 if is_internal_id(&task_id) {
2341 if let Err(ue) = task_manager_for_spawn
2343 .update_task_status_with_duration(
2344 &ExternalId(task_id.clone()),
2345 Status::NeedsReview,
2346 Some(duration),
2347 )
2348 .await
2349 {
2350 tracing::error!(task_id, err = %ue, "update_task_status(NeedsReview) failed — task may be stuck");
2351 }
2352 } else {
2353 if let Err(comment_err) = backend
2355 .post_comment(
2356 &ExternalId(task_id.clone()),
2357 &format!(
2358 "[{}] error: task runner failed: {e}{}",
2359 chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
2360 crate::engine::orch_footer()
2361 ),
2362 )
2363 .await
2364 {
2365 tracing::warn!(
2366 task_id,
2367 ?comment_err,
2368 "failed to post error comment to GitHub"
2369 );
2370 }
2371 if let Err(ue) = task_manager_for_spawn
2375 .update_task_status_with_duration(
2376 &ExternalId(task_id.clone()),
2377 Status::NeedsReview,
2378 Some(duration),
2379 )
2380 .await
2381 {
2382 tracing::error!(task_id, err = %ue, "update_task_status(NeedsReview) failed — task may be stuck");
2383 }
2384 }
2385 }
2386 }
2387
2388 capture.unregister_session(&repo_owned, &task_id_for_cleanup).await;
2390
2391 drop(permit);
2393 }));
2395 }
2396 Ok(())
2397}
2398
2399fn skips_agent(labels: &[String]) -> bool {
2400 labels
2401 .iter()
2402 .any(|label| label == "no-agent" || label == "owner:human")
2403}
2404
2405fn group_orphaned_routed_tasks(
2412 all_routed: Vec<crate::store::Task>,
2413 active_repos: &std::collections::HashSet<String>,
2414) -> std::collections::HashMap<String, Vec<crate::store::Task>> {
2415 let mut by_repo: std::collections::HashMap<String, Vec<crate::store::Task>> =
2416 std::collections::HashMap::new();
2417 for task in all_routed
2418 .into_iter()
2419 .filter(|t| !active_repos.contains(&t.repo))
2420 .filter(|t| !skips_agent(&t.labels))
2421 {
2422 by_repo.entry(task.repo.clone()).or_default().push(task);
2423 }
2424 by_repo
2425}
2426
2427#[allow(clippy::too_many_arguments)]
2447pub(crate) async fn dispatch_routed_tasks_global(
2448 tmux: &Arc<TmuxManager>,
2449 capture: &Arc<CaptureService>,
2450 semaphore: &Arc<Semaphore>,
2451 weight_tx: &mpsc::Sender<WeightSignal>,
2452 dispatch_mode: DispatchMode,
2453 dispatching: &Arc<DashMap<String, String>>,
2454 store: &Arc<TaskStore>,
2455 session_map: &std::collections::HashMap<String, bool>,
2456 active_repos: &std::collections::HashSet<String>,
2457) -> anyhow::Result<()> {
2458 use crate::store::TaskStatus;
2459
2460 let all_routed = store.list_all_by_status_global(TaskStatus::Routed).await?;
2461 let by_repo = group_orphaned_routed_tasks(all_routed, active_repos);
2462
2463 if by_repo.is_empty() {
2464 return Ok(());
2465 }
2466
2467 tracing::info!(
2468 repos = by_repo.len(),
2469 "global dispatch sweep: found routed tasks from inactive repos"
2470 );
2471
2472 for (repo, tasks) in by_repo {
2473 let backend: Arc<dyn ExternalBackend> = match crate::backends::github::GitHubBackend::new(
2474 repo.clone(),
2475 ) {
2476 Ok(b) => Arc::new(b),
2477 Err(e) => {
2478 tracing::warn!(repo = %repo, err = %e, "global sweep: failed to construct backend, skipping repo");
2479 continue;
2480 }
2481 };
2482 let task_manager = Arc::new(TaskManager::with_store(
2483 backend.clone(),
2484 store.clone(),
2485 repo.clone(),
2486 ));
2487 let runner = Arc::new(TaskRunner::new(repo.clone()).with_store(store.clone()));
2488 let dispatchable: Vec<ExternalTask> = tasks
2489 .iter()
2490 .map(crate::engine::tasks::store_task_to_external)
2491 .collect();
2492
2493 if let Err(e) = dispatch_tasks_for_repo(
2494 &backend,
2495 tmux,
2496 &repo,
2497 &runner,
2498 capture,
2499 semaphore,
2500 &task_manager,
2501 weight_tx,
2502 dispatch_mode,
2503 dispatching,
2504 store,
2505 session_map,
2506 dispatchable,
2507 )
2508 .await
2509 {
2510 tracing::warn!(repo = %repo, err = %e, "global sweep: dispatch failed for repo");
2511 }
2512 }
2513
2514 Ok(())
2515}
2516
2517pub(crate) async fn tick_unblock_parents(
2519 backend: &Arc<dyn ExternalBackend>,
2520 task_manager: &Arc<TaskManager>,
2521 store: &Arc<TaskStore>,
2522 repo: &str,
2523) -> anyhow::Result<()> {
2524 let blocked = task_manager.list_all_by_status(Status::Blocked).await?;
2525
2526 let store_blocked = store
2530 .list_by_status(repo, crate::store::TaskStatus::Blocked)
2531 .await?;
2532 let has_block_reason: std::collections::HashSet<String> = store_blocked
2533 .iter()
2534 .filter(|t| t.block_reason.is_some())
2535 .filter_map(|t| t.external_id.clone())
2536 .collect();
2537
2538 let handles: Vec<_> = blocked
2542 .into_iter()
2543 .filter(|task| {
2544 if has_block_reason.contains(&task.id.0) {
2545 tracing::debug!(
2546 task_id = task.id.0,
2547 "skipping blocked task with known block_reason"
2548 );
2549 false
2550 } else {
2551 true
2552 }
2553 })
2554 .map(|task| {
2555 let backend_clone = Arc::clone(backend);
2556 let task_manager_clone = Arc::clone(task_manager);
2557 let store_clone = Arc::clone(store);
2558 let repo_owned = repo.to_string();
2559 tokio::spawn(async move {
2560 let task_id = &task.id;
2561
2562 let mut children: Vec<ExternalId> = Vec::new();
2563 let mut child_statuses: std::collections::HashMap<
2564 String,
2565 crate::store::TaskStatus,
2566 > = std::collections::HashMap::new();
2567
2568 match store_clone.resolve_task_id(&repo_owned, &task_id.0).await {
2569 Ok(Some(parent_store_id)) => {
2570 match store_clone.list_children(&repo_owned, parent_store_id).await {
2571 Ok(store_children) => {
2572 for child in store_children {
2573 let Some(child_external_id) = child.external_id else {
2574 continue;
2575 };
2576 child_statuses.insert(child_external_id.clone(), child.status);
2577 children.push(ExternalId(child_external_id));
2578 }
2579 }
2580 Err(e) => {
2581 tracing::debug!(
2582 task_id = task_id.0,
2583 ?e,
2584 "failed to list store-linked children"
2585 );
2586 }
2587 }
2588 }
2589 Ok(None) => {}
2590 Err(e) => {
2591 tracing::debug!(
2592 task_id = task_id.0,
2593 ?e,
2594 "failed to resolve blocked task in store for child lookup"
2595 );
2596 }
2597 }
2598
2599 if !is_internal_id(&task_id.0) {
2600 match backend_clone.get_sub_issues(task_id).await {
2601 Ok(ids) => {
2602 let mut seen: std::collections::HashSet<String> =
2603 children.iter().map(|child| child.0.clone()).collect();
2604 for id in ids {
2605 if seen.insert(id.0.clone()) {
2606 children.push(id);
2607 }
2608 }
2609 }
2610 Err(e) => {
2611 tracing::debug!(task_id = task_id.0, ?e, "failed to get sub-issues");
2612 }
2613 }
2614 }
2615
2616 if children.is_empty() {
2618 return;
2619 }
2620
2621 let mut all_done = true;
2624 let child_ext_strs: Vec<String> =
2626 children.iter().map(|c| c.0.clone()).collect();
2627 let child_exts: Vec<&str> =
2628 child_ext_strs.iter().map(|s| s.as_str()).collect();
2629 if let Ok(map) = store_clone
2631 .get_statuses_by_external_ids(&repo_owned, &child_exts)
2632 .await
2633 {
2634 for (k, v) in map.into_iter() {
2635 child_statuses.insert(k, v);
2636 }
2637 } else {
2638 tracing::debug!(
2639 task_id = task_id.0,
2640 "failed to batch-query store for child statuses; falling back to per-child lookups"
2641 );
2642 }
2643
2644 for child_id in &children {
2645 if let Some(status) = child_statuses.get(&child_id.0) {
2647 if *status != crate::store::TaskStatus::Done {
2648 all_done = false;
2649 break;
2650 }
2651 continue;
2652 }
2653
2654 match backend_clone.get_task(child_id).await {
2656 Ok(child) => {
2657 if !child.labels.iter().any(|l| l == Status::Done.as_label()) {
2658 all_done = false;
2659 break;
2660 }
2661 }
2662 Err(e) => {
2663 tracing::debug!(task_id = task_id.0, child = %child_id.0, ?e, "failed to fetch child task from backend — treating as not done");
2664 all_done = false;
2665 break;
2666 }
2667 }
2668 }
2669
2670 if all_done {
2671 tracing::info!(
2672 task_id = task_id.0,
2673 children = children.len(),
2674 "all children done, unblocking parent"
2675 );
2676 if let Err(e) =
2677 task_manager_clone.update_task_status(task_id, Status::New).await
2678 {
2679 tracing::warn!(task_id = task_id.0, ?e, "failed to unblock parent");
2680 }
2681 }
2682 })
2683 })
2684 .collect();
2685
2686 for handle in handles {
2687 if let Err(e) = handle.await {
2688 tracing::warn!(?e, "phase 4 sub-issue check panicked");
2689 }
2690 }
2691 Ok(())
2692}
2693
2694pub(crate) async fn tick_job_scheduler(
2696 jobs_path: &std::path::Path,
2697 backend: &Arc<dyn ExternalBackend>,
2698 store: Option<&Arc<TaskStore>>,
2699 repo: &str,
2700 transport: Option<&Arc<crate::channels::transport::Transport>>,
2701) -> anyhow::Result<()> {
2702 jobs::tick(jobs_path, backend, store, repo, transport).await
2703}
2704
2705#[allow(clippy::too_many_arguments)]
2715pub(crate) async fn tick(
2716 backend: &Arc<dyn ExternalBackend>,
2717 tmux: &Arc<TmuxManager>,
2718 repo: &str,
2719 runner: &Arc<TaskRunner>,
2720 capture: &Arc<CaptureService>,
2721 semaphore: &Arc<Semaphore>,
2722 config: &EngineConfig,
2723 jobs_path: &std::path::Path,
2724 router: &mut Router,
2725 task_manager: &Arc<TaskManager>,
2726 weight_tx: &mpsc::Sender<WeightSignal>,
2727 dispatching: &Arc<DashMap<String, String>>,
2728 store: &Arc<TaskStore>,
2729 transport: Option<&Arc<crate::channels::transport::Transport>>,
2730 active_repos: &std::collections::HashSet<String>,
2731 auto_merge_in_flight: &Arc<DashSet<String>>,
2732 routing_quota: &mut RoutingTickQuota,
2733) -> anyhow::Result<()> {
2734 let _tick_span = tracing::info_span!("engine.tick").entered();
2735
2736 startup_cleanup(tmux).await;
2738
2739 if let Err(e) = tick_check_session_completions(tmux, repo, capture, store).await {
2740 tracing::warn!(err = %e, "session completion phase failed");
2741 }
2742 if let Err(e) =
2743 tick_detect_silent_agents(tmux, repo, capture, backend, task_manager, config, store).await
2744 {
2745 tracing::warn!(err = %e, "silent-agent detection phase failed");
2746 }
2747 let session_map = tmux.batch_session_active().await;
2750 if let Err(e) = tick_recover_stuck_tasks(
2751 backend,
2752 tmux,
2753 repo,
2754 task_manager,
2755 config,
2756 store,
2757 &session_map,
2758 router,
2759 dispatching,
2760 auto_merge_in_flight,
2761 )
2762 .await
2763 {
2764 tracing::warn!(err = %e, "stuck-task recovery phase failed");
2765 }
2766 if let Err(err) = crate::engine::dependencies::reconcile_dependencies(
2767 backend.as_ref(),
2768 store,
2769 transport.map(|arc| arc.as_ref()),
2770 )
2771 .await
2772 {
2773 tracing::warn!(err = %err, "dependency reconcile failed");
2774 }
2775 if let Err(e) =
2776 tick_route_tasks(backend, task_manager, router, store, repo, routing_quota).await
2777 {
2778 tracing::warn!(err = %e, "routing phase failed");
2779 }
2780 if let Err(e) = route_new_tasks_global(router, store, active_repos, routing_quota).await {
2784 tracing::warn!(err = %e, "global new-task routing sweep failed");
2785 }
2786 let dispatch_mode = dispatch_mode_from_router(router);
2787 if let Err(e) = tick_dispatch_tasks(
2788 backend,
2789 tmux,
2790 repo,
2791 runner,
2792 capture,
2793 semaphore,
2794 task_manager,
2795 weight_tx,
2796 dispatch_mode,
2797 dispatching,
2798 store,
2799 &session_map,
2800 )
2801 .await
2802 {
2803 tracing::warn!(err = %e, "dispatch phase failed");
2804 }
2805 if let Err(e) = dispatch_routed_tasks_global(
2811 tmux,
2812 capture,
2813 semaphore,
2814 weight_tx,
2815 dispatch_mode,
2816 dispatching,
2817 store,
2818 &session_map,
2819 active_repos,
2820 )
2821 .await
2822 {
2823 tracing::warn!(err = %e, "global routed-task dispatch sweep failed");
2824 }
2825 if let Err(e) = tick_unblock_parents(backend, task_manager, store, repo).await {
2826 tracing::warn!(err = %e, "parent-unblock phase failed");
2827 }
2828 if let Err(e) = tick_job_scheduler(jobs_path, backend, Some(store), repo, transport).await {
2829 tracing::error!(?e, "job scheduler tick failed");
2830 }
2831 Ok(())
2832}
2833
2834#[cfg(test)]
2835mod tests {
2836 use super::*;
2837 use crate::backends::{ExternalId, ExternalTask, Mention, Status};
2838 use crate::channels::transport::Transport;
2839 use crate::engine::tasks::TaskManager;
2840 use crate::store::TaskStore;
2841 use crate::tmux::{SessionGuard, TmuxManager};
2842 use async_trait::async_trait;
2843 use std::sync::{Arc, Mutex};
2844
2845 #[test]
2846 fn skips_agent_for_human_and_no_agent_labels() {
2847 assert!(skips_agent(&["owner:human".to_string()]));
2848 assert!(skips_agent(&["no-agent".to_string()]));
2849 assert!(!skips_agent(&["status:new".to_string()]));
2850 }
2851
2852 fn cooled_review_router_for_test() -> Router {
2856 Router::new_for_test(
2857 crate::engine::router::RouterConfig::default(),
2858 vec!["claude".to_string()],
2859 )
2860 }
2861
2862 struct MockBackend {
2866 blocked_tasks: Vec<ExternalTask>,
2868 sub_issues: std::collections::HashMap<String, Vec<ExternalId>>,
2870 tasks_by_id: std::collections::HashMap<String, ExternalTask>,
2872 get_sub_issues_calls: Arc<Mutex<Vec<String>>>,
2874 status_updates: Arc<Mutex<Vec<(String, Status)>>>,
2876 removed_labels: Arc<Mutex<Vec<(String, String)>>>,
2878 posted_comments: Arc<Mutex<Vec<(String, String)>>>,
2880 }
2881
2882 impl MockBackend {
2883 fn new() -> Self {
2884 Self {
2885 blocked_tasks: vec![],
2886 sub_issues: Default::default(),
2887 tasks_by_id: Default::default(),
2888 get_sub_issues_calls: Arc::new(Mutex::new(vec![])),
2889 status_updates: Arc::new(Mutex::new(vec![])),
2890 removed_labels: Arc::new(Mutex::new(vec![])),
2891 posted_comments: Arc::new(Mutex::new(vec![])),
2892 }
2893 }
2894 }
2895
2896 fn make_task(id: &str, labels: &[&str]) -> ExternalTask {
2897 ExternalTask {
2898 id: ExternalId(id.to_string()),
2899 title: format!("Task {id}"),
2900 body: "".to_string(),
2901 state: "open".to_string(),
2902 labels: labels.iter().map(|s| s.to_string()).collect(),
2903 author: "bot".to_string(),
2904 created_at: "2026-01-01T00:00:00Z".to_string(),
2905 updated_at: "2026-01-01T00:00:00Z".to_string(),
2906 url: format!("https://github.com/test/test/issues/{id}"),
2907 }
2908 }
2909
2910 #[async_trait]
2911 impl ExternalBackend for MockBackend {
2912 fn name(&self) -> &str {
2913 "mock"
2914 }
2915 async fn create_task(
2916 &self,
2917 _title: &str,
2918 _body: &str,
2919 _labels: &[String],
2920 ) -> anyhow::Result<ExternalId> {
2921 Ok(ExternalId("new".to_string()))
2922 }
2923 async fn get_task(&self, id: &ExternalId) -> anyhow::Result<ExternalTask> {
2924 self.tasks_by_id
2925 .get(&id.0)
2926 .cloned()
2927 .ok_or_else(|| anyhow::anyhow!("task not found: {}", id.0))
2928 }
2929 async fn list_by_status(&self, status: Status) -> anyhow::Result<Vec<ExternalTask>> {
2930 if status == Status::Blocked {
2931 Ok(self.blocked_tasks.clone())
2932 } else {
2933 Ok(vec![])
2934 }
2935 }
2936 async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
2937 Ok(vec![])
2938 }
2939 async fn post_comment(&self, _id: &ExternalId, _body: &str) -> anyhow::Result<()> {
2940 self.posted_comments
2941 .lock()
2942 .unwrap()
2943 .push((_id.0.clone(), _body.to_string()));
2944 Ok(())
2945 }
2946 async fn set_labels(&self, _id: &ExternalId, _labels: &[String]) -> anyhow::Result<()> {
2947 Ok(())
2948 }
2949 async fn remove_label(&self, _id: &ExternalId, _label: &str) -> anyhow::Result<()> {
2950 self.removed_labels
2951 .lock()
2952 .unwrap()
2953 .push((_id.0.clone(), _label.to_string()));
2954 Ok(())
2955 }
2956 async fn get_sub_issues(&self, id: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
2957 self.get_sub_issues_calls.lock().unwrap().push(id.0.clone());
2958 Ok(self.sub_issues.get(&id.0).cloned().unwrap_or_default())
2959 }
2960 async fn create_sub_task(
2961 &self,
2962 _parent: &ExternalId,
2963 _title: &str,
2964 _body: &str,
2965 _labels: &[String],
2966 ) -> anyhow::Result<ExternalId> {
2967 Ok(ExternalId("child".to_string()))
2968 }
2969 async fn ensure_status_label(&self, _label: &str) -> anyhow::Result<()> {
2970 Ok(())
2971 }
2972 async fn has_open_issue_with_title(
2973 &self,
2974 _title: &str,
2975 _label: &str,
2976 ) -> anyhow::Result<bool> {
2977 Ok(false)
2978 }
2979 async fn health_check(&self) -> anyhow::Result<()> {
2980 Ok(())
2981 }
2982 async fn is_pr_merged(&self, _branch: &str) -> anyhow::Result<bool> {
2983 Ok(false)
2984 }
2985 async fn get_authenticated_user(&self) -> anyhow::Result<Option<String>> {
2986 Ok(Some("testbot".to_string()))
2987 }
2988 async fn get_mentions(&self, _since: &str) -> anyhow::Result<Vec<Mention>> {
2989 Ok(vec![])
2990 }
2991 async fn update_status(&self, id: &ExternalId, status: Status) -> anyhow::Result<()> {
2992 self.status_updates
2993 .lock()
2994 .unwrap()
2995 .push((id.0.clone(), status));
2996 Ok(())
2997 }
2998 }
2999
3000 fn make_task_manager(backend: Arc<dyn ExternalBackend>) -> Arc<TaskManager> {
3001 Arc::new(TaskManager::new(backend))
3002 }
3003
3004 #[tokio::test]
3007 async fn unblock_parents_unblocks_when_all_children_done() {
3008 let mut mock = MockBackend::new();
3009
3010 let parent = make_task("10", &["status:blocked"]);
3012 mock.blocked_tasks.push(parent.clone());
3013 mock.sub_issues.insert(
3014 "10".to_string(),
3015 vec![ExternalId("11".to_string()), ExternalId("12".to_string())],
3016 );
3017 mock.tasks_by_id
3018 .insert("11".to_string(), make_task("11", &["status:done"]));
3019 mock.tasks_by_id
3020 .insert("12".to_string(), make_task("12", &["status:done"]));
3021
3022 let status_updates = mock.status_updates.clone();
3023 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3024 let task_manager = make_task_manager(backend.clone());
3025 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3026
3027 tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3028 .await
3029 .unwrap();
3030
3031 let updates = status_updates.lock().unwrap();
3032 assert_eq!(updates.len(), 1, "parent should be unblocked");
3033 assert_eq!(updates[0], ("10".to_string(), Status::New));
3034 }
3035
3036 #[tokio::test]
3037 async fn unblock_parents_skips_when_child_not_done() {
3038 let mut mock = MockBackend::new();
3039
3040 let parent = make_task("20", &["status:blocked"]);
3041 mock.blocked_tasks.push(parent.clone());
3042 mock.sub_issues.insert(
3043 "20".to_string(),
3044 vec![ExternalId("21".to_string()), ExternalId("22".to_string())],
3045 );
3046 mock.tasks_by_id
3048 .insert("21".to_string(), make_task("21", &["status:done"]));
3049 mock.tasks_by_id
3050 .insert("22".to_string(), make_task("22", &["status:in_progress"]));
3051
3052 let status_updates = mock.status_updates.clone();
3053 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3054 let task_manager = make_task_manager(backend.clone());
3055 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3056
3057 tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3058 .await
3059 .unwrap();
3060
3061 let updates = status_updates.lock().unwrap();
3062 assert!(
3063 updates.is_empty(),
3064 "should not unblock when a child is still running"
3065 );
3066 }
3067
3068 #[tokio::test]
3069 async fn unblock_parents_skips_task_with_no_children() {
3070 let mut mock = MockBackend::new();
3071
3072 let parent = make_task("30", &["status:blocked"]);
3074 mock.blocked_tasks.push(parent.clone());
3075 let status_updates = mock.status_updates.clone();
3078 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3079 let task_manager = make_task_manager(backend.clone());
3080 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3081
3082 tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3083 .await
3084 .unwrap();
3085
3086 let updates = status_updates.lock().unwrap();
3087 assert!(
3088 updates.is_empty(),
3089 "should not unblock tasks with no sub-issues"
3090 );
3091 }
3092
3093 #[tokio::test]
3094 async fn unblock_parents_skips_task_with_block_reason_in_store() {
3095 let mut mock = MockBackend::new();
3098
3099 let parent = make_task("60", &["status:blocked"]);
3101 mock.blocked_tasks.push(parent.clone());
3102 mock.sub_issues
3105 .insert("60".to_string(), vec![ExternalId("61".to_string())]);
3106 mock.tasks_by_id
3107 .insert("61".to_string(), make_task("61", &["status:done"]));
3108
3109 let status_updates = mock.status_updates.clone();
3110 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3111 let task_manager = make_task_manager(backend.clone());
3112
3113 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3115 let task_id = store
3116 .upsert_external(&crate::store::UpsertExternal {
3117 repo: "test/repo",
3118 ext_id: "60",
3119 title: "Task 60",
3120 body: "",
3121 author: "bot",
3122 url: "https://github.com/test/test/issues/60",
3123 labels: &[],
3124 origin: "github",
3125 })
3126 .await
3127 .unwrap();
3128 store
3129 .update_status(task_id, crate::store::TaskStatus::Blocked)
3130 .await
3131 .unwrap();
3132 store
3133 .set_block_reason(task_id, Some("CI failure limit reached"))
3134 .await
3135 .unwrap();
3136
3137 tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3138 .await
3139 .unwrap();
3140
3141 let updates = status_updates.lock().unwrap();
3142 assert!(
3143 updates.is_empty(),
3144 "task with block_reason should be skipped — no API call, no unblock"
3145 );
3146 }
3147
3148 #[tokio::test]
3149 async fn unblock_parents_unblocks_internal_parent_from_store_children() {
3150 let mock = MockBackend::new();
3151 let get_sub_issues_calls = mock.get_sub_issues_calls.clone();
3152 let status_updates = mock.status_updates.clone();
3153 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3154 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3155 let task_manager = Arc::new(TaskManager::with_store(
3156 backend.clone(),
3157 store.clone(),
3158 "test/repo".to_string(),
3159 ));
3160
3161 let parent_id = store
3162 .create_internal("test/repo", "Parent", "", "job", "daily", None)
3163 .await
3164 .unwrap();
3165 store
3166 .update_status(parent_id, crate::store::TaskStatus::Blocked)
3167 .await
3168 .unwrap();
3169 let child_id = store
3170 .create(&crate::store::NewTask {
3171 external_id: Some("91".to_string()),
3172 repo: "test/repo".to_string(),
3173 origin: "github".to_string(),
3174 title: "Child".to_string(),
3175 body: String::new(),
3176 source: String::new(),
3177 source_id: String::new(),
3178 author: String::new(),
3179 url: String::new(),
3180 labels: vec![],
3181 parent_id: Some(parent_id),
3182 })
3183 .await
3184 .unwrap();
3185 store
3186 .update_status(child_id, crate::store::TaskStatus::Done)
3187 .await
3188 .unwrap();
3189
3190 tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3191 .await
3192 .unwrap();
3193
3194 let parent = store.get(parent_id).await.unwrap();
3195 assert_eq!(parent.status, crate::store::TaskStatus::New);
3196 assert!(
3197 status_updates.lock().unwrap().is_empty(),
3198 "internal parent should be updated in store without backend mirroring"
3199 );
3200 assert!(
3201 get_sub_issues_calls.lock().unwrap().is_empty(),
3202 "internal parent should not query backend sub-issues"
3203 );
3204 }
3205
3206 #[cfg(test)]
3210 async fn set_task_updated_at_past(store: &TaskStore, task_id: i64) {
3211 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
3212 .bind(task_id)
3213 .execute(store.pool())
3214 .await
3215 .unwrap();
3216 }
3217
3218 #[tokio::test]
3219 async fn recover_stuck_tasks_resets_external_in_review_to_needs_review() {
3220 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3221 let mock = MockBackend::new();
3222 let status_updates = mock.status_updates.clone();
3223 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3224 let task_manager = Arc::new(TaskManager::with_store(
3225 backend.clone(),
3226 store.clone(),
3227 "owner/repo".to_string(),
3228 ));
3229 let tmux = Arc::new(TmuxManager::new());
3230 let config = EngineConfig {
3231 no_session_stuck_timeout: 600,
3232 stuck_timeout: 1800,
3233 ..EngineConfig::default()
3234 };
3235
3236 let id = store
3238 .upsert_external(&crate::store::UpsertExternal {
3239 repo: "owner/repo",
3240 ext_id: "99",
3241 title: "Stuck InReview",
3242 body: "",
3243 author: "",
3244 url: "",
3245 labels: &[],
3246 origin: "github",
3247 })
3248 .await
3249 .unwrap();
3250 store
3251 .update_status(id, crate::store::TaskStatus::InReview)
3252 .await
3253 .unwrap();
3254 set_review_session_expected(&store, "owner/repo", "99", true).await;
3255 set_task_updated_at_past(&store, id).await;
3256
3257 tick_recover_stuck_tasks(
3258 &backend,
3259 &tmux,
3260 "owner/repo",
3261 &task_manager,
3262 &config,
3263 &store,
3264 &std::collections::HashMap::new(),
3265 &cooled_review_router_for_test(),
3266 &Arc::new(DashMap::new()),
3267 &Arc::new(DashSet::new()),
3268 )
3269 .await
3270 .unwrap();
3271
3272 let updates = status_updates.lock().unwrap();
3273 assert_eq!(updates.len(), 1, "stuck InReview task should be recovered");
3274 assert_eq!(updates[0].0, "99");
3275 assert_eq!(updates[0].1, Status::NeedsReview);
3276 }
3277
3278 #[tokio::test]
3279 async fn recover_stuck_tasks_skips_recent_in_review() {
3280 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3281 let mock = MockBackend::new();
3282 let status_updates = mock.status_updates.clone();
3283 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3284 let task_manager = Arc::new(TaskManager::with_store(
3285 backend.clone(),
3286 store.clone(),
3287 "owner/repo".to_string(),
3288 ));
3289 let tmux = Arc::new(TmuxManager::new());
3290 let config = EngineConfig {
3291 no_session_stuck_timeout: 600, stuck_timeout: 1800,
3293 ..EngineConfig::default()
3294 };
3295
3296 let id = store
3298 .upsert_external(&crate::store::UpsertExternal {
3299 repo: "owner/repo",
3300 ext_id: "100",
3301 title: "Recent InReview",
3302 body: "",
3303 author: "",
3304 url: "",
3305 labels: &[],
3306 origin: "github",
3307 })
3308 .await
3309 .unwrap();
3310 store
3311 .update_status(id, crate::store::TaskStatus::InReview)
3312 .await
3313 .unwrap();
3314 tick_recover_stuck_tasks(
3317 &backend,
3318 &tmux,
3319 "owner/repo",
3320 &task_manager,
3321 &config,
3322 &store,
3323 &std::collections::HashMap::new(),
3324 &cooled_review_router_for_test(),
3325 &Arc::new(DashMap::new()),
3326 &Arc::new(DashSet::new()),
3327 )
3328 .await
3329 .unwrap();
3330
3331 let updates = status_updates.lock().unwrap();
3332 assert!(
3333 updates.is_empty(),
3334 "recent InReview task should not be recovered"
3335 );
3336 }
3337
3338 #[tokio::test]
3339 async fn recover_stuck_tasks_resets_internal_in_review_to_needs_review() {
3340 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3341 let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
3342 let task_manager = Arc::new(TaskManager::with_store(
3343 backend.clone(),
3344 store.clone(),
3345 "owner/repo".to_string(),
3346 ));
3347 let tmux = Arc::new(TmuxManager::new());
3348 let config = EngineConfig {
3349 no_session_stuck_timeout: 600,
3350 stuck_timeout: 1800,
3351 ..EngineConfig::default()
3352 };
3353
3354 let id = store
3356 .create_internal("owner/repo", "Internal InReview", "", "cron", "1", None)
3357 .await
3358 .unwrap();
3359 store
3360 .update_status(id, crate::store::TaskStatus::InReview)
3361 .await
3362 .unwrap();
3363 set_review_session_expected(&store, "owner/repo", "internal:1", true).await;
3364 set_task_updated_at_past(&store, id).await;
3365
3366 tick_recover_stuck_tasks(
3367 &backend,
3368 &tmux,
3369 "owner/repo",
3370 &task_manager,
3371 &config,
3372 &store,
3373 &std::collections::HashMap::new(),
3374 &cooled_review_router_for_test(),
3375 &Arc::new(DashMap::new()),
3376 &Arc::new(DashSet::new()),
3377 )
3378 .await
3379 .unwrap();
3380
3381 let task = store.get(id).await.unwrap();
3382 assert_eq!(
3383 task.status,
3384 crate::store::TaskStatus::NeedsReview,
3385 "stuck internal InReview task should be reset to NeedsReview"
3386 );
3387 }
3388
3389 #[tokio::test]
3390 async fn recover_stuck_tasks_resets_cross_repo_external_in_progress_to_new() {
3391 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3392 let mock = MockBackend::new();
3393 let status_updates = mock.status_updates.clone();
3394 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3395 let task_manager = Arc::new(TaskManager::with_store(
3396 backend.clone(),
3397 store.clone(),
3398 "owner/repo".to_string(),
3399 ));
3400 let tmux = Arc::new(TmuxManager::new());
3401 let config = EngineConfig {
3402 no_session_stuck_timeout: 600,
3403 stuck_timeout: 1800,
3404 ..EngineConfig::default()
3405 };
3406
3407 let id = store
3409 .upsert_external(&crate::store::UpsertExternal {
3410 repo: "owner/other",
3411 ext_id: "777",
3412 title: "Cross-repo stuck task",
3413 body: "",
3414 author: "",
3415 url: "",
3416 labels: &[],
3417 origin: "github",
3418 })
3419 .await
3420 .unwrap();
3421 store
3422 .update_status(id, crate::store::TaskStatus::InProgress)
3423 .await
3424 .unwrap();
3425 set_task_updated_at_past(&store, id).await;
3426
3427 tick_recover_stuck_tasks(
3428 &backend,
3429 &tmux,
3430 "owner/repo",
3431 &task_manager,
3432 &config,
3433 &store,
3434 &std::collections::HashMap::new(),
3435 &cooled_review_router_for_test(),
3436 &Arc::new(DashMap::new()),
3437 &Arc::new(DashSet::new()),
3438 )
3439 .await
3440 .unwrap();
3441
3442 let task = store.get(id).await.unwrap();
3443 assert_eq!(
3444 task.status,
3445 crate::store::TaskStatus::New,
3446 "cross-repo stale in_progress task should be reset to New"
3447 );
3448 assert_eq!(task.route_attempts, 0, "route attempts should be cleared");
3449 assert!(task.agent.is_none(), "agent should be cleared");
3450 assert!(task.model.is_none(), "model should be cleared");
3451
3452 assert!(
3454 status_updates.lock().unwrap().is_empty(),
3455 "cross-repo recovery should not call backend update_status"
3456 );
3457 }
3458
3459 #[tokio::test]
3460 async fn recover_stuck_tasks_finalizes_open_task_runs_rows() {
3461 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3462 let mock = MockBackend::new();
3463 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3464 let task_manager = Arc::new(TaskManager::with_store(
3465 backend.clone(),
3466 store.clone(),
3467 "owner/repo".to_string(),
3468 ));
3469 let tmux = Arc::new(TmuxManager::new());
3470 let config = EngineConfig {
3471 no_session_stuck_timeout: 600,
3472 stuck_timeout: 1800,
3473 ..EngineConfig::default()
3474 };
3475
3476 let id = store
3477 .create_internal("owner/repo", "Stuck with open run", "", "cron", "1", None)
3478 .await
3479 .unwrap();
3480 store
3481 .set_fields(
3482 id,
3483 &[
3484 ("agent", serde_json::json!("claude")),
3485 ("model", serde_json::json!("sonnet")),
3486 ("route_attempts", serde_json::json!(1)),
3487 ],
3488 )
3489 .await
3490 .unwrap();
3491 store
3492 .update_status(id, crate::store::TaskStatus::InProgress)
3493 .await
3494 .unwrap();
3495 set_task_updated_at_past(&store, id).await;
3496
3497 store
3499 .start_run(&crate::store::StartRun {
3500 task_id: id,
3501 attempt: 1,
3502 run_type: "agent",
3503 agent: "claude",
3504 model: "sonnet",
3505 command: "claude --model sonnet",
3506 prompt: "fix it",
3507 })
3508 .await
3509 .unwrap();
3510
3511 tick_recover_stuck_tasks(
3512 &backend,
3513 &tmux,
3514 "owner/repo",
3515 &task_manager,
3516 &config,
3517 &store,
3518 &std::collections::HashMap::new(),
3519 &cooled_review_router_for_test(),
3520 &Arc::new(DashMap::new()),
3521 &Arc::new(DashSet::new()),
3522 )
3523 .await
3524 .unwrap();
3525
3526 let task = store.get(id).await.unwrap();
3527 assert_eq!(task.status, crate::store::TaskStatus::New);
3528
3529 let runs = store.get_runs(id).await.unwrap();
3530 assert_eq!(runs.len(), 1);
3531 assert_eq!(runs[0].outcome, "aborted");
3532 assert!(runs[0].completed_at.is_some());
3533 assert_eq!(
3534 runs[0].error,
3535 "stuck-task recovery: internal no session found"
3536 );
3537 }
3538
3539 #[tokio::test]
3556 async fn recover_stuck_tasks_skips_reclaim_while_dispatch_guard_held() {
3557 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3558 let mock = MockBackend::new();
3559 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3560 let task_manager = Arc::new(TaskManager::with_store(
3561 backend.clone(),
3562 store.clone(),
3563 "owner/repo".to_string(),
3564 ));
3565 let tmux = Arc::new(TmuxManager::new());
3566 let config = EngineConfig {
3567 no_session_stuck_timeout: 600,
3568 stuck_timeout: 1800,
3569 ..EngineConfig::default()
3570 };
3571
3572 let id = store
3573 .create_internal(
3574 "owner/repo",
3575 "Long-running daily job",
3576 "",
3577 "cron",
3578 "1",
3579 None,
3580 )
3581 .await
3582 .unwrap();
3583 store
3584 .set_fields(
3585 id,
3586 &[
3587 ("agent", serde_json::json!("claude")),
3588 ("model", serde_json::json!("sonnet")),
3589 ("route_attempts", serde_json::json!(1)),
3590 ],
3591 )
3592 .await
3593 .unwrap();
3594 store
3595 .update_status(id, crate::store::TaskStatus::InProgress)
3596 .await
3597 .unwrap();
3598 set_task_updated_at_past(&store, id).await;
3601
3602 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3605 dispatching.insert(
3606 format!("owner/repo/internal:{id}"),
3607 format!("internal:{id}"),
3608 );
3609
3610 tick_recover_stuck_tasks(
3611 &backend,
3612 &tmux,
3613 "owner/repo",
3614 &task_manager,
3615 &config,
3616 &store,
3617 &std::collections::HashMap::new(),
3618 &cooled_review_router_for_test(),
3619 &dispatching,
3620 &Arc::new(DashSet::new()),
3621 )
3622 .await
3623 .unwrap();
3624
3625 let task = store.get(id).await.unwrap();
3626 assert_eq!(
3627 task.status,
3628 crate::store::TaskStatus::InProgress,
3629 "task should not be reclaimed while its dispatch guard is still held"
3630 );
3631
3632 let runs = store.get_runs(id).await.unwrap();
3634 assert!(runs.is_empty());
3635
3636 dispatching.clear();
3639 tick_recover_stuck_tasks(
3640 &backend,
3641 &tmux,
3642 "owner/repo",
3643 &task_manager,
3644 &config,
3645 &store,
3646 &std::collections::HashMap::new(),
3647 &cooled_review_router_for_test(),
3648 &dispatching,
3649 &Arc::new(DashSet::new()),
3650 )
3651 .await
3652 .unwrap();
3653 let task = store.get(id).await.unwrap();
3654 assert_eq!(
3655 task.status,
3656 crate::store::TaskStatus::New,
3657 "task should still be reclaimed once the dispatch guard is released"
3658 );
3659 }
3660
3661 #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
3677 async fn recover_stuck_tasks_never_reclaims_while_guard_held_under_real_concurrency() {
3678 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3679 let mock = MockBackend::new();
3680 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3681 let task_manager = Arc::new(TaskManager::with_store(
3682 backend.clone(),
3683 store.clone(),
3684 "owner/repo".to_string(),
3685 ));
3686 let tmux = Arc::new(TmuxManager::new());
3687 let config = EngineConfig {
3688 no_session_stuck_timeout: 600,
3689 stuck_timeout: 1800,
3690 ..EngineConfig::default()
3691 };
3692
3693 let id = store
3694 .create_internal(
3695 "owner/repo",
3696 "Long-running daily job",
3697 "",
3698 "cron",
3699 "1",
3700 None,
3701 )
3702 .await
3703 .unwrap();
3704 store
3705 .set_fields(
3706 id,
3707 &[
3708 ("agent", serde_json::json!("claude")),
3709 ("model", serde_json::json!("sonnet")),
3710 ("route_attempts", serde_json::json!(1)),
3711 ],
3712 )
3713 .await
3714 .unwrap();
3715 store
3716 .update_status(id, crate::store::TaskStatus::InProgress)
3717 .await
3718 .unwrap();
3719 set_task_updated_at_past(&store, id).await;
3720
3721 let dispatch_key = format!("owner/repo/internal:{id}");
3722 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3723
3724 let claim_task_id = format!("internal:{id}");
3731 {
3732 use dashmap::mapref::entry::Entry;
3733 match dispatching.entry(dispatch_key.clone()) {
3734 Entry::Vacant(slot) => {
3735 slot.insert(claim_task_id);
3736 }
3737 Entry::Occupied(_) => panic!("dispatch key already claimed"),
3738 }
3739 }
3740 let guard = DispatchGuard::new(Arc::clone(&dispatching), dispatch_key.clone());
3741
3742 let (release_tx, release_rx) = tokio::sync::oneshot::channel::<()>();
3753 let holder = tokio::spawn(async move {
3754 let _ = release_rx.await;
3755 drop(guard);
3756 });
3757
3758 const HAMMER_ITERATIONS: u32 = 300;
3766 for i in 0..HAMMER_ITERATIONS {
3767 tick_recover_stuck_tasks(
3768 &backend,
3769 &tmux,
3770 "owner/repo",
3771 &task_manager,
3772 &config,
3773 &store,
3774 &std::collections::HashMap::new(),
3775 &cooled_review_router_for_test(),
3776 &dispatching,
3777 &Arc::new(DashSet::new()),
3778 )
3779 .await
3780 .unwrap();
3781
3782 let task = store.get(id).await.unwrap();
3783 assert_eq!(
3784 task.status,
3785 crate::store::TaskStatus::InProgress,
3786 "task must never be reclaimed while the dispatch guard is deterministically \
3787 still held, even under real cross-thread concurrency (dispatch_key={dispatch_key}, \
3788 iteration={i}, contains_key_now={}, map_len_now={})",
3789 dispatching.contains_key(&dispatch_key),
3790 dispatching.len(),
3791 );
3792 tokio::task::yield_now().await;
3793 }
3794
3795 let _ = release_tx.send(());
3797 holder.await.unwrap();
3798
3799 tick_recover_stuck_tasks(
3801 &backend,
3802 &tmux,
3803 "owner/repo",
3804 &task_manager,
3805 &config,
3806 &store,
3807 &std::collections::HashMap::new(),
3808 &cooled_review_router_for_test(),
3809 &dispatching,
3810 &Arc::new(DashSet::new()),
3811 )
3812 .await
3813 .unwrap();
3814 let task = store.get(id).await.unwrap();
3815 assert_eq!(
3816 task.status,
3817 crate::store::TaskStatus::New,
3818 "task should be reclaimed once the dispatch guard is released"
3819 );
3820 }
3821
3822 #[tokio::test]
3823 async fn recover_stuck_tasks_skips_external_in_review_waiting_for_human_review() {
3824 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3825 let mock = MockBackend::new();
3826 let status_updates = mock.status_updates.clone();
3827 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3828 let task_manager = Arc::new(TaskManager::with_store(
3829 backend.clone(),
3830 store.clone(),
3831 "owner/repo".to_string(),
3832 ));
3833 let tmux = Arc::new(TmuxManager::new());
3834 let config = EngineConfig {
3835 no_session_stuck_timeout: 600,
3836 stuck_timeout: 1800,
3837 ..EngineConfig::default()
3838 };
3839
3840 let id = store
3841 .upsert_external(&crate::store::UpsertExternal {
3842 repo: "owner/repo",
3843 ext_id: "101",
3844 title: "Waiting For Human Review",
3845 body: "",
3846 author: "",
3847 url: "",
3848 labels: &[],
3849 origin: "github",
3850 })
3851 .await
3852 .unwrap();
3853 store
3854 .update_status(id, crate::store::TaskStatus::InReview)
3855 .await
3856 .unwrap();
3857 set_task_updated_at_past(&store, id).await;
3858
3859 tick_recover_stuck_tasks(
3860 &backend,
3861 &tmux,
3862 "owner/repo",
3863 &task_manager,
3864 &config,
3865 &store,
3866 &std::collections::HashMap::new(),
3867 &cooled_review_router_for_test(),
3868 &Arc::new(DashMap::new()),
3869 &Arc::new(DashSet::new()),
3870 )
3871 .await
3872 .unwrap();
3873
3874 let updates = status_updates.lock().unwrap();
3875 assert!(
3876 updates.is_empty(),
3877 "InReview task waiting for human review should not be reset"
3878 );
3879 }
3880
3881 #[tokio::test]
3882 async fn unblock_parents_no_blocked_tasks() {
3883 let mock = MockBackend::new(); let status_updates = mock.status_updates.clone();
3885 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3886 let task_manager = make_task_manager(backend.clone());
3887 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3888
3889 tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3890 .await
3891 .unwrap();
3892
3893 assert!(status_updates.lock().unwrap().is_empty());
3894 }
3895
3896 #[tokio::test]
3897 async fn unblock_parents_handles_multiple_blocked_tasks() {
3898 let mut mock = MockBackend::new();
3899
3900 let t40 = make_task("40", &["status:blocked"]);
3902 mock.blocked_tasks.push(t40);
3903 mock.sub_issues
3904 .insert("40".to_string(), vec![ExternalId("41".to_string())]);
3905 mock.tasks_by_id
3906 .insert("41".to_string(), make_task("41", &["status:done"]));
3907
3908 let t50 = make_task("50", &["status:blocked"]);
3910 mock.blocked_tasks.push(t50);
3911 mock.sub_issues
3912 .insert("50".to_string(), vec![ExternalId("51".to_string())]);
3913 mock.tasks_by_id
3914 .insert("51".to_string(), make_task("51", &["status:in_progress"]));
3915
3916 let status_updates = mock.status_updates.clone();
3917 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3918 let task_manager = make_task_manager(backend.clone());
3919 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3920
3921 tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3922 .await
3923 .unwrap();
3924
3925 let updates = status_updates.lock().unwrap();
3926 assert_eq!(updates.len(), 1, "only one parent should be unblocked");
3927 assert_eq!(updates[0].0, "40", "task 40 should be unblocked");
3928 assert_eq!(updates[0].1, Status::New);
3929 }
3930
3931 #[tokio::test]
3932 async fn detect_silent_agents_skips_backend_for_internal_tasks() {
3933 let transport = Arc::new(Transport::new());
3934 let capture = Arc::new(CaptureService::new(transport));
3935 let mock = Arc::new(MockBackend::new());
3936 let backend: Arc<dyn ExternalBackend> = mock.clone();
3937 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3938 let task_manager = Arc::new(TaskManager::with_store(
3939 backend.clone(),
3940 store.clone(),
3941 "owner/repo".to_string(),
3942 ));
3943 let tmux = Arc::new(TmuxManager::new());
3944 let config = EngineConfig {
3945 silence_grace_period: 0,
3946 ..EngineConfig::default()
3947 };
3948
3949 let internal_id = store
3950 .create_internal("owner/repo", "Silent internal", "", "cron", "1", None)
3951 .await
3952 .unwrap();
3953 store
3954 .update_status(internal_id, crate::store::TaskStatus::InProgress)
3955 .await
3956 .unwrap();
3957
3958 let task_id = format!("internal:{internal_id}");
3959 capture
3960 .register_session("owner/repo", &task_id, "orch-test-internal")
3961 .await;
3962 tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
3963
3964 tick_detect_silent_agents(
3965 &tmux,
3966 "owner/repo",
3967 &capture,
3968 &backend,
3969 &task_manager,
3970 &config,
3971 &store,
3972 )
3973 .await
3974 .unwrap();
3975
3976 assert!(
3977 mock.removed_labels.lock().unwrap().is_empty(),
3978 "internal tasks should skip backend label removal"
3979 );
3980 assert!(
3981 mock.posted_comments.lock().unwrap().is_empty(),
3982 "internal tasks should skip backend comments"
3983 );
3984
3985 let task = store.get(internal_id).await.unwrap();
3986 assert!(
3990 matches!(
3991 task.status,
3992 crate::store::TaskStatus::Routed | crate::store::TaskStatus::New
3993 ),
3994 "internal task should be routed to fallback or reset to new, got {:?}",
3995 task.status
3996 );
3997 }
3998
3999 #[tokio::test]
4000 async fn detect_silent_agents_finalizes_open_task_runs_rows() {
4001 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4002 let mock = MockBackend::new();
4003 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4004 let task_manager = Arc::new(TaskManager::with_store(
4005 backend.clone(),
4006 store.clone(),
4007 "owner/repo".to_string(),
4008 ));
4009 let transport = Arc::new(Transport::new());
4010 let capture = Arc::new(CaptureService::new(transport));
4011 let tmux = Arc::new(TmuxManager::new());
4012 let config = EngineConfig {
4013 silence_grace_period: 60,
4014 silence_cooldown: 300,
4015 ..EngineConfig::default()
4016 };
4017
4018 let internal_id = store
4019 .create_internal("owner/repo", "Silent with open run", "", "cron", "1", None)
4020 .await
4021 .unwrap();
4022 store
4023 .set_fields(
4024 internal_id,
4025 &[
4026 ("agent", serde_json::json!("claude")),
4027 ("model", serde_json::json!("sonnet")),
4028 ],
4029 )
4030 .await
4031 .unwrap();
4032 store
4033 .update_status(internal_id, crate::store::TaskStatus::InProgress)
4034 .await
4035 .unwrap();
4036
4037 let task_id = format!("internal:{internal_id}");
4038 capture
4039 .register_session("owner/repo", &task_id, "orch-test-silent-run")
4040 .await;
4041 capture
4042 .set_buffer_state_for_test(
4043 "owner/repo",
4044 &task_id,
4045 false,
4046 false,
4047 chrono::Utc::now() - chrono::Duration::seconds(120),
4048 )
4049 .await;
4050
4051 store
4053 .start_run(&crate::store::StartRun {
4054 task_id: internal_id,
4055 attempt: 1,
4056 run_type: "agent",
4057 agent: "claude",
4058 model: "sonnet",
4059 command: "claude --model sonnet",
4060 prompt: "fix it",
4061 })
4062 .await
4063 .unwrap();
4064
4065 tick_detect_silent_agents(
4066 &tmux,
4067 "owner/repo",
4068 &capture,
4069 &backend,
4070 &task_manager,
4071 &config,
4072 &store,
4073 )
4074 .await
4075 .unwrap();
4076
4077 let runs = store.get_runs(internal_id).await.unwrap();
4078 assert_eq!(runs.len(), 1);
4079 assert_eq!(runs[0].outcome, "aborted");
4080 assert!(runs[0].completed_at.is_some());
4081 assert_eq!(runs[0].error, "silence detection: session killed");
4082 }
4083
4084 #[tokio::test]
4095 async fn phase1_session_completion_touch_prevents_phase2_reclaim() {
4096 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4097 let mock = MockBackend::new();
4098 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4099 let task_manager = Arc::new(TaskManager::with_store(
4100 backend.clone(),
4101 store.clone(),
4102 "owner/repo".to_string(),
4103 ));
4104 let tmux = Arc::new(TmuxManager::new());
4105 let config = EngineConfig {
4106 no_session_stuck_timeout: 600,
4107 stuck_timeout: 1800,
4108 ..EngineConfig::default()
4109 };
4110
4111 let id = store
4115 .create_internal("owner/repo", "Completed task", "", "cron", "1", None)
4116 .await
4117 .unwrap();
4118 store
4119 .update_status(id, crate::store::TaskStatus::InProgress)
4120 .await
4121 .unwrap();
4122 set_task_updated_at_past(&store, id).await;
4123
4124 let task_id = format!("internal:{id}");
4127 store_touch_updated_at(&Some(Arc::clone(&store)), "owner/repo", &task_id).await;
4128
4129 tick_recover_stuck_tasks(
4131 &backend,
4132 &tmux,
4133 "owner/repo",
4134 &task_manager,
4135 &config,
4136 &store,
4137 &std::collections::HashMap::new(),
4138 &cooled_review_router_for_test(),
4139 &Arc::new(DashMap::new()),
4140 &Arc::new(DashSet::new()),
4141 )
4142 .await
4143 .unwrap();
4144
4145 let task = store.get(id).await.unwrap();
4146 assert_eq!(
4147 task.status,
4148 crate::store::TaskStatus::InProgress,
4149 "Phase 2 must not reclaim a task whose updated_at was just touched by Phase 1"
4150 );
4151 }
4152
4153 #[tokio::test]
4154 async fn stuck_task_timing_treats_dead_session_as_missing() {
4155 let tmux = Arc::new(TmuxManager::new());
4156 let repo = "owner/repo";
4157 let task_id = "internal:33498";
4158 let session_name = tmux.session_name(repo, task_id);
4159
4160 let create_result = tokio::process::Command::new("tmux")
4161 .args(["new-session", "-d", "-s", &session_name, "-c", "/tmp"])
4162 .output()
4163 .await;
4164
4165 match create_result {
4166 Ok(o) if o.status.success() => {}
4167 _ => {
4168 eprintln!("Skipping test: tmux not available or failed to create test session");
4169 return;
4170 }
4171 }
4172
4173 let _guard = SessionGuard(session_name.clone());
4174
4175 let set_option_result = tokio::process::Command::new("tmux")
4176 .args(["set-option", "-t", &session_name, "remain-on-exit", "on"])
4177 .output()
4178 .await;
4179 if !matches!(set_option_result, Ok(ref o) if o.status.success()) {
4180 eprintln!("Skipping test: unable to set tmux remain-on-exit option");
4181 return;
4182 }
4183
4184 let send_exit_result = tokio::process::Command::new("tmux")
4185 .args(["send-keys", "-t", &session_name, "exit", "Enter"])
4186 .output()
4187 .await;
4188 if !matches!(send_exit_result, Ok(ref o) if o.status.success()) {
4189 eprintln!("Skipping test: unable to exit tmux pane");
4190 return;
4191 }
4192
4193 for _ in 0..20 {
4195 tokio::time::sleep(std::time::Duration::from_millis(100)).await;
4196 if !tmux.session_is_running(&session_name).await {
4197 break;
4198 }
4199 }
4200
4201 let mut session_map = tmux.batch_session_active().await;
4206 for _ in 0..20 {
4207 if !session_map.get(&session_name).copied().unwrap_or(false) {
4208 break;
4209 }
4210 tokio::time::sleep(std::time::Duration::from_millis(100)).await;
4211 session_map = tmux.batch_session_active().await;
4212 }
4213 if session_map.get(&session_name).copied().unwrap_or(false) {
4214 eprintln!("Skipping test: session still alive in batch_session_active after retries (CI tmux lag)");
4215 return;
4216 }
4217
4218 let config = EngineConfig {
4219 no_session_stuck_timeout: 600,
4220 stuck_timeout: 1800,
4221 ..EngineConfig::default()
4222 };
4223 let updated_at = (chrono::Utc::now()
4224 - chrono::Duration::seconds(config.no_session_stuck_timeout as i64 + 5))
4225 .to_rfc3339();
4226
4227 let timing = stuck_task_timing_from_map(
4228 &tmux,
4229 repo,
4230 task_id,
4231 &updated_at,
4232 &config,
4233 "parse failure",
4234 &session_map,
4235 )
4236 .expect("dead session should be treated as missing");
4237
4238 assert!(!timing.has_session);
4239 assert_eq!(timing.threshold, config.no_session_stuck_timeout);
4240 }
4241
4242 #[tokio::test]
4243 #[serial_test::serial(suspend_state)]
4244 async fn stuck_task_timing_discounts_host_suspend_gap() {
4245 super::super::suspend::clear_for_test();
4246
4247 let tmux = Arc::new(TmuxManager::new());
4248 let repo = "owner/repo";
4249 let task_id = "internal:99001";
4250 let session_map = std::collections::HashMap::new();
4251
4252 let config = EngineConfig {
4253 no_session_stuck_timeout: 1800,
4254 stuck_timeout: 1800,
4255 ..EngineConfig::default()
4256 };
4257
4258 let updated_at = (chrono::Utc::now() - chrono::Duration::minutes(33)).to_rfc3339();
4262 let suspend_gap_detected_at = chrono::Utc::now() - chrono::Duration::minutes(1);
4263 super::super::suspend::inject_gap_for_test(
4264 suspend_gap_detected_at,
4265 chrono::Duration::minutes(30),
4266 );
4267
4268 let timing = stuck_task_timing_from_map(
4269 &tmux,
4270 repo,
4271 task_id,
4272 &updated_at,
4273 &config,
4274 "parse failure",
4275 &session_map,
4276 );
4277
4278 assert!(
4279 timing.is_none(),
4280 "task age should be discounted below the stuck threshold once suspend time is subtracted, got {timing:?}"
4281 );
4282
4283 super::super::suspend::clear_for_test();
4284 }
4285
4286 #[tokio::test]
4287 async fn recover_stuck_tasks_reclaims_routed_task_blocked_by_stale_session() {
4288 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4289 let mock = MockBackend::new();
4290 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4291 let task_manager = Arc::new(TaskManager::with_store(
4292 backend.clone(),
4293 store.clone(),
4294 "owner/repo".to_string(),
4295 ));
4296 let tmux = Arc::new(TmuxManager::new());
4297 let config = EngineConfig {
4298 no_session_stuck_timeout: 600,
4299 stuck_timeout: 1,
4300 ..EngineConfig::default()
4301 };
4302
4303 let id = store
4304 .create_internal("owner/repo", "Blocked routed task", "", "cron", "1", None)
4305 .await
4306 .unwrap();
4307 store
4308 .update_status(id, crate::store::TaskStatus::Routed)
4309 .await
4310 .unwrap();
4311 set_task_updated_at_past(&store, id).await;
4312
4313 let task_id = format!("internal:{id}");
4314 let session_name = tmux.session_name("owner/repo", &task_id);
4315 let create_result = tokio::process::Command::new("tmux")
4316 .args([
4317 "new-session",
4318 "-d",
4319 "-s",
4320 &session_name,
4321 "-c",
4322 "/tmp",
4323 "sleep 60",
4324 ])
4325 .output()
4326 .await;
4327
4328 match create_result {
4329 Ok(o) if o.status.success() => {}
4330 _ => {
4331 eprintln!("Skipping test: tmux not available or failed to create test session");
4332 return;
4333 }
4334 }
4335
4336 let session_map = tmux.batch_session_active().await;
4337 assert_eq!(session_map.get(&session_name).copied(), Some(true));
4338
4339 tick_recover_stuck_tasks(
4340 &backend,
4341 &tmux,
4342 "owner/repo",
4343 &task_manager,
4344 &config,
4345 &store,
4346 &session_map,
4347 &cooled_review_router_for_test(),
4348 &Arc::new(DashMap::new()),
4349 &Arc::new(DashSet::new()),
4350 )
4351 .await
4352 .unwrap();
4353
4354 let task = store.get(id).await.unwrap();
4355 assert_eq!(
4356 task.status,
4357 crate::store::TaskStatus::New,
4358 "routed task blocked by a stale session should be reset for re-routing"
4359 );
4360 assert!(
4361 !tmux.session_exists(&session_name).await,
4362 "recovery should kill the stale tmux session so dispatch can proceed"
4363 );
4364 }
4365
4366 #[tokio::test]
4380 async fn dispatch_does_not_deadlock_under_write_lock() {
4381 use crate::engine::router::Router;
4382 use tokio::sync::RwLock;
4383
4384 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4385 let mock = MockBackend::new();
4386 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4387 let task_manager = Arc::new(TaskManager::with_store(
4388 backend.clone(),
4389 store.clone(),
4390 "owner/repo".to_string(),
4391 ));
4392 let tmux = Arc::new(TmuxManager::new());
4393 let semaphore = Arc::new(Semaphore::new(4));
4394 let (weight_tx, _weight_rx) = mpsc::channel(16);
4395 let dispatching = Arc::new(DashMap::new());
4396 let transport = Arc::new(Transport::new());
4397 let capture = Arc::new(CaptureService::new(transport));
4398
4399 let router_config = crate::engine::router::RouterConfig::default();
4400 let router = Router::new(router_config);
4401 let router_arc = Arc::new(RwLock::new(router));
4402
4403 let write_guard = router_arc.write().await;
4407 let dispatch_mode = dispatch_mode_from_router(&write_guard);
4408
4409 let result = tokio::time::timeout(
4410 std::time::Duration::from_secs(2),
4411 tick_dispatch_tasks(
4412 &backend,
4413 &tmux,
4414 "owner/repo",
4415 &Arc::new(crate::engine::runner::TaskRunner::new(
4416 "owner/repo".to_string(),
4417 )),
4418 &capture,
4419 &semaphore,
4420 &task_manager,
4421 &weight_tx,
4422 dispatch_mode,
4423 &dispatching,
4424 &store,
4425 &std::collections::HashMap::new(),
4426 ),
4427 )
4428 .await;
4429
4430 assert!(
4431 result.is_ok(),
4432 "tick_dispatch_tasks deadlocked! It tried to acquire a read lock \
4433 on router_arc while a write lock was already held (issue #1361)"
4434 );
4435 }
4436
4437 #[tokio::test]
4445 async fn dispatch_skips_task_whose_routed_model_entered_cooldown() {
4446 use crate::store::{StoreRoute, TaskStatus as DbStatus, UpsertExternal};
4447
4448 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4449 let mock = MockBackend::new();
4450 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4451 let task_manager = Arc::new(TaskManager::with_store(
4452 backend.clone(),
4453 store.clone(),
4454 "owner/repo".to_string(),
4455 ));
4456 let tmux = Arc::new(TmuxManager::new());
4457 let runner = Arc::new(TaskRunner::new("owner/repo".to_string()));
4458 let semaphore = Arc::new(Semaphore::new(4));
4459 let (weight_tx, _weight_rx) = mpsc::channel(16);
4460 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4461 let transport = Arc::new(Transport::new());
4462 let capture = Arc::new(CaptureService::new(transport));
4463
4464 let id = store
4465 .upsert_external(&UpsertExternal {
4466 repo: "owner/repo",
4467 ext_id: "999",
4468 title: "Stale routed task",
4469 body: "",
4470 author: "user",
4471 url: "https://github.com/owner/repo/issues/999",
4472 labels: &[],
4473 origin: "github",
4474 })
4475 .await
4476 .unwrap();
4477 store
4478 .store_route(&StoreRoute {
4479 id,
4480 agent: "minimax",
4481 model: Some("opus"),
4482 complexity: "medium",
4483 estimate: 0,
4484 reason: "test",
4485 profile: "{}",
4486 skills: "[]",
4487 })
4488 .await
4489 .unwrap();
4490 store.update_status(id, DbStatus::Routed).await.unwrap();
4491
4492 crate::engine::cooldown::set_model_cooldown(
4495 "minimax",
4496 "opus",
4497 300,
4498 "rate_limit_usage_window",
4499 )
4500 .await;
4501
4502 let dispatch_mode = DispatchMode {
4503 is_degraded: false,
4504 healthy_agents: 1,
4505 threshold: 1,
4506 };
4507
4508 tick_dispatch_tasks(
4509 &backend,
4510 &tmux,
4511 "owner/repo",
4512 &runner,
4513 &capture,
4514 &semaphore,
4515 &task_manager,
4516 &weight_tx,
4517 dispatch_mode,
4518 &dispatching,
4519 &store,
4520 &std::collections::HashMap::new(),
4521 )
4522 .await
4523 .unwrap();
4524
4525 let task = store.get(id).await.unwrap();
4526 assert_eq!(
4527 task.status,
4528 DbStatus::Routed,
4529 "task whose routed model is now cooled down must stay Routed, not be dispatched"
4530 );
4531 assert!(
4532 dispatching.is_empty(),
4533 "dispatch guard must be released when skipping a stale-cooldown task"
4534 );
4535 }
4536
4537 #[tokio::test]
4538 async fn dispatch_skips_a_task_with_an_open_dependency() {
4539 use crate::store::{
4540 DepSource, NewDependency, StoreRoute, TaskStatus as DbStatus, UpsertExternal,
4541 };
4542
4543 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4544 let mock = MockBackend::new();
4545 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4546 let task_manager = Arc::new(TaskManager::with_store(
4547 backend.clone(),
4548 store.clone(),
4549 "owner/repo".to_string(),
4550 ));
4551 let tmux = Arc::new(TmuxManager::new());
4552 let runner = Arc::new(TaskRunner::new("owner/repo".to_string()));
4553 let semaphore = Arc::new(Semaphore::new(4));
4554 let (weight_tx, _weight_rx) = mpsc::channel(16);
4555 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4556 let transport = Arc::new(Transport::new());
4557 let capture = Arc::new(CaptureService::new(transport));
4558
4559 let id = store
4560 .upsert_external(&UpsertExternal {
4561 repo: "owner/repo",
4562 ext_id: "42",
4563 title: "Waiting routed task",
4564 body: "",
4565 author: "user",
4566 url: "https://github.com/owner/repo/issues/42",
4567 labels: &[],
4568 origin: "github",
4569 })
4570 .await
4571 .unwrap();
4572 store
4573 .replace_dependencies(
4574 id,
4575 &[NewDependency {
4576 repo: "owner/repo".into(),
4577 number: 7,
4578 source: DepSource::Github,
4579 }],
4580 )
4581 .await
4582 .unwrap();
4583 store
4584 .store_route(&StoreRoute {
4585 id,
4586 agent: "claude",
4587 model: Some("haiku"),
4588 complexity: "simple",
4589 estimate: 0,
4590 reason: "test",
4591 profile: "{}",
4592 skills: "[]",
4593 })
4594 .await
4595 .unwrap();
4596 store.update_status(id, DbStatus::Routed).await.unwrap();
4597
4598 let dispatch_mode = DispatchMode {
4599 is_degraded: false,
4600 healthy_agents: 1,
4601 threshold: 1,
4602 };
4603 tick_dispatch_tasks(
4604 &backend,
4605 &tmux,
4606 "owner/repo",
4607 &runner,
4608 &capture,
4609 &semaphore,
4610 &task_manager,
4611 &weight_tx,
4612 dispatch_mode,
4613 &dispatching,
4614 &store,
4615 &std::collections::HashMap::new(),
4616 )
4617 .await
4618 .unwrap();
4619
4620 let task = store.get(id).await.unwrap();
4621 assert_eq!(task.status, DbStatus::Waiting);
4622 assert!(dispatching.is_empty());
4623 }
4624
4625 #[tokio::test]
4638 async fn completed_session_does_not_cause_stuck_reclaim_race() {
4639 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4640 let mock = MockBackend::new();
4641 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4642 let task_manager = Arc::new(TaskManager::with_store(
4643 backend.clone(),
4644 store.clone(),
4645 "owner/repo".to_string(),
4646 ));
4647 let tmux = Arc::new(TmuxManager::new());
4648 let transport = Arc::new(Transport::new());
4649 let capture = Arc::new(CaptureService::new(transport));
4650
4651 let id = store
4653 .upsert_external(&crate::store::UpsertExternal {
4654 repo: "owner/repo",
4655 ext_id: "200",
4656 title: "Race Task",
4657 body: "",
4658 author: "",
4659 url: "",
4660 labels: &[],
4661 origin: "github",
4662 })
4663 .await
4664 .unwrap();
4665 store
4666 .update_status(id, crate::store::TaskStatus::InProgress)
4667 .await
4668 .unwrap();
4669
4670 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4672 .bind(id)
4673 .execute(store.pool())
4674 .await
4675 .unwrap();
4676
4677 let task_id = "200";
4679 let session_name = tmux.session_name("owner/repo", task_id);
4680 let create_result = tokio::process::Command::new("tmux")
4681 .args(["new-session", "-d", "-s", &session_name, "-c", "/tmp"])
4682 .output()
4683 .await;
4684 match create_result {
4685 Ok(o) if o.status.success() => {}
4686 _ => {
4687 eprintln!("Skipping test: tmux not available or failed to create test session");
4688 return;
4689 }
4690 }
4691 let _guard = SessionGuard(session_name.clone());
4692
4693 let _ = tokio::process::Command::new("tmux")
4695 .args(["set-option", "-t", &session_name, "remain-on-exit", "on"])
4696 .output()
4697 .await;
4698 let _ = tokio::process::Command::new("tmux")
4699 .args(["send-keys", "-t", &session_name, "exit", "Enter"])
4700 .output()
4701 .await;
4702
4703 for _ in 0..20 {
4705 tokio::time::sleep(std::time::Duration::from_millis(100)).await;
4706 if !tmux.session_is_running(&session_name).await {
4707 break;
4708 }
4709 }
4710
4711 let task_key = task_id.to_string();
4713 capture
4714 .register_session("owner/repo", &task_key, &session_name)
4715 .await;
4716
4717 let config = EngineConfig {
4719 no_session_stuck_timeout: 600,
4720 stuck_timeout: 1800,
4721 ..EngineConfig::default()
4722 };
4723
4724 tick_check_session_completions(&tmux, "owner/repo", &capture, &store)
4725 .await
4726 .unwrap();
4727
4728 {
4734 let task_state = store.get(id).await.unwrap();
4735 if task_state.updated_at.starts_with("2020") {
4736 eprintln!(
4737 "Skipping test: tick_check_session_completions did not update stored \
4738 updated_at — session was not in snapshot (CI tmux environment issue)"
4739 );
4740 return;
4741 }
4742 }
4743
4744 tick_recover_stuck_tasks(
4746 &backend,
4747 &tmux,
4748 "owner/repo",
4749 &task_manager,
4750 &config,
4751 &store,
4752 &std::collections::HashMap::new(),
4753 &cooled_review_router_for_test(),
4754 &Arc::new(DashMap::new()),
4755 &Arc::new(DashSet::new()),
4756 )
4757 .await
4758 .unwrap();
4759
4760 let task = store.get(id).await.unwrap();
4762 assert_eq!(
4763 task.status,
4764 crate::store::TaskStatus::InProgress,
4765 "task should not be reclaimed to New"
4766 );
4767 }
4768
4769 #[tokio::test]
4779 async fn session_completion_matches_repo_configured_with_git_suffix() {
4780 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4781 let tmux = Arc::new(TmuxManager::new());
4782 let transport = Arc::new(Transport::new());
4783 let capture = Arc::new(CaptureService::new(transport));
4784
4785 let repo = "owner/repo.git";
4786 let id = store
4787 .upsert_external(&crate::store::UpsertExternal {
4788 repo,
4789 ext_id: "201",
4790 title: "Git-suffix repo task",
4791 body: "",
4792 author: "",
4793 url: "",
4794 labels: &[],
4795 origin: "github",
4796 })
4797 .await
4798 .unwrap();
4799 store
4800 .update_status(id, crate::store::TaskStatus::InProgress)
4801 .await
4802 .unwrap();
4803 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4804 .bind(id)
4805 .execute(store.pool())
4806 .await
4807 .unwrap();
4808
4809 let task_id = "201";
4812 let session_name = tmux.session_name(repo, task_id);
4813 let create_result = tokio::process::Command::new("tmux")
4814 .args(["new-session", "-d", "-s", &session_name, "-c", "/tmp"])
4815 .output()
4816 .await;
4817 match create_result {
4818 Ok(o) if o.status.success() => {}
4819 _ => {
4820 eprintln!("Skipping test: tmux not available or failed to create test session");
4821 return;
4822 }
4823 }
4824 let _guard = SessionGuard(session_name.clone());
4825
4826 let _ = tokio::process::Command::new("tmux")
4827 .args(["set-option", "-t", &session_name, "remain-on-exit", "on"])
4828 .output()
4829 .await;
4830 let _ = tokio::process::Command::new("tmux")
4831 .args(["send-keys", "-t", &session_name, "exit", "Enter"])
4832 .output()
4833 .await;
4834
4835 for _ in 0..20 {
4836 tokio::time::sleep(std::time::Duration::from_millis(100)).await;
4837 if !tmux.session_is_running(&session_name).await {
4838 break;
4839 }
4840 }
4841
4842 capture.register_session(repo, task_id, &session_name).await;
4843
4844 tick_check_session_completions(&tmux, repo, &capture, &store)
4845 .await
4846 .unwrap();
4847
4848 let task_state = store.get(id).await.unwrap();
4849 if task_state.updated_at.starts_with("2020") {
4850 eprintln!(
4851 "Skipping test: tick_check_session_completions did not update stored \
4852 updated_at — session was not in snapshot (CI tmux environment issue)"
4853 );
4854 return;
4855 }
4856 assert!(
4857 !task_state.updated_at.starts_with("2020"),
4858 "tick_check_session_completions must match its own repo's live sessions \
4859 even when the repo is configured with a \".git\" suffix"
4860 );
4861 }
4862
4863 #[tokio::test]
4869 async fn job_scheduler_returns_ok_on_duplicate_job_config() {
4870 let tmp = tempfile::tempdir().unwrap();
4871 let cfg = tmp.path().join(".orch.yml");
4872 std::fs::write(
4873 &cfg,
4874 "jobs:\n - id: trading-scan\n schedule: \"0 9 * * *\"\n task:\n title: Scan\n body: do it\n",
4875 )
4876 .unwrap();
4877 let jobs_dir = tmp.path().join("prompts").join("jobs");
4878 std::fs::create_dir_all(&jobs_dir).unwrap();
4879 std::fs::write(
4880 jobs_dir.join("trading-scan.md"),
4881 "---\nid: trading-scan\nschedule: '0 9 * * *'\ntitle: Trading Scan\n---\n\nDo trading scan.\n",
4882 )
4883 .unwrap();
4884
4885 let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
4886 let result = tick_job_scheduler(&cfg, &backend, None, "test/repo", None).await;
4887 assert!(
4888 result.is_ok(),
4889 "duplicate job id must not propagate as Err: {result:?}"
4890 );
4891 }
4892
4893 #[tokio::test]
4897 async fn job_scheduler_processes_valid_project_when_other_has_duplicate() {
4898 let bad_tmp = tempfile::tempdir().unwrap();
4900 let bad_cfg = bad_tmp.path().join(".orch.yml");
4901 std::fs::write(
4902 &bad_cfg,
4903 "jobs:\n - id: dup\n schedule: \"0 9 * * *\"\n task:\n title: Dup\n body: x\n",
4904 )
4905 .unwrap();
4906 let bad_jobs_dir = bad_tmp.path().join("prompts").join("jobs");
4907 std::fs::create_dir_all(&bad_jobs_dir).unwrap();
4908 std::fs::write(
4909 bad_jobs_dir.join("dup.md"),
4910 "---\nid: dup\nschedule: '0 9 * * *'\ntitle: Dup\n---\n\nbody\n",
4911 )
4912 .unwrap();
4913
4914 let good_tmp = tempfile::tempdir().unwrap();
4916 let good_cfg = good_tmp.path().join(".orch.yml");
4917 std::fs::write(
4918 &good_cfg,
4919 "jobs:\n - id: valid-job\n schedule: \"0 9 * * *\"\n task:\n title: Valid\n body: ok\n",
4920 )
4921 .unwrap();
4922
4923 let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
4924
4925 let bad_result = tick_job_scheduler(&bad_cfg, &backend, None, "test/bad-repo", None).await;
4926 assert!(
4927 bad_result.is_ok(),
4928 "bad project must not abort scheduler: {bad_result:?}"
4929 );
4930
4931 let good_result =
4932 tick_job_scheduler(&good_cfg, &backend, None, "test/good-repo", None).await;
4933 assert!(
4934 good_result.is_ok(),
4935 "valid project must still run after bad project: {good_result:?}"
4936 );
4937 }
4938
4939 #[serial_test::serial(cooldown_state)]
4947 #[tokio::test]
4948 async fn global_sweep_routes_inactive_repo_tasks() {
4949 use crate::engine::router::{Router, RouterConfig};
4950 use crate::store::{TaskStatus, UpsertExternal};
4951
4952 crate::engine::cooldown::reset_global_state().await;
4953
4954 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4955
4956 let active_id = store
4958 .upsert_external(&UpsertExternal {
4959 repo: "active/repo",
4960 ext_id: "1",
4961 title: "Active task",
4962 body: "",
4963 author: "user",
4964 url: "https://github.com/active/repo/issues/1",
4965 labels: &[],
4966 origin: "github",
4967 })
4968 .await
4969 .unwrap();
4970
4971 let inactive_id = store
4973 .upsert_external(&UpsertExternal {
4974 repo: "inactive/repo",
4975 ext_id: "42",
4976 title: "Orphaned task",
4977 body: "",
4978 author: "user",
4979 url: "https://github.com/inactive/repo/issues/42",
4980 labels: &[],
4981 origin: "github",
4982 })
4983 .await
4984 .unwrap();
4985
4986 assert_eq!(store.get(active_id).await.unwrap().status, TaskStatus::New);
4987 assert_eq!(
4988 store.get(inactive_id).await.unwrap().status,
4989 TaskStatus::New
4990 );
4991
4992 let mut model_map: std::collections::HashMap<
4993 String,
4994 std::collections::HashMap<String, Vec<String>>,
4995 > = std::collections::HashMap::new();
4996 let mut tier = std::collections::HashMap::new();
4997 tier.insert("claude".to_string(), vec!["haiku".to_string()]);
4998 model_map.insert("medium".to_string(), tier);
4999 let config = RouterConfig {
5000 mode: "round_robin".to_string(),
5001 agents: vec!["claude".to_string()],
5002 model_map,
5003 ..Default::default()
5004 };
5005 let mut router = Router::new_for_test(config, vec!["claude".to_string()]);
5006
5007 let active_repos: std::collections::HashSet<String> =
5008 ["active/repo".to_string()].into_iter().collect();
5009 let mut routing_quota = RoutingTickQuota::new(1);
5010 route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
5011 .await
5012 .unwrap();
5013
5014 assert_eq!(
5016 store.get(active_id).await.unwrap().status,
5017 TaskStatus::New,
5018 "global sweep must not touch tasks for the active repo"
5019 );
5020
5021 let after = store.get(inactive_id).await.unwrap();
5023 assert_eq!(
5024 after.status,
5025 TaskStatus::Routed,
5026 "global sweep must route tasks from inactive repos"
5027 );
5028 assert!(
5029 matches!(&after.agent, Some(a) if !a.is_empty()),
5030 "agent must be set after routing"
5031 );
5032 }
5033
5034 #[serial_test::serial(cooldown_state)]
5042 #[tokio::test]
5043 async fn global_sweep_does_not_touch_other_active_repo_tasks() {
5044 use crate::engine::router::{Router, RouterConfig};
5045 use crate::store::{TaskStatus, UpsertExternal};
5046
5047 crate::engine::cooldown::reset_global_state().await;
5048
5049 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5050
5051 let other_active_id = store
5054 .upsert_external(&UpsertExternal {
5055 repo: "other-active/repo",
5056 ext_id: "7",
5057 title: "Other active project's task",
5058 body: "",
5059 author: "user",
5060 url: "https://github.com/other-active/repo/issues/7",
5061 labels: &[],
5062 origin: "github",
5063 })
5064 .await
5065 .unwrap();
5066
5067 let inactive_id = store
5069 .upsert_external(&UpsertExternal {
5070 repo: "inactive/repo",
5071 ext_id: "42",
5072 title: "Orphaned task",
5073 body: "",
5074 author: "user",
5075 url: "https://github.com/inactive/repo/issues/42",
5076 labels: &[],
5077 origin: "github",
5078 })
5079 .await
5080 .unwrap();
5081
5082 let mut model_map: std::collections::HashMap<
5083 String,
5084 std::collections::HashMap<String, Vec<String>>,
5085 > = std::collections::HashMap::new();
5086 let mut tier = std::collections::HashMap::new();
5087 tier.insert("claude".to_string(), vec!["haiku".to_string()]);
5088 model_map.insert("medium".to_string(), tier);
5089 let config = RouterConfig {
5090 mode: "round_robin".to_string(),
5091 agents: vec!["claude".to_string()],
5092 model_map,
5093 ..Default::default()
5094 };
5095 let mut router = Router::new_for_test(config, vec!["claude".to_string()]);
5096
5097 let active_repos: std::collections::HashSet<String> =
5101 ["active/repo".to_string(), "other-active/repo".to_string()]
5102 .into_iter()
5103 .collect();
5104 let mut routing_quota = RoutingTickQuota::new(1);
5105 route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
5106 .await
5107 .unwrap();
5108
5109 assert_eq!(
5110 store.get(other_active_id).await.unwrap().status,
5111 TaskStatus::New,
5112 "global sweep run from a different active repo's tick must not route \
5113 another active repo's new tasks — that repo's own tick will route them"
5114 );
5115 assert_eq!(
5116 store.get(inactive_id).await.unwrap().status,
5117 TaskStatus::Routed,
5118 "global sweep must still route tasks from genuinely orphaned repos"
5119 );
5120 }
5121
5122 #[serial_test::serial(cooldown_state)]
5124 #[tokio::test]
5125 async fn global_sweep_skips_no_agent_tasks() {
5126 use crate::engine::router::{Router, RouterConfig};
5127 use crate::store::{TaskStatus, UpsertExternal};
5128
5129 crate::engine::cooldown::reset_global_state().await;
5130
5131 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5132
5133 let id = store
5134 .upsert_external(&UpsertExternal {
5135 repo: "inactive/repo",
5136 ext_id: "99",
5137 title: "No-agent task",
5138 body: "",
5139 author: "user",
5140 url: "https://github.com/inactive/repo/issues/99",
5141 labels: &["no-agent".to_string()],
5142 origin: "github",
5143 })
5144 .await
5145 .unwrap();
5146
5147 let mut model_map: std::collections::HashMap<
5148 String,
5149 std::collections::HashMap<String, Vec<String>>,
5150 > = std::collections::HashMap::new();
5151 let mut tier = std::collections::HashMap::new();
5152 tier.insert("claude".to_string(), vec!["haiku".to_string()]);
5153 model_map.insert("medium".to_string(), tier);
5154 let config = RouterConfig {
5155 mode: "round_robin".to_string(),
5156 agents: vec!["claude".to_string()],
5157 model_map,
5158 ..Default::default()
5159 };
5160 let mut router = Router::new_for_test(config, vec!["claude".to_string()]);
5161
5162 let active_repos: std::collections::HashSet<String> =
5163 ["active/repo".to_string()].into_iter().collect();
5164 let mut routing_quota = RoutingTickQuota::new(1);
5165 route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
5166 .await
5167 .unwrap();
5168
5169 assert_eq!(
5170 store.get(id).await.unwrap().status,
5171 TaskStatus::New,
5172 "no-agent tasks must not be routed"
5173 );
5174 }
5175
5176 #[serial_test::serial(cooldown_state)]
5181 #[tokio::test]
5182 async fn global_sweep_honors_shared_routing_quota() {
5183 use crate::engine::router::{Router, RouterConfig};
5184 use crate::store::{TaskStatus, UpsertExternal};
5185
5186 crate::engine::cooldown::reset_global_state().await;
5187
5188 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5189 for ext_id in ["1", "2"] {
5190 store
5191 .upsert_external(&UpsertExternal {
5192 repo: "inactive/repo",
5193 ext_id,
5194 title: "Orphaned task",
5195 body: "",
5196 author: "user",
5197 url: &format!("https://github.com/inactive/repo/issues/{ext_id}"),
5198 labels: &[],
5199 origin: "github",
5200 })
5201 .await
5202 .unwrap();
5203 }
5204
5205 let mut model_map: std::collections::HashMap<
5206 String,
5207 std::collections::HashMap<String, Vec<String>>,
5208 > = std::collections::HashMap::new();
5209 let mut tier = std::collections::HashMap::new();
5210 tier.insert("claude".to_string(), vec!["haiku".to_string()]);
5211 model_map.insert("medium".to_string(), tier);
5212 let config = RouterConfig {
5213 mode: "round_robin".to_string(),
5214 agents: vec!["claude".to_string()],
5215 model_map,
5216 ..Default::default()
5217 };
5218 let mut router = Router::new_for_test(config, vec!["claude".to_string()]);
5219
5220 let active_repos: std::collections::HashSet<String> =
5221 ["active/repo".to_string()].into_iter().collect();
5222 let mut routing_quota = RoutingTickQuota::new(1);
5223 route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
5224 .await
5225 .unwrap();
5226
5227 let all_tasks = store.list_all_global().await.unwrap();
5228 let routed = all_tasks
5229 .iter()
5230 .filter(|task| task.status == TaskStatus::Routed)
5231 .count();
5232 let still_new = all_tasks
5233 .iter()
5234 .filter(|task| task.status == TaskStatus::New)
5235 .count();
5236
5237 assert_eq!(routed, 1, "only one task may consume the shared quota");
5238 assert_eq!(still_new, 1, "remaining tasks wait for the next tick");
5239
5240 route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
5241 .await
5242 .unwrap();
5243 let after_exhausted = store.list_all_global().await.unwrap();
5244 assert_eq!(
5245 after_exhausted
5246 .iter()
5247 .filter(|task| task.status == TaskStatus::Routed)
5248 .count(),
5249 1,
5250 "an exhausted shared quota must not reset within the same tick"
5251 );
5252 }
5253
5254 #[tokio::test]
5271 async fn group_orphaned_routed_tasks_selects_inactive_repo_tasks() {
5272 use crate::store::{StoreRoute, TaskStatus, UpsertExternal};
5273
5274 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5275
5276 async fn make_routed(store: &TaskStore, repo: &str, ext_id: &str) -> i64 {
5277 let id = store
5278 .upsert_external(&UpsertExternal {
5279 repo,
5280 ext_id,
5281 title: "Routed task",
5282 body: "",
5283 author: "user",
5284 url: &format!("https://github.com/{repo}/issues/{ext_id}"),
5285 labels: &[],
5286 origin: "github",
5287 })
5288 .await
5289 .unwrap();
5290 store
5291 .store_route(&StoreRoute {
5292 id,
5293 agent: "claude",
5294 model: Some("sonnet"),
5295 complexity: "medium",
5296 estimate: 0,
5297 reason: "test",
5298 profile: "{}",
5299 skills: "[]",
5300 })
5301 .await
5302 .unwrap();
5303 store.update_status(id, TaskStatus::Routed).await.unwrap();
5304 id
5305 }
5306
5307 let active_id = make_routed(&store, "active/repo", "1").await;
5308 let inactive_id_a = make_routed(&store, "inactive/repo", "2").await;
5309 let other_inactive_id = make_routed(&store, "other-inactive/repo", "3").await;
5310 let inactive_id_b = make_routed(&store, "inactive/repo", "4").await;
5311
5312 let all_routed = store
5313 .list_all_by_status_global(TaskStatus::Routed)
5314 .await
5315 .unwrap();
5316 let active_repos: std::collections::HashSet<String> =
5317 ["active/repo".to_string()].into_iter().collect();
5318 let by_repo = group_orphaned_routed_tasks(all_routed, &active_repos);
5319
5320 assert!(
5321 !by_repo.contains_key("active/repo"),
5322 "global sweep must not select tasks for the active repo"
5323 );
5324 let inactive_ids: std::collections::HashSet<i64> = by_repo
5325 .get("inactive/repo")
5326 .expect("inactive/repo must be selected")
5327 .iter()
5328 .map(|t| t.id)
5329 .collect();
5330 assert_eq!(
5331 inactive_ids,
5332 [inactive_id_a, inactive_id_b].into_iter().collect(),
5333 "global sweep must group all orphaned tasks for a repo together"
5334 );
5335 assert_eq!(
5336 by_repo
5337 .get("other-inactive/repo")
5338 .map(|tasks| tasks.iter().map(|t| t.id).collect::<Vec<_>>()),
5339 Some(vec![other_inactive_id])
5340 );
5341
5342 assert_eq!(
5345 store.get(active_id).await.unwrap().status,
5346 TaskStatus::Routed
5347 );
5348 }
5349
5350 #[tokio::test]
5355 async fn group_orphaned_routed_tasks_excludes_all_active_repos_not_just_current() {
5356 use crate::store::{StoreRoute, TaskStatus, UpsertExternal};
5357
5358 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5359
5360 async fn make_routed(store: &TaskStore, repo: &str, ext_id: &str) -> i64 {
5361 let id = store
5362 .upsert_external(&UpsertExternal {
5363 repo,
5364 ext_id,
5365 title: "Routed task",
5366 body: "",
5367 author: "user",
5368 url: &format!("https://github.com/{repo}/issues/{ext_id}"),
5369 labels: &[],
5370 origin: "github",
5371 })
5372 .await
5373 .unwrap();
5374 store
5375 .store_route(&StoreRoute {
5376 id,
5377 agent: "claude",
5378 model: Some("sonnet"),
5379 complexity: "medium",
5380 estimate: 0,
5381 reason: "test",
5382 profile: "{}",
5383 skills: "[]",
5384 })
5385 .await
5386 .unwrap();
5387 store.update_status(id, TaskStatus::Routed).await.unwrap();
5388 id
5389 }
5390
5391 let repo_a_id = make_routed(&store, "active/repo-a", "1").await;
5392 let repo_b_id = make_routed(&store, "active/repo-b", "2").await;
5393 let orphaned_id = make_routed(&store, "inactive/repo", "3").await;
5394
5395 let all_routed = store
5396 .list_all_by_status_global(TaskStatus::Routed)
5397 .await
5398 .unwrap();
5399 let active_repos: std::collections::HashSet<String> =
5402 ["active/repo-a".to_string(), "active/repo-b".to_string()]
5403 .into_iter()
5404 .collect();
5405 let by_repo = group_orphaned_routed_tasks(all_routed, &active_repos);
5406
5407 assert!(
5408 !by_repo.contains_key("active/repo-a"),
5409 "the current repo's own tasks must not be swept"
5410 );
5411 assert!(
5412 !by_repo.contains_key("active/repo-b"),
5413 "another active repo's tasks must not be swept during repo A's tick"
5414 );
5415 assert_eq!(
5416 by_repo
5417 .get("inactive/repo")
5418 .map(|tasks| tasks.iter().map(|t| t.id).collect::<Vec<_>>()),
5419 Some(vec![orphaned_id]),
5420 "a genuinely inactive repo's tasks must still be swept"
5421 );
5422
5423 assert_eq!(
5425 store.get(repo_a_id).await.unwrap().status,
5426 TaskStatus::Routed
5427 );
5428 assert_eq!(
5429 store.get(repo_b_id).await.unwrap().status,
5430 TaskStatus::Routed
5431 );
5432 }
5433
5434 #[tokio::test]
5436 async fn global_dispatch_sweep_skips_no_agent_tasks() {
5437 use crate::store::{StoreRoute, TaskStatus, UpsertExternal};
5438
5439 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5440
5441 let id = store
5442 .upsert_external(&UpsertExternal {
5443 repo: "inactive/repo",
5444 ext_id: "99",
5445 title: "No-agent routed task",
5446 body: "",
5447 author: "user",
5448 url: "https://github.com/inactive/repo/issues/99",
5449 labels: &["no-agent".to_string()],
5450 origin: "github",
5451 })
5452 .await
5453 .unwrap();
5454 store
5455 .store_route(&StoreRoute {
5456 id,
5457 agent: "claude",
5458 model: Some("sonnet"),
5459 complexity: "medium",
5460 estimate: 0,
5461 reason: "test",
5462 profile: "{}",
5463 skills: "[]",
5464 })
5465 .await
5466 .unwrap();
5467 store.update_status(id, TaskStatus::Routed).await.unwrap();
5468
5469 let tmux = Arc::new(TmuxManager::new());
5474 let transport = Arc::new(Transport::new());
5475 let capture = Arc::new(CaptureService::new(transport));
5476 let semaphore = Arc::new(Semaphore::new(4));
5477 let (weight_tx, _weight_rx) = mpsc::channel(16);
5478 let dispatching = Arc::new(DashMap::new());
5479 let dispatch_mode = DispatchMode {
5480 is_degraded: false,
5481 healthy_agents: 1,
5482 threshold: 1,
5483 };
5484
5485 let active_repos: std::collections::HashSet<String> =
5486 ["active/repo".to_string()].into_iter().collect();
5487 dispatch_routed_tasks_global(
5488 &tmux,
5489 &capture,
5490 &semaphore,
5491 &weight_tx,
5492 dispatch_mode,
5493 &dispatching,
5494 &store,
5495 &std::collections::HashMap::new(),
5496 &active_repos,
5497 )
5498 .await
5499 .unwrap();
5500
5501 assert_eq!(
5502 store.get(id).await.unwrap().status,
5503 TaskStatus::Routed,
5504 "no-agent tasks must not be dispatched"
5505 );
5506 }
5507}