1use crate::backends::{ExternalBackend, ExternalId, ExternalTask, Status};
17use crate::channels::capture::CaptureService;
18use crate::config;
19use crate::engine::cooldown::{
20 SILENCE_AGENT_COOLDOWN_SECS, SILENCE_EXTENDED_COOLDOWN_SECS, github_circuit_remaining_secs,
21 is_agent_in_cooldown, is_github_circuit_open, is_model_in_cooldown,
22 record_agent_failure_with_message, record_silence_detection, set_agent_cooldown,
23 set_model_cooldown,
24};
25use crate::engine::dispatch_guard::DispatchGuard;
26use crate::engine::jobs;
27use crate::engine::router::{AllAgentsCooledError, RouteResultError, Router, get_route_result};
28use crate::engine::runner::{TaskRunner, WeightSignal};
29use crate::engine::tasks::{TaskManager, is_internal_id};
30use crate::repo_context::REPO_CONTEXT;
31use crate::store::TaskStore;
32use crate::tmux::TmuxManager;
33use dashmap::{DashMap, DashSet};
34use std::sync::Arc;
35use std::time::{Instant, SystemTime};
36use tokio::sync::{Semaphore, mpsc};
37
38pub(crate) struct RoutingTickQuota {
44 remaining: usize,
45}
46
47impl RoutingTickQuota {
48 pub(crate) fn new(max: usize) -> Self {
49 Self { remaining: max }
50 }
51
52 fn is_exhausted(&self) -> bool {
53 self.remaining == 0
54 }
55
56 fn remaining(&self) -> usize {
57 self.remaining
58 }
59
60 fn consume_one(&mut self) -> bool {
61 if self.remaining == 0 {
62 return false;
63 }
64 self.remaining -= 1;
65 true
66 }
67}
68
69pub(crate) async fn startup_cleanup(tmux: &TmuxManager) {
72 let start_time = SystemTime::now()
73 .duration_since(SystemTime::UNIX_EPOCH)
74 .unwrap_or_default()
75 .as_secs();
76 tracing::info!(
77 start_time,
78 "running startup cleanup for stale tmux sessions"
79 );
80
81 match tmux.kill_stale_sessions(start_time).await {
82 Ok(killed) => {
83 if killed > 0 {
84 tracing::info!(killed, "startup cleanup killed stale sessions");
85 } else {
86 tracing::debug!("no stale tmux sessions found on startup");
87 }
88 }
89 Err(e) => {
90 tracing::warn!(error = %e, "failed to run startup cleanup");
91 }
92 }
93}
94
95use super::{EngineConfig, ProjectEngine};
96use crate::store::{store_set_by_id, store_touch_updated_at};
97
98async fn recover_routed_blocked_dispatch(
99 backend: &Arc<dyn ExternalBackend>,
100 store: &Arc<TaskStore>,
101 task_manager: &Arc<TaskManager>,
102 repo: &str,
103 task: &ExternalTask,
104 timing: &StuckTaskTiming,
105) {
106 tracing::warn!(
107 task_id = task.id.0,
108 age_mins = timing.age.num_minutes(),
109 threshold_mins = timing.threshold / 60,
110 "recovering routed task: dispatch blocked by stale tmux session → new"
111 );
112
113 let resolved_store_id = match store.resolve_task_id(repo, &task.id.0).await {
114 Ok(Some(id)) => Some(id),
115 Ok(None) => {
116 tracing::warn!(
117 task_id = %task.id.0,
118 repo,
119 "resolve_task_id returned None during routed-task recovery"
120 );
121 None
122 }
123 Err(e) => {
124 tracing::error!(
125 task_id = %task.id.0,
126 repo,
127 error = %e,
128 "resolve_task_id failed during routed-task recovery"
129 );
130 None
131 }
132 };
133
134 if let Some(store_id) = resolved_store_id {
135 if let Err(e) = store_set_by_id(
136 Some(store),
137 store_id,
138 &[
139 ("agent", serde_json::Value::Null),
140 ("model", serde_json::Value::Null),
141 ("route_attempts", serde_json::json!(0)),
142 ],
143 )
144 .await
145 {
146 tracing::warn!(
147 task_id = task.id.0,
148 ?e,
149 "failed to clear routing fields for routed stuck task — will retry next tick"
150 );
151 return;
152 }
153 finalize_open_runs_for_recovery(
154 store,
155 store_id,
156 "stuck-task recovery: stale session blocking dispatch killed",
157 )
158 .await;
159 }
160
161 if let Err(e) = task_manager.update_task_status(&task.id, Status::New).await {
162 tracing::warn!(
163 task_id = task.id.0,
164 ?e,
165 "failed to reset routed stuck task status"
166 );
167 return;
168 }
169
170 crate::store::store_log_activity(
171 &Some(Arc::clone(store)),
172 repo,
173 &task.id.0,
174 "rerouted",
175 Some("routed"),
176 Some("new"),
177 None::<&str>,
178 None::<&str>,
179 Some(&serde_json::json!({
180 "failure_reason": "routed_blocked_by_existing_session",
181 "age_minutes": timing.age.num_minutes(),
182 "had_session": true,
183 })),
184 )
185 .await;
186
187 if !is_internal_id(&task.id.0) {
188 let backend_clone = Arc::clone(backend);
189 let task_id_clone = task.id.clone();
190 let body = format!(
191 "[{}] recovered: stuck routed task — existing tmux session blocked dispatch for {}m; cleared route for retry{}",
192 chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
193 timing.age.num_minutes(),
194 crate::engine::orch_footer()
195 );
196 tokio::spawn(async move {
197 if let Err(e) = backend_clone.post_comment(&task_id_clone, &body).await {
198 tracing::warn!(
199 task_id = task_id_clone.0,
200 ?e,
201 "failed to post routed-task recovery comment"
202 );
203 }
204 });
205 }
206}
207
208pub(crate) async fn tick_check_session_completions(
210 tmux: &Arc<TmuxManager>,
211 repo: &str,
212 capture: &Arc<CaptureService>,
213 store: &Arc<TaskStore>,
214) -> anyhow::Result<()> {
215 let _span = tracing::info_span!("engine.tick.phase1.sessions").entered();
216 let session_snapshot = tmux.snapshot().await;
217 let repo_name = repo
223 .rsplit('/')
224 .next()
225 .unwrap_or(repo)
226 .trim_end_matches(".git");
227 for (session, active) in session_snapshot {
228 if session.project != repo_name {
230 continue;
231 }
232
233 if !active {
234 tracing::info!(
235 session = %session.name,
236 task_id = %session.task_id,
237 "session completed, collecting results"
238 );
239 let store_task_id = {
250 let base = session
251 .task_id
252 .strip_suffix("-review")
253 .unwrap_or(&session.task_id);
254 if let Some(n) = base.strip_prefix("internal-") {
255 format!("internal:{n}")
256 } else {
257 base.to_string()
258 }
259 };
260 store_touch_updated_at(&Some(Arc::clone(store)), repo, &store_task_id).await;
261 capture.unregister_session(repo, &session.task_id).await;
264 if let Err(e) = tmux.kill_session(&session.name).await {
266 tracing::debug!(
267 session = %session.name,
268 ?e,
269 "kill_session failed (session may already be gone)"
270 );
271 }
272 }
273 }
274 Ok(())
275}
276
277#[derive(Debug)]
278struct StuckTaskTiming {
279 has_session: bool,
280 age: chrono::Duration,
281 threshold: u64,
282}
283
284fn stuck_task_timing_from_map(
285 tmux: &Arc<TmuxManager>,
286 repo: &str,
287 session_task_id: &str,
288 updated_at: &str,
289 config: &EngineConfig,
290 parse_error_message: &'static str,
291 session_map: &std::collections::HashMap<String, bool>,
292) -> Option<StuckTaskTiming> {
293 let session_name = tmux.session_name(repo, session_task_id);
294 let has_session = session_map.get(&session_name).copied().unwrap_or(false);
296 let threshold = if has_session {
297 config.stuck_timeout
298 } else {
299 config.no_session_stuck_timeout
300 };
301
302 let updated = match chrono::DateTime::parse_from_rfc3339(updated_at) {
303 Ok(dt) => dt.with_timezone(&chrono::Utc),
304 Err(e) => {
305 tracing::warn!(
306 task_id = session_task_id,
307 updated_at,
308 ?e,
309 error_message = parse_error_message,
310 "cannot parse updated_at"
311 );
312 return None;
313 }
314 };
315
316 let raw_age = chrono::Utc::now() - updated;
317 let suspended = super::suspend::suspended_duration_since(updated);
321 let age = (raw_age - suspended).max(chrono::Duration::zero());
322 if age.num_seconds() <= i64::try_from(threshold).unwrap_or(i64::MAX) {
323 return None;
324 }
325
326 Some(StuckTaskTiming {
327 has_session,
328 age,
329 threshold,
330 })
331}
332
333async fn in_review_stuck_entry(store: &TaskStore, store_id: i64, fallback: &str) -> (String, bool) {
336 match store.in_review_entry(store_id).await {
337 Ok(Some(entry)) => entry,
338 Ok(None) => (fallback.to_string(), false),
339 Err(e) => {
340 tracing::warn!(
341 store_id,
342 error = %e,
343 "failed to read in_review entry, falling back to updated_at"
344 );
345 (fallback.to_string(), false)
346 }
347 }
348}
349
350async fn finalize_open_runs_for_recovery(store: &TaskStore, store_id: i64, reason: &str) {
353 if let Err(e) = store
354 .finalize_incomplete_runs(store_id, "aborted", reason)
355 .await
356 {
357 tracing::warn!(
358 store_id,
359 error = %e,
360 reason,
361 "failed to finalize incomplete runs during recovery"
362 );
363 }
364}
365
366pub(crate) async fn tick_detect_silent_agents(
375 tmux: &Arc<TmuxManager>,
376 repo: &str,
377 capture: &Arc<CaptureService>,
378 backend: &Arc<dyn ExternalBackend>,
379 task_manager: &Arc<TaskManager>,
380 config: &EngineConfig,
381 store: &Arc<TaskStore>,
382) -> anyhow::Result<()> {
383 let _span = tracing::info_span!("engine.tick.phase1b.silence").entered();
384 let grace = std::time::Duration::from_secs(config.silence_grace_period);
385 let silent_sessions = capture.get_silent_sessions_for_repo(repo, grace).await;
386
387 for (task_id, session_name, silence_age_secs) in silent_sessions {
388 let use_backend = should_use_backend(&task_id);
389 let store_task = match store.resolve_task_id(repo, &task_id).await {
391 Ok(Some(store_id)) => match store.get(store_id).await {
392 Ok(task) => Some(task),
393 Err(e) => {
394 tracing::warn!(
395 task_id,
396 repo,
397 store_id,
398 operation = "store.get",
399 error = %e,
400 "silence detection: db error fetching task — agent/model cooldown will be skipped"
401 );
402 None
403 }
404 },
405 Ok(None) => {
406 tracing::debug!(
407 task_id,
408 repo,
409 operation = "resolve_task_id",
410 "silence detection: task not found in store (may have been cleaned up)"
411 );
412 None
413 }
414 Err(e) => {
415 tracing::warn!(
416 task_id,
417 repo,
418 operation = "resolve_task_id",
419 error = %e,
420 "silence detection: db error resolving task id — agent/model cooldown will be skipped"
421 );
422 None
423 }
424 };
425 let agent_name = store_task
426 .as_ref()
427 .and_then(|t| t.agent.clone())
428 .unwrap_or_default();
429 let model_name = store_task
430 .as_ref()
431 .and_then(|t| t.model.clone())
432 .unwrap_or_default();
433
434 tracing::warn!(
435 task_id,
436 agent = %agent_name,
437 model = %model_name,
438 silence_age_secs,
439 grace_secs = config.silence_grace_period,
440 cooldown_secs = config.silence_cooldown,
441 "agent silent since session start — killing session, cooling down model + agent, failing over"
442 );
443
444 if let Err(e) = tmux.kill_session(&session_name).await {
446 tracing::debug!(
447 task_id,
448 ?e,
449 "kill_session failed for silent agent (may already be gone)"
450 );
451 }
452
453 capture.unregister_session(repo, &task_id).await;
455
456 if let Some(store_id) = store_task.as_ref().map(|t| t.id) {
458 finalize_open_runs_for_recovery(store, store_id, "silence detection: session killed")
459 .await;
460 }
461
462 let mut extended_note = String::new();
463
464 if !agent_name.is_empty() && !model_name.is_empty() {
466 set_model_cooldown(
467 &agent_name,
468 &model_name,
469 config.silence_cooldown,
470 "silence_detected",
471 )
472 .await;
473 if let Some(result) = record_silence_detection(&agent_name, &model_name).await
474 && result.extended_cooldown_applied
475 {
476 extended_note = format!(
477 " ({} silences in 24h -> extended cooldown {}s)",
478 result.count, SILENCE_EXTENDED_COOLDOWN_SECS
479 );
480 }
481 }
482
483 if !agent_name.is_empty() {
487 set_agent_cooldown(&agent_name, SILENCE_AGENT_COOLDOWN_SECS).await;
488 }
489
490 let task_eid = ExternalId(task_id.clone());
495
496 let available: Vec<String> = crate::engine::configured_agents()
498 .into_iter()
499 .filter(|a| crate::cmd_cache::command_exists(a))
500 .collect();
501 let chain = crate::engine::runner::response::get_reroute_chain(
502 &task_id,
503 &Some(Arc::clone(store)),
504 repo,
505 )
506 .await;
507 let chain = crate::engine::runner::response::update_reroute_chain(
508 &task_id,
509 &agent_name,
510 &chain,
511 &Some(Arc::clone(store)),
512 repo,
513 )
514 .await;
515
516 let (next_status, next_agent) = if let Some(fallback) =
517 crate::engine::runner::response::pick_fallback_agent(&agent_name, &chain, &available)
518 {
519 tracing::info!(
520 task_id,
521 from = %agent_name,
522 to = %fallback,
523 "silence detection: failover to different agent, setting routed"
524 );
525 (Status::Routed, Some(fallback))
526 } else {
527 tracing::warn!(
528 task_id,
529 "silence detection: no fallback agents available, resetting to new for re-routing"
530 );
531 (Status::New, None)
532 };
533
534 if use_backend && let Some(ref st) = store_task {
535 let stale_labels: Vec<String> = st
537 .labels
538 .iter()
539 .filter(|l| {
540 l.starts_with("agent:")
541 || l.starts_with("complexity:")
542 || l.starts_with("model:")
543 })
544 .cloned()
545 .collect();
546 if !stale_labels.is_empty() {
547 let backend_clone = Arc::clone(backend);
548 let task_eid_clone = task_eid.clone();
549 tokio::spawn(async move {
550 for label in &stale_labels {
551 if let Err(e) = backend_clone.remove_label(&task_eid_clone, label).await {
552 tracing::warn!(task_id = task_eid_clone.0, label, error = %e, "failed to remove label during silence detection re-route");
553 }
554 }
555 });
556 }
557 }
558
559 if let Some(store_id) = store_task.as_ref().map(|t| t.id) {
560 if let Some(ref fallback) = next_agent {
561 let _ = store_set_by_id(
565 Some(store),
566 store_id,
567 &[
568 ("agent", serde_json::json!(fallback)),
569 ("model", serde_json::json!("")),
570 (
571 "last_error",
572 serde_json::json!(format!(
573 "silence detected after {}s, failing over to {}",
574 config.silence_grace_period, fallback
575 )),
576 ),
577 ],
578 )
579 .await;
580 } else {
581 let _ = store_set_by_id(
582 Some(store),
583 store_id,
584 &[
585 ("agent", serde_json::Value::Null),
586 ("model", serde_json::Value::Null),
587 (
588 "last_error",
589 serde_json::json!(format!(
590 "silence detected after {}s, no fallback agents available",
591 config.silence_grace_period
592 )),
593 ),
594 ],
595 )
596 .await;
597 }
598 }
599
600 if let Err(e) = task_manager
601 .update_task_status(&task_eid, next_status)
602 .await
603 {
604 tracing::warn!(task_id, ?e, "failed to update silent task status");
605 continue;
606 }
607
608 let details = serde_json::json!({
610 "failure_reason": "agent_silence",
611 "silence_duration_secs": config.silence_grace_period,
612 "cooldown_applied": !agent_name.is_empty() && !model_name.is_empty(),
613 "fallback_available": next_agent.is_some(),
614 });
615 crate::store::store_log_activity(
616 &Some(Arc::clone(store)),
617 repo,
618 &task_id,
619 "rerouted",
620 Some("in_progress"),
621 Some(next_status.as_str()),
622 next_agent.as_deref(),
623 None,
624 Some(&details),
625 )
626 .await;
627
628 let action = if let Some(ref fallback) = next_agent {
630 format!("failing over to {fallback}")
631 } else {
632 "marking needs_review (no fallback agents)".to_string()
633 };
634 let comment = format!(
635 "[{}] agent silent for {}s since session start — killed session, cooled down model `{}:{}` for {}s{}, agent `{}` for {}s, {}{}",
636 chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
637 config.silence_grace_period,
638 agent_name,
639 model_name,
640 config.silence_cooldown,
641 extended_note,
642 agent_name,
643 SILENCE_AGENT_COOLDOWN_SECS,
644 action,
645 crate::engine::orch_footer(),
646 );
647 if use_backend {
648 let backend_clone = Arc::clone(backend);
650 let task_eid_clone = task_eid.clone();
651 tokio::spawn(async move {
652 if let Err(e) = backend_clone.post_comment(&task_eid_clone, &comment).await {
653 tracing::warn!(
654 task_id = task_eid_clone.0,
655 ?e,
656 "failed to post silence detection comment"
657 );
658 }
659 });
660 }
661 }
662
663 Ok(())
664}
665
666fn should_use_backend(task_id: &str) -> bool {
667 !is_internal_id(task_id)
668}
669
670#[expect(
675 clippy::too_many_arguments,
676 reason = "each argument is engine state the tick passes down"
677)]
678pub(crate) async fn tick_recover_stuck_tasks(
679 backend: &Arc<dyn ExternalBackend>,
680 tmux: &Arc<TmuxManager>,
681 repo: &str,
682 task_manager: &Arc<TaskManager>,
683 config: &EngineConfig,
684 store: &Arc<TaskStore>,
685 session_map: &std::collections::HashMap<String, bool>,
686 dispatching: &Arc<DashMap<String, String>>,
687) -> anyhow::Result<()> {
688 use crate::store::TaskStatus as DbStatus;
689
690 let _span = tracing::info_span!("engine.tick.phase2.stuck_tasks").entered();
691
692 let in_progress = match task_manager
693 .list_external_by_status(Status::InProgress)
694 .await
695 {
696 Ok(tasks) => tasks,
697 Err(e) => {
698 tracing::warn!(?e, "failed to list in_progress tasks for stuck recovery");
699 vec![]
700 }
701 };
702 for task in &in_progress {
703 let Some(timing) = stuck_task_timing_from_map(
704 tmux,
705 repo,
706 &task.id.0,
707 &task.updated_at,
708 config,
709 "cannot parse updated_at, skipping stuck-task check",
710 session_map,
711 ) else {
712 continue;
713 };
714
715 if !timing.has_session {
722 let dispatch_key = format!("{repo}/{}", task.id.0);
723 if dispatching.contains_key(&dispatch_key) {
724 tracing::info!(
732 task_id = task.id.0,
733 dispatch_key,
734 age_mins = timing.age.num_minutes(),
735 "skipping stuck-task reclaim: runner still actively processing this task"
736 );
737 continue;
738 }
739 }
740
741 if timing.has_session {
742 tracing::warn!(
743 task_id = task.id.0,
744 age_mins = timing.age.num_minutes(),
745 threshold_mins = timing.threshold / 60,
746 "recovering stuck task: timed out with active session → new"
747 );
748 } else {
749 tracing::warn!(
750 task_id = task.id.0,
751 dispatch_key = format!("{repo}/{}", task.id.0),
752 age_mins = timing.age.num_minutes(),
753 threshold_mins = timing.threshold / 60,
754 "recovering stuck task: no session found — reclaiming early → new"
755 );
756 }
757 let mut cached_store_id: Option<i64> = None;
762 if timing.has_session {
763 let store_task = match store.resolve_task_id(repo, &task.id.0).await {
764 Ok(Some(store_id)) => {
765 cached_store_id = Some(store_id);
766 match store.get(store_id).await {
767 Ok(t) => Some(t),
768 Err(e) => {
769 tracing::warn!(task_id = task.id.0, error = %e, "failed to fetch task from store for stuck-task cooldown");
770 None
771 }
772 }
773 }
774 Ok(None) => None,
775 Err(e) => {
776 tracing::warn!(task_id = task.id.0, error = %e, "failed to resolve task id for stuck-task cooldown");
777 None
778 }
779 };
780 let agent_name = store_task
781 .as_ref()
782 .and_then(|t| t.agent.clone())
783 .unwrap_or_default();
784 let model_name = store_task
785 .as_ref()
786 .and_then(|t| t.model.clone())
787 .unwrap_or_default();
788
789 if !agent_name.is_empty() && !model_name.is_empty() {
790 set_model_cooldown(
791 &agent_name,
792 &model_name,
793 config.silence_cooldown,
794 "silence_detected",
795 )
796 .await;
797 record_agent_failure_with_message(
798 &agent_name,
799 &format!(
800 "stuck with active session after {}m",
801 timing.age.num_minutes()
802 ),
803 )
804 .await;
805 }
806 if !agent_name.is_empty() {
807 set_agent_cooldown(&agent_name, SILENCE_AGENT_COOLDOWN_SECS).await;
808 }
809 tracing::warn!(
810 task_id = task.id.0,
811 agent = %agent_name,
812 model = %model_name,
813 cooldown_secs = config.silence_cooldown,
814 "stuck-task cooldown applied — router will avoid this agent/model on next attempt"
815 );
816 }
817
818 {
821 let stale_labels: Vec<String> = task
822 .labels
823 .iter()
824 .filter(|l| l.starts_with("agent:") || l.starts_with("model:"))
825 .cloned()
826 .collect();
827 if !stale_labels.is_empty() {
828 let backend_clone = Arc::clone(backend);
829 let task_id_clone = task.id.clone();
830 tokio::spawn(async move {
831 for label in &stale_labels {
832 if let Err(e) = backend_clone.remove_label(&task_id_clone, label).await {
833 tracing::warn!(task_id = task_id_clone.0, label, error = %e, "failed to remove stale routing label during stuck-task recovery");
834 }
835 }
836 });
837 }
838 }
839 let resolved_store_id = match cached_store_id {
840 Some(id) => Some(id),
841 None => match store.resolve_task_id(repo, &task.id.0).await {
842 Ok(Some(id)) => Some(id),
843 Ok(None) => {
844 tracing::warn!(
845 task_id = %task.id.0,
846 repo,
847 "resolve_task_id returned None during stuck-task recovery"
848 );
849 None
850 }
851 Err(e) => {
852 tracing::error!(
853 task_id = %task.id.0,
854 repo,
855 error = %e,
856 "resolve_task_id failed during stuck-task recovery"
857 );
858 None
859 }
860 },
861 };
862 if let Some(store_id) = resolved_store_id {
863 if let Err(e) = store_set_by_id(
864 Some(store),
865 store_id,
866 &[
867 ("agent", serde_json::Value::Null),
868 ("model", serde_json::Value::Null),
869 ("route_attempts", serde_json::json!(0)),
870 ],
871 )
872 .await
873 {
874 tracing::warn!(
875 task_id = task.id.0,
876 ?e,
877 "failed to clear routing fields for stuck task — will retry next tick"
878 );
879 continue;
880 }
881 finalize_open_runs_for_recovery(
882 store,
883 store_id,
884 if timing.has_session {
885 "stuck-task recovery: active session killed"
886 } else {
887 "stuck-task recovery: no session found"
888 },
889 )
890 .await;
891 }
892 if let Err(e) = task_manager.update_task_status(&task.id, Status::New).await {
893 tracing::warn!(task_id = task.id.0, ?e, "failed to reset stuck task status");
894 continue;
895 }
896
897 let details = serde_json::json!({
899 "failure_reason": if timing.has_session { "stuck_with_session" } else { "stuck_no_session" },
900 "age_minutes": timing.age.num_minutes(),
901 "had_session": timing.has_session,
902 });
903 crate::store::store_log_activity(
904 &Some(Arc::clone(store)),
905 repo,
906 &task.id.0,
907 "rerouted",
908 Some("in_progress"),
909 Some("new"),
910 None::<&str>,
911 None::<&str>,
912 Some(&details),
913 )
914 .await;
915
916 let reason = if timing.has_session {
917 format!(
918 "timed out after {}m with active session (cleared agent for re-routing)",
919 timing.age.num_minutes()
920 )
921 } else {
922 format!(
923 "no session found — reclaiming early after {}m (cleared agent for re-routing)",
924 timing.age.num_minutes()
925 )
926 };
927 {
929 let backend_clone = Arc::clone(backend);
930 let task_id_clone = task.id.clone();
931 let body = format!(
932 "[{}] recovered: stuck in_progress — {}{}",
933 chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
934 reason,
935 crate::engine::orch_footer()
936 );
937 tokio::spawn(async move {
938 if let Err(e) = backend_clone.post_comment(&task_id_clone, &body).await {
939 tracing::warn!(
940 task_id = task_id_clone.0,
941 ?e,
942 "failed to post stuck-task recovery comment"
943 );
944 }
945 });
946 }
947 }
948
949 let mut routed = match task_manager.list_external_by_status(Status::Routed).await {
950 Ok(tasks) => tasks,
951 Err(e) => {
952 tracing::warn!(?e, "failed to list routed tasks for stuck recovery");
953 vec![]
954 }
955 };
956 match task_manager
957 .list_internal_by_status(crate::store::TaskStatus::Routed)
958 .await
959 {
960 Ok(tasks) => routed.extend(tasks),
961 Err(e) => tracing::warn!(
962 ?e,
963 "failed to list internal routed tasks for stuck recovery"
964 ),
965 }
966 for task in &routed {
967 let Some(timing) = stuck_task_timing_from_map(
968 tmux,
969 repo,
970 &task.id.0,
971 &task.updated_at,
972 config,
973 "cannot parse updated_at, skipping routed stuck-task check",
974 session_map,
975 ) else {
976 continue;
977 };
978
979 if !timing.has_session {
980 continue;
981 }
982
983 let session_name = tmux.session_name(repo, &task.id.0);
984 if let Err(e) = tmux.kill_session(&session_name).await {
985 tracing::warn!(
986 task_id = task.id.0,
987 session = %session_name,
988 error = %e,
989 "failed to kill stale tmux session blocking routed task"
990 );
991 continue;
992 }
993
994 recover_routed_blocked_dispatch(backend, store, task_manager, repo, task, &timing).await;
995 }
996
997 let internal_in_progress = match task_manager
1000 .list_internal_by_status(DbStatus::InProgress)
1001 .await
1002 {
1003 Ok(tasks) => tasks,
1004 Err(e) => {
1005 tracing::warn!(
1006 ?e,
1007 "failed to list internal in_progress tasks for stuck recovery"
1008 );
1009 vec![]
1010 }
1011 };
1012 for task in &internal_in_progress {
1013 let task_id = task.id.0.clone();
1014 let Some(timing) = stuck_task_timing_from_map(
1015 tmux,
1016 repo,
1017 &task_id,
1018 &task.updated_at,
1019 config,
1020 "cannot parse updated_at, skipping stuck internal-task check",
1021 session_map,
1022 ) else {
1023 continue;
1024 };
1025
1026 if !timing.has_session {
1032 let dispatch_key = format!("{repo}/{task_id}");
1033 if dispatching.contains_key(&dispatch_key) {
1034 tracing::info!(
1044 task_id,
1045 dispatch_key,
1046 age_mins = timing.age.num_minutes(),
1047 "skipping stuck-task reclaim: runner still actively processing this task"
1048 );
1049 continue;
1050 }
1051 }
1052
1053 if timing.has_session {
1054 tracing::warn!(
1055 task_id,
1056 age_mins = timing.age.num_minutes(),
1057 threshold_mins = timing.threshold / 60,
1058 "recovering stuck task: timed out with active session → new"
1059 );
1060 } else {
1061 tracing::warn!(
1062 task_id,
1063 dispatch_key = format!("{repo}/{task_id}"),
1064 age_mins = timing.age.num_minutes(),
1065 threshold_mins = timing.threshold / 60,
1066 "recovering stuck task: no session found — reclaiming early → new"
1067 );
1068 }
1069 let mut cached_store_id: Option<i64> = None;
1073 if timing.has_session {
1074 let store_task = match store.resolve_task_id(repo, &task_id).await {
1075 Ok(Some(store_id)) => {
1076 cached_store_id = Some(store_id);
1077 match store.get(store_id).await {
1078 Ok(t) => Some(t),
1079 Err(e) => {
1080 tracing::warn!(task_id, error = %e, "failed to fetch task from store for stuck internal-task cooldown");
1081 None
1082 }
1083 }
1084 }
1085 Ok(None) => None,
1086 Err(e) => {
1087 tracing::warn!(task_id, error = %e, "failed to resolve task id for stuck internal-task cooldown");
1088 None
1089 }
1090 };
1091 let agent_name = store_task
1092 .as_ref()
1093 .and_then(|t| t.agent.clone())
1094 .unwrap_or_default();
1095 let model_name = store_task
1096 .as_ref()
1097 .and_then(|t| t.model.clone())
1098 .unwrap_or_default();
1099
1100 if !agent_name.is_empty() && !model_name.is_empty() {
1101 set_model_cooldown(
1102 &agent_name,
1103 &model_name,
1104 config.silence_cooldown,
1105 "silence_detected",
1106 )
1107 .await;
1108 record_agent_failure_with_message(
1109 &agent_name,
1110 &format!(
1111 "internal task stuck with active session after {}m",
1112 timing.age.num_minutes()
1113 ),
1114 )
1115 .await;
1116 }
1117 if !agent_name.is_empty() {
1118 set_agent_cooldown(&agent_name, SILENCE_AGENT_COOLDOWN_SECS).await;
1119 }
1120 tracing::warn!(
1121 task_id,
1122 agent = %agent_name,
1123 model = %model_name,
1124 cooldown_secs = config.silence_cooldown,
1125 "internal stuck-task cooldown applied — router will avoid this agent/model on next attempt"
1126 );
1127 }
1128 let resolved_store_id = match cached_store_id {
1131 Some(id) => Some(id),
1132 None => match store.resolve_task_id(repo, &task_id).await {
1133 Ok(Some(id)) => Some(id),
1134 Ok(None) => {
1135 tracing::warn!(
1136 task_id = %task_id,
1137 repo,
1138 "resolve_task_id returned None during stuck internal-task recovery"
1139 );
1140 None
1141 }
1142 Err(e) => {
1143 tracing::error!(
1144 task_id = %task_id,
1145 repo,
1146 error = %e,
1147 "resolve_task_id failed during stuck internal-task recovery"
1148 );
1149 None
1150 }
1151 },
1152 };
1153 if let Some(store_id) = resolved_store_id {
1154 if let Err(e) = store_set_by_id(
1155 Some(store),
1156 store_id,
1157 &[
1158 ("agent", serde_json::Value::Null),
1159 ("model", serde_json::Value::Null),
1160 ("route_attempts", serde_json::json!(0)),
1161 ],
1162 )
1163 .await
1164 {
1165 tracing::warn!(
1166 task_id,
1167 ?e,
1168 "failed to clear routing fields for stuck internal task — will retry next tick"
1169 );
1170 continue;
1171 }
1172 finalize_open_runs_for_recovery(
1173 store,
1174 store_id,
1175 if timing.has_session {
1176 "stuck-task recovery: internal active session killed"
1177 } else {
1178 "stuck-task recovery: internal no session found"
1179 },
1180 )
1181 .await;
1182 }
1183 if let Err(e) = task_manager
1184 .update_task_status(&ExternalId(task_id.clone()), Status::New)
1185 .await
1186 {
1187 tracing::warn!(task_id, ?e, "failed to reset stuck internal task status");
1188 }
1189 }
1190
1191 let in_review = match store
1198 .list_by_status(repo, crate::store::TaskStatus::InReview)
1199 .await
1200 {
1201 Ok(v) => v,
1202 Err(e) => {
1203 tracing::warn!(?e, "failed to list in_review tasks for stuck recovery");
1204 vec![]
1205 }
1206 };
1207 let in_review_config = crate::engine::EngineConfig {
1211 no_session_stuck_timeout: config.in_review_no_session_stuck_timeout,
1212 ..config.clone()
1213 };
1214 for task in &in_review {
1215 let Some(task_id) = task.external_id.clone() else {
1216 tracing::warn!(
1217 store_id = task.id,
1218 "in_review task has no external id, skipping stuck recovery"
1219 );
1220 continue;
1221 };
1222
1223 if dispatching.contains_key(&format!("{repo}/{task_id}")) {
1225 tracing::debug!(
1226 task_id,
1227 "task locked by dispatch flow, skipping stuck check"
1228 );
1229 continue;
1230 }
1231
1232 let (entered_at, decided) = in_review_stuck_entry(store, task.id, &task.updated_at).await;
1235 if decided {
1236 tracing::debug!(
1237 task_id,
1238 "in_review task already has a review decision, waiting on merge"
1239 );
1240 continue;
1241 }
1242
1243 let review_task_id = format!("{task_id}-review");
1244 let Some(timing) = stuck_task_timing_from_map(
1245 tmux,
1246 repo,
1247 &review_task_id,
1248 &entered_at,
1249 &in_review_config,
1250 "cannot parse in_review timestamp, skipping stuck in_review check",
1251 session_map,
1252 ) else {
1253 continue;
1254 };
1255
1256 tracing::warn!(
1257 task_id,
1258 age_mins = timing.age.num_minutes(),
1259 threshold_mins = timing.threshold / 60,
1260 has_session = timing.has_session,
1261 "recovering stuck in_review task — resetting to needs_review"
1262 );
1263
1264 match task_manager
1265 .update_task_status_if(
1266 &ExternalId(task_id.clone()),
1267 Status::NeedsReview,
1268 Status::InReview,
1269 )
1270 .await
1271 {
1272 Err(e) => {
1273 tracing::warn!(task_id, ?e, "failed to reset stuck in_review task status");
1274 continue;
1275 }
1276 Ok(false) => {
1277 tracing::debug!(
1278 task_id,
1279 "stuck in_review reset skipped — task already transitioned"
1280 );
1281 continue;
1282 }
1283 Ok(true) => {}
1284 }
1285
1286 if let Err(e) = store
1289 .set_fields(task.id, &[("review_agent_failures", serde_json::json!(0))])
1290 .await
1291 {
1292 tracing::warn!(task_id, ?e, "failed to reset review_agent_failures");
1293 }
1294
1295 finalize_open_runs_for_recovery(
1296 store,
1297 task.id,
1298 if timing.has_session {
1299 "stuck-task recovery: in_review session killed"
1300 } else {
1301 "stuck-task recovery: in_review no session found"
1302 },
1303 )
1304 .await;
1305
1306 let details = serde_json::json!({
1307 "failure_reason": "stuck_in_review",
1308 "age_minutes": timing.age.num_minutes(),
1309 "had_session": timing.has_session,
1310 });
1311 crate::store::store_log_activity(
1312 &Some(Arc::clone(store)),
1313 repo,
1314 &task_id,
1315 "rerouted",
1316 Some("in_review"),
1317 Some("needs_review"),
1318 None::<&str>,
1319 None::<&str>,
1320 Some(&details),
1321 )
1322 .await;
1323
1324 if task.origin == "internal" {
1326 continue;
1327 }
1328
1329 if let Err(e) = backend
1330 .post_comment(
1331 &ExternalId(task_id.clone()),
1332 &format!(
1333 "[{}] recovered: stuck in_review — {} after {}m, resetting to needs_review{}",
1334 chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
1335 if timing.has_session {
1336 "review session hung"
1337 } else {
1338 "no session found"
1339 },
1340 timing.age.num_minutes(),
1341 crate::engine::orch_footer()
1342 ),
1343 )
1344 .await
1345 {
1346 tracing::warn!(
1347 task_id,
1348 ?e,
1349 "failed to post stuck in_review recovery comment"
1350 );
1351 }
1352 }
1353
1354 Ok(())
1355}
1356
1357pub(crate) async fn recover_untracked_repo_stuck_tasks(
1360 tmux: &Arc<TmuxManager>,
1361 config: &EngineConfig,
1362 store: &Arc<TaskStore>,
1363 session_map: &std::collections::HashMap<String, bool>,
1364 dispatching: &Arc<DashMap<String, String>>,
1365 active_repos: &std::collections::HashSet<String>,
1366) {
1367 let all_external_in_progress = match store
1368 .list_all_by_status_global(crate::store::TaskStatus::InProgress)
1369 .await
1370 {
1371 Ok(tasks) => tasks,
1372 Err(e) => {
1373 tracing::warn!(
1374 ?e,
1375 "failed to list global in_progress tasks for cross-repo stuck recovery"
1376 );
1377 vec![]
1378 }
1379 };
1380 for task in all_external_in_progress
1381 .iter()
1382 .filter(|t| t.origin != "internal" && !active_repos.contains(&t.repo))
1383 {
1384 let external_id = task
1385 .external_id
1386 .clone()
1387 .unwrap_or_else(|| task.id.to_string());
1388 let Some(timing) = stuck_task_timing_from_map(
1389 tmux,
1390 &task.repo,
1391 &external_id,
1392 &task.updated_at,
1393 config,
1394 "cannot parse updated_at, skipping cross-repo stuck-task check",
1395 session_map,
1396 ) else {
1397 continue;
1398 };
1399
1400 if !timing.has_session {
1404 let dispatch_key = format!("{}/{external_id}", task.repo);
1405 if dispatching.contains_key(&dispatch_key) {
1406 tracing::info!(
1410 task_id = task.id,
1411 repo = %task.repo,
1412 dispatch_key,
1413 age_mins = timing.age.num_minutes(),
1414 "skipping cross-repo stuck-task reclaim: runner still actively processing this task"
1415 );
1416 continue;
1417 }
1418 }
1419
1420 if timing.has_session {
1421 tracing::warn!(
1422 task_id = task.id,
1423 repo = %task.repo,
1424 age_mins = timing.age.num_minutes(),
1425 threshold_mins = timing.threshold / 60,
1426 "recovering cross-repo stuck task: timed out with active session → new"
1427 );
1428 } else {
1429 tracing::warn!(
1430 task_id = task.id,
1431 repo = %task.repo,
1432 dispatch_key = format!("{}/{external_id}", task.repo),
1433 age_mins = timing.age.num_minutes(),
1434 threshold_mins = timing.threshold / 60,
1435 "recovering cross-repo stuck task: no session found — reclaiming early → new"
1436 );
1437 }
1438
1439 if let Err(e) = store_set_by_id(
1440 Some(store),
1441 task.id,
1442 &[
1443 ("agent", serde_json::Value::Null),
1444 ("model", serde_json::Value::Null),
1445 ("route_attempts", serde_json::json!(0)),
1446 ],
1447 )
1448 .await
1449 {
1450 tracing::warn!(
1451 task_id = task.id,
1452 repo = %task.repo,
1453 ?e,
1454 "failed to clear routing fields for cross-repo stuck task — will retry next tick"
1455 );
1456 continue;
1457 }
1458
1459 finalize_open_runs_for_recovery(
1460 store,
1461 task.id,
1462 if timing.has_session {
1463 "stuck-task recovery: cross-repo active session killed"
1464 } else {
1465 "stuck-task recovery: cross-repo no session found"
1466 },
1467 )
1468 .await;
1469
1470 if let Err(e) = store
1471 .update_status(task.id, crate::store::TaskStatus::New)
1472 .await
1473 {
1474 tracing::warn!(
1475 task_id = task.id,
1476 repo = %task.repo,
1477 ?e,
1478 "failed to reset cross-repo stuck task status"
1479 );
1480 continue;
1481 }
1482
1483 let details = serde_json::json!({
1484 "failure_reason": if timing.has_session { "stuck_with_session" } else { "stuck_no_session" },
1485 "age_minutes": timing.age.num_minutes(),
1486 "had_session": timing.has_session,
1487 "scope": "cross_repo_global_sweep",
1488 });
1489 crate::store::store_log_activity(
1490 &Some(Arc::clone(store)),
1491 &task.repo,
1492 &external_id,
1493 "rerouted",
1494 Some("in_progress"),
1495 Some("new"),
1496 None::<&str>,
1497 None::<&str>,
1498 Some(&details),
1499 )
1500 .await;
1501 }
1502}
1503
1504pub(crate) async fn tick_global_sweeps(
1513 task_manager: &Arc<TaskManager>,
1514 store: &Arc<TaskStore>,
1515 router: &Router,
1516 dispatching: &Arc<DashMap<String, String>>,
1517 auto_merge_in_flight: &Arc<DashSet<String>>,
1518) {
1519 if let Err(e) = crate::engine::sync::auto_unblock_ci_failure_blocked_tasks_global(
1524 task_manager,
1525 store,
1526 auto_merge_in_flight,
1527 )
1528 .await
1529 {
1530 tracing::warn!(err = %e, "global CI-failure blocked sweep failed");
1531 }
1532
1533 if let Err(e) = crate::engine::sync::refire_and_escalate_stale_needs_review_global(
1536 task_manager,
1537 store,
1538 router,
1539 dispatching,
1540 )
1541 .await
1542 {
1543 tracing::warn!(err = %e, "global NeedsReview refire/escalation sweep failed");
1544 }
1545
1546 if let Err(e) = crate::engine::sync::auto_recover_rebroadcast_blocked_tasks_global(
1549 router,
1550 task_manager,
1551 store,
1552 )
1553 .await
1554 {
1555 tracing::warn!(err = %e, "global rebroadcast-blocked recovery sweep failed");
1556 }
1557}
1558
1559pub(crate) async fn tick_route_tasks(
1561 backend: &Arc<dyn ExternalBackend>,
1562 task_manager: &Arc<TaskManager>,
1563 router: &mut Router,
1564 store: &Arc<TaskStore>,
1565 repo: &str,
1566 routing_quota: &mut RoutingTickQuota,
1567) -> anyhow::Result<()> {
1568 let _span = tracing::info_span!("engine.tick.phase3a.route").entered();
1569 if let Err(err) = crate::engine::dependencies::apply_claim_gates(store).await {
1570 tracing::warn!(err = %err, "dependency claim gate failed");
1571 }
1572 if routing_quota.is_exhausted() {
1573 tracing::debug!("routing quota exhausted for this tick — skipping repo routing phase");
1574 return Ok(());
1575 }
1576
1577 if is_github_circuit_open() {
1581 let remaining = github_circuit_remaining_secs();
1582 tracing::info!(
1583 remaining_secs = remaining,
1584 "GitHub 5xx circuit breaker open — skipping routing phase"
1585 );
1586 return Ok(());
1587 }
1588 let new_tasks = task_manager.list_routable().await?;
1589 let routable: Vec<&ExternalTask> = new_tasks
1590 .iter()
1591 .filter(|t| !skips_agent(&t.labels))
1592 .collect();
1593
1594 if !routable.is_empty() {
1598 let start = std::time::Instant::now();
1599 router.refresh_health(store).await;
1600 tracing::debug!(
1601 duration_ms = start.elapsed().as_millis(),
1602 "router.refresh_health completed"
1603 );
1604 }
1605
1606 for task in routable.into_iter().take(routing_quota.remaining()) {
1608 if !routing_quota.consume_one() {
1609 break;
1610 }
1611 let _task_span = tracing::info_span!("engine.route", task_id = %task.id.0).entered();
1612
1613 let task_start = Instant::now();
1614 match router.route(task, store, repo).await {
1615 Ok(result) => {
1616 tracing::debug!(task_id = %task.id.0, duration_ms = task_start.elapsed().as_millis(), "route completed");
1617 if let Err(e) = router
1619 .store_route_result(&task.id.0, &result, store, repo)
1620 .await
1621 {
1622 tracing::warn!(
1623 task_id = task.id.0,
1624 ?e,
1625 "failed to store route result; skipping Routed transition"
1626 );
1627 continue;
1628 }
1629
1630 if is_internal_id(&task.id.0) {
1631 if let Err(e) = task_manager
1633 .update_task_status(&task.id, Status::Routed)
1634 .await
1635 {
1636 tracing::warn!(
1637 task_id = task.id.0,
1638 ?e,
1639 "failed to set internal status:routed"
1640 );
1641 continue;
1642 }
1643 } else {
1644 let backend_clone = Arc::clone(backend);
1648 let task_id_clone = task.id.clone();
1649 let task_labels: Vec<String> = task.labels.clone();
1650 let new_labels = {
1651 let mut labels = vec![
1652 format!("agent:{}", result.agent),
1653 format!("complexity:{}", result.complexity),
1654 ];
1655 if let Some(ref model) = result.model {
1656 labels.push(format!("model:{model}"));
1657 }
1658 labels
1659 };
1660 tokio::spawn(async move {
1661 for label in &task_labels {
1663 if (label.starts_with("agent:")
1664 || label.starts_with("complexity:")
1665 || label.starts_with("model:"))
1666 && let Err(e) =
1667 backend_clone.remove_label(&task_id_clone, label).await
1668 {
1669 tracing::warn!(task_id = task_id_clone.0, label, error = %e, "failed to remove stale routing label during re-route");
1670 }
1671 }
1672 if let Err(e) = backend_clone.set_labels(&task_id_clone, &new_labels).await
1674 {
1675 tracing::warn!(
1676 task_id = task_id_clone.0,
1677 ?e,
1678 "failed to set routing labels"
1679 );
1680 }
1681 });
1682
1683 if let Err(e) = task_manager
1685 .update_task_status(&task.id, Status::Routed)
1686 .await
1687 {
1688 tracing::warn!(task_id = task.id.0, ?e, "failed to set status:routed");
1689 continue;
1690 }
1691 }
1692
1693 match store.ensure_external_task(repo, task).await {
1695 Ok(store_id) => {
1696 let profile_json =
1697 serde_json::to_string(&result.profile).unwrap_or_default();
1698 let skills_json =
1699 serde_json::to_string(&result.selected_skills).unwrap_or_default();
1700 if let Err(e) = store
1701 .store_route(&crate::store::StoreRoute {
1702 id: store_id,
1703 agent: &result.agent,
1704 model: result.model.as_deref(),
1705 complexity: &result.complexity,
1706 estimate: result.estimate,
1707 reason: &result.reason,
1708 profile: &profile_json,
1709 skills: &skills_json,
1710 })
1711 .await
1712 {
1713 tracing::debug!(
1714 task_id = task.id.0,
1715 ?e,
1716 "dual-write: store_route failed"
1717 );
1718 }
1719 if let Err(e) = store
1720 .update_status(store_id, crate::store::TaskStatus::Routed)
1721 .await
1722 {
1723 tracing::debug!(
1724 task_id = task.id.0,
1725 ?e,
1726 "dual-write: update_status failed"
1727 );
1728 }
1729 if result.estimate > 0 {
1733 let backend_clone = Arc::clone(backend);
1734 let task_id = task.id.clone();
1735 let estimate = result.estimate;
1736 tokio::spawn(async move {
1737 if let Err(e) = backend_clone
1738 .sync_estimate_to_project(&task_id, estimate)
1739 .await
1740 {
1741 tracing::debug!(
1742 task_id = task_id.0,
1743 estimate,
1744 err = %e,
1745 "dual-write: estimate project sync failed"
1746 );
1747 }
1748 });
1749 }
1750 }
1751 Err(e) => {
1752 tracing::debug!(
1753 task_id = task.id.0,
1754 ?e,
1755 "dual-write: ensure_external_task failed"
1756 );
1757 }
1758 }
1759
1760 if let Some(ref warning) = result.warning {
1761 tracing::warn!(task_id = task.id.0, warning, "routing sanity warning");
1762 }
1763
1764 tracing::info!(
1765 task_id = task.id.0,
1766 agent = %result.agent,
1767 complexity = %result.complexity,
1768 reason = %result.reason,
1769 "task routed"
1770 );
1771 }
1772 Err(e) => {
1773 if let Some(cooled) = e.downcast_ref::<AllAgentsCooledError>() {
1774 tracing::error!(
1775 task_id = %task.id.0,
1776 scope = %cooled.scope(),
1777 remaining_secs = ?cooled.remaining_secs(),
1778 degraded = ?cooled.degraded(),
1779 "ALL AGENTS COOLED — no agent available to route this task. \
1780 Task stays in 'new' and will be retried on every tick. \
1781 Check `orch cooldown list` to see what is blocking dispatch."
1782 );
1783 continue;
1784 }
1785 if let Some(waiting) = e.downcast_ref::<crate::store::WaitingOnDependency>() {
1786 tracing::info!(
1787 task_id = %task.id.0,
1788 blockers = %waiting.blockers.join(", "),
1789 "task is waiting on a dependency"
1790 );
1791 continue;
1792 }
1793 tracing::error!(task_id = task.id.0, ?e, "routing failed, skipping task");
1794 }
1795 }
1796 }
1797 Ok(())
1798}
1799
1800pub(crate) async fn route_new_tasks_global(
1816 router: &mut Router,
1817 store: &Arc<TaskStore>,
1818 active_repos: &std::collections::HashSet<String>,
1819 routing_quota: &mut RoutingTickQuota,
1820) -> anyhow::Result<()> {
1821 use crate::store::{StoreRoute, TaskStatus};
1822
1823 if routing_quota.is_exhausted() {
1824 tracing::debug!("routing quota exhausted for this tick — skipping global routing sweep");
1825 return Ok(());
1826 }
1827
1828 let all_new = store.list_all_by_status_global(TaskStatus::New).await?;
1829 let orphaned: Vec<&crate::store::Task> = all_new
1830 .iter()
1831 .filter(|t| !active_repos.contains(&t.repo))
1832 .filter(|t| !skips_agent(&t.labels))
1833 .collect();
1834
1835 if orphaned.is_empty() {
1836 return Ok(());
1837 }
1838
1839 tracing::info!(
1840 count = orphaned.len(),
1841 "global routing sweep: found new tasks from inactive repos"
1842 );
1843
1844 router.refresh_health(store).await;
1845
1846 for task in orphaned.into_iter().take(routing_quota.remaining()) {
1847 if !routing_quota.consume_one() {
1848 break;
1849 }
1850 let ext_task = crate::engine::tasks::store_task_to_external(task);
1851 let task_start = Instant::now();
1852 match router.route(&ext_task, store, &task.repo).await {
1853 Ok(result) => {
1854 tracing::debug!(
1855 task_id = %ext_task.id.0,
1856 repo = %task.repo,
1857 duration_ms = task_start.elapsed().as_millis(),
1858 "global sweep: route completed"
1859 );
1860 let profile_json = serde_json::to_string(&result.profile).unwrap_or_default();
1861 let skills_json =
1862 serde_json::to_string(&result.selected_skills).unwrap_or_default();
1863 if let Err(e) = store
1864 .store_route(&StoreRoute {
1865 id: task.id,
1866 agent: &result.agent,
1867 model: result.model.as_deref(),
1868 complexity: &result.complexity,
1869 estimate: result.estimate,
1870 reason: &result.reason,
1871 profile: &profile_json,
1872 skills: &skills_json,
1873 })
1874 .await
1875 {
1876 tracing::warn!(
1877 task_id = %ext_task.id.0,
1878 repo = %task.repo,
1879 err = %e,
1880 "global sweep: failed to store route result; skipping status update"
1881 );
1882 continue;
1883 }
1884 match store.update_status(task.id, TaskStatus::Routed).await {
1885 Err(e) => {
1886 tracing::warn!(
1887 task_id = %ext_task.id.0,
1888 repo = %task.repo,
1889 err = %e,
1890 "global sweep: failed to set status:routed"
1891 );
1892 }
1893 _ => {
1894 tracing::info!(
1895 task_id = %ext_task.id.0,
1896 repo = %task.repo,
1897 agent = %result.agent,
1898 complexity = %result.complexity,
1899 "global sweep: task routed"
1900 );
1901 }
1902 }
1903 if let Some(ref warning) = result.warning {
1904 tracing::warn!(task_id = %ext_task.id.0, warning, "global sweep: routing sanity warning");
1905 }
1906 }
1907 Err(e) => {
1908 if let Some(cooled) = e.downcast_ref::<AllAgentsCooledError>() {
1909 tracing::error!(
1910 task_id = %ext_task.id.0,
1911 repo = %task.repo,
1912 scope = %cooled.scope(),
1913 remaining_secs = ?cooled.remaining_secs(),
1914 degraded = ?cooled.degraded(),
1915 "global sweep: ALL AGENTS COOLED — task stays in 'new'"
1916 );
1917 continue;
1918 }
1919 if let Some(waiting) = e.downcast_ref::<crate::store::WaitingOnDependency>() {
1920 tracing::info!(
1921 task_id = %ext_task.id.0,
1922 repo = %task.repo,
1923 blockers = %waiting.blockers.join(", "),
1924 "global sweep: task is waiting on a dependency"
1925 );
1926 continue;
1927 }
1928 tracing::error!(
1929 task_id = %ext_task.id.0,
1930 repo = %task.repo,
1931 err = ?e,
1932 "global sweep: routing failed, skipping task"
1933 );
1934 }
1935 }
1936 }
1937
1938 Ok(())
1939}
1940
1941#[derive(Debug, Clone, Copy)]
1943pub(crate) struct DispatchMode {
1944 pub(crate) is_degraded: bool,
1945 pub(crate) healthy_agents: usize,
1946 pub(crate) threshold: usize,
1947}
1948
1949pub(crate) fn dispatch_mode_from_router(router: &Router) -> DispatchMode {
1950 let threshold = crate::engine::router::config::min_healthy_agents_threshold();
1951 let healthy_agents = router.healthy_agent_count("simple");
1952 let is_degraded = router.is_degraded(threshold);
1953 DispatchMode {
1954 is_degraded,
1955 healthy_agents,
1956 threshold,
1957 }
1958}
1959
1960#[expect(
1962 clippy::too_many_arguments,
1963 reason = "each argument is engine state the tick passes down"
1964)]
1965#[tracing::instrument(skip_all, name = "engine.tick.phase3b.dispatch")]
1966pub(crate) async fn tick_dispatch_tasks(
1967 backend: &Arc<dyn ExternalBackend>,
1968 tmux: &Arc<TmuxManager>,
1969 repo: &str,
1970 runner: &Arc<TaskRunner>,
1971 capture: &Arc<CaptureService>,
1972 semaphore: &Arc<Semaphore>,
1973 task_manager: &Arc<TaskManager>,
1974 weight_tx: &mpsc::Sender<WeightSignal>,
1975 dispatch_mode: DispatchMode,
1976 dispatching: &Arc<DashMap<String, String>>,
1977 store: &Arc<TaskStore>,
1978 session_map: &std::collections::HashMap<String, bool>,
1979) -> anyhow::Result<()> {
1980 let mut routed_tasks = task_manager.list_external_by_status(Status::Routed).await?;
1982
1983 use crate::store::TaskStatus as DbStatus;
1985 let internal_routed = task_manager
1986 .list_internal_by_status(DbStatus::Routed)
1987 .await?;
1988 routed_tasks.extend(internal_routed);
1989
1990 let dispatchable: Vec<ExternalTask> = routed_tasks
1992 .into_iter()
1993 .filter(|t| !skips_agent(&t.labels))
1994 .collect();
1995
1996 dispatch_tasks_for_repo(
1997 backend,
1998 tmux,
1999 repo,
2000 runner,
2001 capture,
2002 semaphore,
2003 task_manager,
2004 weight_tx,
2005 dispatch_mode,
2006 dispatching,
2007 store,
2008 session_map,
2009 dispatchable,
2010 )
2011 .await
2012}
2013
2014#[expect(
2020 clippy::too_many_arguments,
2021 reason = "each argument is engine state the tick passes down"
2022)]
2023async fn dispatch_tasks_for_repo(
2024 backend: &Arc<dyn ExternalBackend>,
2025 tmux: &Arc<TmuxManager>,
2026 repo: &str,
2027 runner: &Arc<TaskRunner>,
2028 capture: &Arc<CaptureService>,
2029 semaphore: &Arc<Semaphore>,
2030 task_manager: &Arc<TaskManager>,
2031 weight_tx: &mpsc::Sender<WeightSignal>,
2032 dispatch_mode: DispatchMode,
2033 dispatching: &Arc<DashMap<String, String>>,
2034 store: &Arc<TaskStore>,
2035 session_map: &std::collections::HashMap<String, bool>,
2036 dispatchable: Vec<ExternalTask>,
2037) -> anyhow::Result<()> {
2038 if dispatchable.is_empty() {
2039 tracing::debug!(count = 0, "dispatchable tasks found");
2040 return Ok(());
2041 }
2042
2043 tracing::info!(count = dispatchable.len(), "dispatchable tasks found");
2044 if dispatch_mode.is_degraded {
2045 tracing::warn!(
2046 healthy_agents = dispatch_mode.healthy_agents,
2047 threshold = dispatch_mode.threshold,
2048 "degraded mode: using sequential dispatch"
2049 );
2050 }
2051
2052 let sequential_delay = if dispatch_mode.is_degraded {
2053 crate::engine::router::config::sequential_dispatch_delay_ms()
2054 } else {
2055 0
2056 };
2057
2058 let mut ranked: Vec<(crate::engine::priority::Priority, ExternalTask)> =
2059 Vec::with_capacity(dispatchable.len());
2060 for task in dispatchable {
2061 let has_pr = match store.resolve_id_by_external(repo, &task.id.0).await {
2062 Ok(Some(id)) => store.get(id).await.is_ok_and(|t| t.pr_number.is_some()),
2063 _ => false,
2064 };
2065 ranked.push((
2066 crate::engine::priority::Priority::from_labels(&task.labels).with_pr(has_pr),
2067 task,
2068 ));
2069 }
2070 crate::engine::priority::sort_by_priority(&mut ranked, |t| t.0);
2071
2072 for (idx, (priority, task)) in ranked.into_iter().enumerate() {
2073 if crate::engine::dependencies::dispatch_is_waiting(store, repo, &task.id.0).await {
2074 continue;
2075 }
2076 if idx > 0 && dispatch_mode.is_degraded {
2078 let delay_ms = sequential_delay;
2079 tracing::debug!(
2080 task_id = task.id.0,
2081 delay_ms = delay_ms,
2082 "sequential dispatch: waiting before dispatch"
2083 );
2084 tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
2085 }
2086 let dispatch_key = format!("{}/{}", repo, task.id.0);
2093 {
2098 use dashmap::mapref::entry::Entry;
2099 match dispatching.entry(dispatch_key.clone()) {
2100 Entry::Occupied(existing) => {
2101 let existing_id = existing.get().clone();
2102 drop(existing); if existing_id == task.id.0 {
2104 tracing::debug!(
2105 task_id = task.id.0,
2106 "task already dispatching, skipping duplicate"
2107 );
2108 } else {
2109 tracing::warn!(
2110 task_id = task.id.0,
2111 existing_task_id = existing_id,
2112 dispatch_key,
2113 "dispatch key collision: unexpected task already holds this key"
2114 );
2115 }
2116 continue;
2117 }
2118 Entry::Vacant(slot) => {
2119 slot.insert(task.id.0.clone());
2120 tracing::info!(task_id = task.id.0, dispatch_key, "dispatch guard claimed");
2127 }
2128 }
2129 }
2130 let dispatch_guard =
2132 DispatchGuard::new(std::sync::Arc::clone(dispatching), dispatch_key.clone());
2133 let task_id = task.id.0.clone();
2134
2135 let route_result = match get_route_result(store, repo, &task_id).await {
2138 Ok(r) => Some(r),
2139 Err(RouteResultError::NoAgent { .. }) => {
2140 tracing::warn!(
2141 task_id,
2142 "routed task missing agent — resetting to new for re-routing (#1604)"
2143 );
2144 if let Err(e2) = task_manager.update_task_status(&task.id, Status::New).await {
2145 tracing::error!(task_id, error = %e2, "failed to reset task to new");
2146 }
2147 continue; }
2149 Err(e) => {
2150 tracing::warn!(task_id, error = %e, "get_route_result failed — dispatching without route info");
2151 None
2152 }
2153 };
2154
2155 if let Some(route) = route_result.as_ref() {
2161 let agent_cooled = is_agent_in_cooldown(&route.agent);
2162 let model_cooled = route
2163 .model
2164 .as_deref()
2165 .is_some_and(|model| is_model_in_cooldown(&route.agent, model));
2166 if agent_cooled || model_cooled {
2167 tracing::warn!(
2168 task_id,
2169 agent = route.agent,
2170 model = route.model.as_deref().unwrap_or(""),
2171 "routed agent/model entered cooldown after routing — deferring dispatch to next tick"
2172 );
2173 continue; }
2175 }
2176
2177 let session_name = tmux.session_name(repo, &task_id);
2179 if tmux
2180 .session_blocks_dispatch_from_map(&session_name, session_map)
2181 .await
2182 {
2183 tracing::warn!(
2184 task_id,
2185 session_name,
2186 "task has existing tmux session, skipping dispatch"
2187 );
2188 continue; }
2190
2191 let Some(permit) = crate::engine::priority::acquire_dispatch_slot(semaphore, priority)
2193 else {
2194 tracing::debug!("all parallel slots busy, skipping remaining tasks");
2195 break; };
2197
2198 let set_in_progress_result = task_manager
2200 .update_task_status(&task.id, Status::InProgress)
2201 .await;
2202 if let Err(e) = set_in_progress_result {
2203 tracing::error!(task_id, ?e, "failed to set in_progress, skipping dispatch");
2204 drop(permit);
2205 continue; }
2207 tracing::info!(task_id, "dispatching task");
2208
2209 capture
2211 .register_session(repo, &task_id, &session_name)
2212 .await;
2213
2214 let runner = std::sync::Arc::clone(runner);
2216 let backend = std::sync::Arc::clone(backend);
2217 let tmux = std::sync::Arc::clone(tmux);
2218 let capture = std::sync::Arc::clone(capture);
2219 let task_id_for_cleanup = task_id.clone();
2220 let task_owned = task;
2221 let weight_tx = weight_tx.clone();
2222 let repo_owned = repo.to_string();
2223 let task_manager_for_spawn = std::sync::Arc::clone(task_manager);
2224 let store_for_spawn = std::sync::Arc::clone(store);
2225
2226 let repo_ctx = repo_owned.clone();
2227 tokio::spawn(REPO_CONTEXT.scope(repo_ctx, async move {
2228 let _dispatch_guard = dispatch_guard; let dispatch_start = std::time::Instant::now();
2230 match runner
2231 .run_with_context(&task_owned, &backend, &tmux, route_result.as_ref())
2232 .await
2233 {
2234 Ok(signal) => {
2235 tracing::info!(task_id, "task runner completed");
2236
2237 let duration = dispatch_start.elapsed().as_secs_f64();
2238
2239 let enable_review = config::get("workflow.enable_review_agent")
2245 .map_or(true, |v| v != "false");
2246 let display_status = match &signal {
2247 WeightSignal::Success { .. } if enable_review => {
2248 let has_pr = match store_for_spawn
2252 .resolve_task_id(&repo_owned, &task_id)
2253 .await
2254 {
2255 Ok(Some(store_id)) => match store_for_spawn
2256 .get(store_id)
2257 .await
2258 {
2259 Ok(t) => t.pr_number.is_some(),
2260 Err(e) => {
2261 tracing::warn!(task_id, err = %e, "store.get() failed — defaulting to needs_review");
2262 true
2263 }
2264 },
2265 Ok(None) => false,
2266 Err(e) => {
2267 tracing::warn!(task_id, err = %e, "failed to check PR status — defaulting to needs_review");
2268 true
2269 }
2270 };
2271 if has_pr {
2272 "needs_review"
2273 } else {
2274 "done"
2275 }
2276 }
2277 WeightSignal::RateLimited { .. } | WeightSignal::Rerouted => "new",
2282 WeightSignal::Blocked => "blocked",
2283 WeightSignal::None if enable_review => "needs_review",
2288 WeightSignal::Success { .. } | WeightSignal::None => "done",
2289 };
2290
2291 let _ = weight_tx.send(signal).await;
2293
2294 if display_status == "needs_review" {
2298 if let Err(e) = task_manager_for_spawn
2301 .update_task_status_with_duration(
2302 &ExternalId(task_id.clone()),
2303 Status::NeedsReview,
2304 Some(duration),
2305 )
2306 .await
2307 {
2308 tracing::error!(task_id, err = %e, "update_task_status(NeedsReview) failed — task may be stuck");
2309 }
2310 } else {
2312 let final_status = match display_status {
2314 "done" => Status::Done,
2315 "blocked" => Status::Blocked,
2316 _ => Status::New,
2317 };
2318 if let Err(e) = task_manager_for_spawn
2319 .update_task_status_with_duration(
2320 &ExternalId(task_id.clone()),
2321 final_status,
2322 Some(duration),
2323 )
2324 .await
2325 {
2326 tracing::warn!(task_id, ?e, "failed to update task status after completion");
2327 }
2328 }
2329 }
2330 Err(e) => {
2331 tracing::error!(task_id, ?e, "task runner failed");
2332 let duration = dispatch_start.elapsed().as_secs_f64();
2333 if is_internal_id(&task_id) {
2334 if let Err(ue) = task_manager_for_spawn
2336 .update_task_status_with_duration(
2337 &ExternalId(task_id.clone()),
2338 Status::NeedsReview,
2339 Some(duration),
2340 )
2341 .await
2342 {
2343 tracing::error!(task_id, err = %ue, "update_task_status(NeedsReview) failed — task may be stuck");
2344 }
2345 } else {
2346 if let Err(comment_err) = backend
2348 .post_comment(
2349 &ExternalId(task_id.clone()),
2350 &format!(
2351 "[{}] error: task runner failed: {e}{}",
2352 chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
2353 crate::engine::orch_footer()
2354 ),
2355 )
2356 .await
2357 {
2358 tracing::warn!(
2359 task_id,
2360 ?comment_err,
2361 "failed to post error comment to GitHub"
2362 );
2363 }
2364 if let Err(ue) = task_manager_for_spawn
2368 .update_task_status_with_duration(
2369 &ExternalId(task_id.clone()),
2370 Status::NeedsReview,
2371 Some(duration),
2372 )
2373 .await
2374 {
2375 tracing::error!(task_id, err = %ue, "update_task_status(NeedsReview) failed — task may be stuck");
2376 }
2377 }
2378 }
2379 }
2380
2381 capture.unregister_session(&repo_owned, &task_id_for_cleanup).await;
2383
2384 drop(permit);
2386 }));
2388 }
2389 Ok(())
2390}
2391
2392fn skips_agent(labels: &[String]) -> bool {
2393 labels
2394 .iter()
2395 .any(|label| label == "no-agent" || label == "owner:human")
2396}
2397
2398fn group_orphaned_routed_tasks(
2405 all_routed: Vec<crate::store::Task>,
2406 active_repos: &std::collections::HashSet<String>,
2407) -> std::collections::HashMap<String, Vec<crate::store::Task>> {
2408 let mut by_repo: std::collections::HashMap<String, Vec<crate::store::Task>> =
2409 std::collections::HashMap::new();
2410 for task in all_routed
2411 .into_iter()
2412 .filter(|t| !active_repos.contains(&t.repo))
2413 .filter(|t| !skips_agent(&t.labels))
2414 {
2415 by_repo.entry(task.repo.clone()).or_default().push(task);
2416 }
2417 by_repo
2418}
2419
2420#[expect(
2439 clippy::too_many_arguments,
2440 reason = "each argument is engine state the tick passes down"
2441)]
2442pub(crate) async fn dispatch_routed_tasks_global(
2443 tmux: &Arc<TmuxManager>,
2444 capture: &Arc<CaptureService>,
2445 semaphore: &Arc<Semaphore>,
2446 weight_tx: &mpsc::Sender<WeightSignal>,
2447 dispatch_mode: DispatchMode,
2448 dispatching: &Arc<DashMap<String, String>>,
2449 store: &Arc<TaskStore>,
2450 session_map: &std::collections::HashMap<String, bool>,
2451 active_repos: &std::collections::HashSet<String>,
2452) -> anyhow::Result<()> {
2453 use crate::store::TaskStatus;
2454
2455 let all_routed = store.list_all_by_status_global(TaskStatus::Routed).await?;
2456 let by_repo = group_orphaned_routed_tasks(all_routed, active_repos);
2457
2458 if by_repo.is_empty() {
2459 return Ok(());
2460 }
2461
2462 tracing::info!(
2463 repos = by_repo.len(),
2464 "global dispatch sweep: found routed tasks from inactive repos"
2465 );
2466
2467 for (repo, tasks) in by_repo {
2468 let backend: Arc<dyn ExternalBackend> = match crate::backends::github::GitHubBackend::new(
2469 repo.clone(),
2470 ) {
2471 Ok(b) => Arc::new(b),
2472 Err(e) => {
2473 tracing::warn!(repo = %repo, err = %e, "global sweep: failed to construct backend, skipping repo");
2474 continue;
2475 }
2476 };
2477 let task_manager = Arc::new(TaskManager::with_store(
2478 std::sync::Arc::clone(&backend),
2479 std::sync::Arc::clone(store),
2480 repo.clone(),
2481 ));
2482 let runner =
2483 Arc::new(TaskRunner::new(repo.clone()).with_store(std::sync::Arc::clone(store)));
2484 let dispatchable: Vec<ExternalTask> = tasks
2485 .iter()
2486 .map(crate::engine::tasks::store_task_to_external)
2487 .collect();
2488
2489 if let Err(e) = dispatch_tasks_for_repo(
2490 &backend,
2491 tmux,
2492 &repo,
2493 &runner,
2494 capture,
2495 semaphore,
2496 &task_manager,
2497 weight_tx,
2498 dispatch_mode,
2499 dispatching,
2500 store,
2501 session_map,
2502 dispatchable,
2503 )
2504 .await
2505 {
2506 tracing::warn!(repo = %repo, err = %e, "global sweep: dispatch failed for repo");
2507 }
2508 }
2509
2510 Ok(())
2511}
2512
2513pub(crate) async fn tick_unblock_parents(
2515 backend: &Arc<dyn ExternalBackend>,
2516 task_manager: &Arc<TaskManager>,
2517 store: &Arc<TaskStore>,
2518 repo: &str,
2519) -> anyhow::Result<()> {
2520 let blocked = task_manager.list_all_by_status(Status::Blocked).await?;
2521
2522 let store_blocked = store
2526 .list_by_status(repo, crate::store::TaskStatus::Blocked)
2527 .await?;
2528 let has_block_reason: std::collections::HashSet<String> = store_blocked
2529 .iter()
2530 .filter(|t| t.block_reason.is_some())
2531 .filter_map(|t| t.external_id.clone())
2532 .collect();
2533
2534 let handles: Vec<_> = blocked
2538 .into_iter()
2539 .filter(|task| {
2540 if has_block_reason.contains(&task.id.0) {
2541 tracing::debug!(
2542 task_id = task.id.0,
2543 "skipping blocked task with known block_reason"
2544 );
2545 false
2546 } else {
2547 true
2548 }
2549 })
2550 .map(|task| {
2551 let backend_clone = Arc::clone(backend);
2552 let task_manager_clone = Arc::clone(task_manager);
2553 let store_clone = Arc::clone(store);
2554 let repo_owned = repo.to_string();
2555 tokio::spawn(async move {
2556 let task_id = &task.id;
2557
2558 let mut children: Vec<ExternalId> = Vec::new();
2559 let mut child_statuses: std::collections::HashMap<
2560 String,
2561 crate::store::TaskStatus,
2562 > = std::collections::HashMap::new();
2563
2564 match store_clone.resolve_task_id(&repo_owned, &task_id.0).await {
2565 Ok(Some(parent_store_id)) => {
2566 match store_clone.list_children(&repo_owned, parent_store_id).await {
2567 Ok(store_children) => {
2568 for child in store_children {
2569 let Some(child_external_id) = child.external_id else {
2570 continue;
2571 };
2572 child_statuses.insert(child_external_id.clone(), child.status);
2573 children.push(ExternalId(child_external_id));
2574 }
2575 }
2576 Err(e) => {
2577 tracing::debug!(
2578 task_id = task_id.0,
2579 ?e,
2580 "failed to list store-linked children"
2581 );
2582 }
2583 }
2584 }
2585 Ok(None) => {}
2586 Err(e) => {
2587 tracing::debug!(
2588 task_id = task_id.0,
2589 ?e,
2590 "failed to resolve blocked task in store for child lookup"
2591 );
2592 }
2593 }
2594
2595 if !is_internal_id(&task_id.0) {
2596 match backend_clone.get_sub_issues(task_id).await {
2597 Ok(ids) => {
2598 let mut seen: std::collections::HashSet<String> =
2599 children.iter().map(|child| child.0.clone()).collect();
2600 for id in ids {
2601 if seen.insert(id.0.clone()) {
2602 children.push(id);
2603 }
2604 }
2605 }
2606 Err(e) => {
2607 tracing::debug!(task_id = task_id.0, ?e, "failed to get sub-issues");
2608 }
2609 }
2610 }
2611
2612 if children.is_empty() {
2614 return;
2615 }
2616
2617 let mut all_done = true;
2620 let child_ext_strs: Vec<String> =
2622 children.iter().map(|c| c.0.clone()).collect();
2623 let child_exts: Vec<&str> =
2624 child_ext_strs.iter().map(std::string::String::as_str).collect();
2625 match store_clone
2627 .get_statuses_by_external_ids(&repo_owned, &child_exts)
2628 .await
2629 { Ok(map) => {
2630 for (k, v) in map {
2631 child_statuses.insert(k, v);
2632 }
2633 } _ => {
2634 tracing::debug!(
2635 task_id = task_id.0,
2636 "failed to batch-query store for child statuses; falling back to per-child lookups"
2637 );
2638 }}
2639
2640 for child_id in &children {
2641 if let Some(status) = child_statuses.get(&child_id.0) {
2643 if *status != crate::store::TaskStatus::Done {
2644 all_done = false;
2645 break;
2646 }
2647 continue;
2648 }
2649
2650 match backend_clone.get_task(child_id).await {
2652 Ok(child) => {
2653 if !child.labels.iter().any(|l| l == Status::Done.as_label()) {
2654 all_done = false;
2655 break;
2656 }
2657 }
2658 Err(e) => {
2659 tracing::debug!(task_id = task_id.0, child = %child_id.0, ?e, "failed to fetch child task from backend — treating as not done");
2660 all_done = false;
2661 break;
2662 }
2663 }
2664 }
2665
2666 if all_done {
2667 tracing::info!(
2668 task_id = task_id.0,
2669 children = children.len(),
2670 "all children done, unblocking parent"
2671 );
2672 if let Err(e) =
2673 task_manager_clone.update_task_status(task_id, Status::New).await
2674 {
2675 tracing::warn!(task_id = task_id.0, ?e, "failed to unblock parent");
2676 }
2677 }
2678 })
2679 })
2680 .collect();
2681
2682 for handle in handles {
2683 if let Err(e) = handle.await {
2684 tracing::warn!(?e, "phase 4 sub-issue check panicked");
2685 }
2686 }
2687 Ok(())
2688}
2689
2690pub(crate) async fn tick_job_scheduler(
2692 jobs_path: &std::path::Path,
2693 backend: &Arc<dyn ExternalBackend>,
2694 store: Option<&Arc<TaskStore>>,
2695 repo: &str,
2696 transport: Option<&Arc<crate::channels::transport::Transport>>,
2697) -> anyhow::Result<()> {
2698 jobs::tick(jobs_path, backend, store, repo, transport).await
2699}
2700
2701#[expect(
2711 clippy::too_many_arguments,
2712 reason = "each argument is engine state the tick passes down"
2713)]
2714pub(crate) async fn tick_before_routing(
2715 backend: &Arc<dyn ExternalBackend>,
2716 tmux: &Arc<TmuxManager>,
2717 repo: &str,
2718 capture: &Arc<CaptureService>,
2719 config: &EngineConfig,
2720 task_manager: &Arc<TaskManager>,
2721 dispatching: &Arc<DashMap<String, String>>,
2722 store: &Arc<TaskStore>,
2723 session_map: &std::collections::HashMap<String, bool>,
2724) {
2725 let mut phases: Vec<(&str, u128)> = Vec::new();
2726 let mut at = Instant::now();
2727
2728 if let Err(e) = tick_check_session_completions(tmux, repo, capture, store).await {
2729 tracing::warn!(err = %e, "session completion phase failed");
2730 }
2731 phases.push(("sessions", at.elapsed().as_millis()));
2732
2733 if !store.is_leader() {
2736 super::log_phase_timings("tick.pre_routing", repo, &phases);
2737 return;
2738 }
2739
2740 at = Instant::now();
2741 if let Err(e) =
2742 tick_detect_silent_agents(tmux, repo, capture, backend, task_manager, config, store).await
2743 {
2744 tracing::warn!(err = %e, "silent-agent detection phase failed");
2745 }
2746 phases.push(("silent_agents", at.elapsed().as_millis()));
2747
2748 at = Instant::now();
2749 if let Err(e) = tick_recover_stuck_tasks(
2750 backend,
2751 tmux,
2752 repo,
2753 task_manager,
2754 config,
2755 store,
2756 session_map,
2757 dispatching,
2758 )
2759 .await
2760 {
2761 tracing::warn!(err = %e, "stuck-task recovery phase failed");
2762 }
2763 phases.push(("stuck_tasks", at.elapsed().as_millis()));
2764
2765 super::log_phase_timings("tick.pre_routing", repo, &phases);
2766}
2767
2768#[expect(
2776 clippy::too_many_arguments,
2777 reason = "each argument is engine state the tick passes down"
2778)]
2779pub(crate) async fn tick_after_routing(
2780 backend: &Arc<dyn ExternalBackend>,
2781 tmux: &Arc<TmuxManager>,
2782 repo: &str,
2783 runner: &Arc<TaskRunner>,
2784 capture: &Arc<CaptureService>,
2785 semaphore: &Arc<Semaphore>,
2786 project_dir: &std::path::Path,
2787 task_manager: &Arc<TaskManager>,
2788 weight_tx: &mpsc::Sender<WeightSignal>,
2789 dispatch_mode: DispatchMode,
2790 dispatching: &Arc<DashMap<String, String>>,
2791 store: &Arc<TaskStore>,
2792 transport: Option<&Arc<crate::channels::transport::Transport>>,
2793 session_map: &std::collections::HashMap<String, bool>,
2794) {
2795 let mut phases: Vec<(&str, u128)> = Vec::new();
2796 let mut at = Instant::now();
2797
2798 if let Err(e) = tick_dispatch_tasks(
2799 backend,
2800 tmux,
2801 repo,
2802 runner,
2803 capture,
2804 semaphore,
2805 task_manager,
2806 weight_tx,
2807 dispatch_mode,
2808 dispatching,
2809 store,
2810 session_map,
2811 )
2812 .await
2813 {
2814 tracing::warn!(err = %e, "dispatch phase failed");
2815 }
2816 phases.push(("dispatch", at.elapsed().as_millis()));
2817
2818 at = Instant::now();
2819 if let Err(e) = tick_unblock_parents(backend, task_manager, store, repo).await {
2820 tracing::warn!(err = %e, "parent-unblock phase failed");
2821 }
2822 phases.push(("unblock_parents", at.elapsed().as_millis()));
2823
2824 at = Instant::now();
2825 let jobs_path = project_dir.join(".orch.yml");
2826 if let Err(e) = tick_job_scheduler(&jobs_path, backend, Some(store), repo, transport).await {
2827 tracing::error!(?e, "job scheduler tick failed");
2828 }
2829 phases.push(("jobs", at.elapsed().as_millis()));
2830
2831 super::log_phase_timings("tick.post_routing", repo, &phases);
2832}
2833
2834#[expect(
2844 clippy::too_many_arguments,
2845 reason = "each argument is engine state the tick passes down"
2846)]
2847pub(crate) async fn tick_all(
2848 engines: &[ProjectEngine],
2849 tmux: &Arc<TmuxManager>,
2850 capture: &Arc<CaptureService>,
2851 semaphore: &Arc<Semaphore>,
2852 config: &EngineConfig,
2853 router: &Arc<tokio::sync::RwLock<Router>>,
2854 weight_tx: &mpsc::Sender<WeightSignal>,
2855 dispatching: &Arc<DashMap<String, String>>,
2856 transport: Option<&Arc<crate::channels::transport::Transport>>,
2857 active_repos: &std::collections::HashSet<String>,
2858 auto_merge_in_flight: &Arc<DashSet<String>>,
2859) {
2860 let _tick_span = tracing::info_span!("engine.tick").entered();
2861
2862 let Some(first) = engines.first() else {
2863 return;
2864 };
2865 let store = &first.store;
2868
2869 for (project, call_type, calls) in crate::github::http::drain_call_counts()
2871 .into_iter()
2872 .take(10)
2873 {
2874 tracing::debug!(
2875 project,
2876 call_type,
2877 calls,
2878 "github api calls since last tick"
2879 );
2880 }
2881
2882 let session_map = tmux.batch_session_active().await;
2884
2885 let mut passes = Vec::with_capacity(engines.len());
2886 for engine in engines {
2887 passes.push(REPO_CONTEXT.scope(
2888 engine.repo.clone(),
2889 tick_before_routing(
2890 &engine.backend,
2891 tmux,
2892 &engine.repo,
2893 capture,
2894 config,
2895 &engine.task_manager,
2896 dispatching,
2897 &engine.store,
2898 &session_map,
2899 ),
2900 ));
2901 }
2902 futures::future::join_all(passes).await;
2903
2904 if !store.is_leader() {
2905 return;
2906 }
2907
2908 let at = Instant::now();
2912 {
2913 let router_guard = router.read().await;
2914 tick_global_sweeps(
2915 &first.task_manager,
2916 store,
2917 &router_guard,
2918 dispatching,
2919 auto_merge_in_flight,
2920 )
2921 .await;
2922 }
2923 let sweeps_ms = at.elapsed().as_millis();
2924
2925 let at = Instant::now();
2926 recover_untracked_repo_stuck_tasks(
2927 tmux,
2928 config,
2929 store,
2930 &session_map,
2931 dispatching,
2932 active_repos,
2933 )
2934 .await;
2935 let untracked_ms = at.elapsed().as_millis();
2936
2937 let at = Instant::now();
2938 if let Err(err) = crate::engine::dependencies::reconcile_dependencies(
2939 first.backend.as_ref(),
2940 store,
2941 transport.map(std::convert::AsRef::as_ref),
2942 )
2943 .await
2944 {
2945 tracing::warn!(err = %err, "dependency reconcile failed");
2946 }
2947 super::log_phase_timings(
2948 "tick.global",
2949 &first.repo,
2950 &[
2951 ("global_sweeps", sweeps_ms),
2952 ("untracked_stuck", untracked_ms),
2953 ("dependencies", at.elapsed().as_millis()),
2954 ],
2955 );
2956
2957 {
2958 let mut router_guard = router.write().await;
2959 let mut routing_quota =
2960 RoutingTickQuota::new(crate::engine::router::config::max_tasks_per_routing_tick());
2961 for engine in engines {
2962 let at = Instant::now();
2963 REPO_CONTEXT
2964 .scope(engine.repo.clone(), async {
2965 if let Err(e) = tick_route_tasks(
2966 &engine.backend,
2967 &engine.task_manager,
2968 &mut router_guard,
2969 &engine.store,
2970 &engine.repo,
2971 &mut routing_quota,
2972 )
2973 .await
2974 {
2975 tracing::warn!(err = %e, "routing phase failed");
2976 }
2977 })
2978 .await;
2979 super::log_phase_timings(
2980 "tick.routing",
2981 &engine.repo,
2982 &[("route", at.elapsed().as_millis())],
2983 );
2984 }
2985
2986 if let Err(e) =
2990 route_new_tasks_global(&mut router_guard, store, active_repos, &mut routing_quota).await
2991 {
2992 tracing::warn!(err = %e, "global new-task routing sweep failed");
2993 }
2994 }
2995
2996 let dispatch_mode = dispatch_mode_from_router(&*router.read().await);
2997
2998 let session_map = tmux.batch_session_active().await;
3001
3002 let mut passes = Vec::with_capacity(engines.len());
3003 for engine in engines {
3004 passes.push(REPO_CONTEXT.scope(
3005 engine.repo.clone(),
3006 tick_after_routing(
3007 &engine.backend,
3008 tmux,
3009 &engine.repo,
3010 &engine.runner,
3011 capture,
3012 semaphore,
3013 &engine.project_dir,
3014 &engine.task_manager,
3015 weight_tx,
3016 dispatch_mode,
3017 dispatching,
3018 &engine.store,
3019 transport,
3020 &session_map,
3021 ),
3022 ));
3023 }
3024 futures::future::join_all(passes).await;
3025
3026 if let Err(e) = dispatch_routed_tasks_global(
3032 tmux,
3033 capture,
3034 semaphore,
3035 weight_tx,
3036 dispatch_mode,
3037 dispatching,
3038 store,
3039 &session_map,
3040 active_repos,
3041 )
3042 .await
3043 {
3044 tracing::warn!(err = %e, "global routed-task dispatch sweep failed");
3045 }
3046}
3047
3048#[cfg(test)]
3049mod tests {
3050 use super::*;
3051 use crate::backends::{ExternalId, ExternalTask, Mention, Status};
3052 use crate::channels::transport::Transport;
3053 use crate::engine::tasks::TaskManager;
3054 use crate::store::TaskStore;
3055 use crate::tmux::{SessionGuard, TmuxManager};
3056 use async_trait::async_trait;
3057 use std::sync::{Arc, Mutex};
3058
3059 #[test]
3060 fn skips_agent_for_human_and_no_agent_labels() {
3061 assert!(skips_agent(&["owner:human".to_string()]));
3062 assert!(skips_agent(&["no-agent".to_string()]));
3063 assert!(!skips_agent(&["status:new".to_string()]));
3064 }
3065
3066 fn cooled_review_router_for_test() -> Router {
3070 Router::new_for_test(
3071 crate::engine::router::RouterConfig::default(),
3072 vec!["claude".to_string()],
3073 )
3074 }
3075
3076 struct MockBackend {
3080 blocked_tasks: Vec<ExternalTask>,
3082 sub_issues: std::collections::HashMap<String, Vec<ExternalId>>,
3084 tasks_by_id: std::collections::HashMap<String, ExternalTask>,
3086 get_sub_issues_calls: Arc<Mutex<Vec<String>>>,
3088 status_updates: Arc<Mutex<Vec<(String, Status)>>>,
3090 removed_labels: Arc<Mutex<Vec<(String, String)>>>,
3092 posted_comments: Arc<Mutex<Vec<(String, String)>>>,
3094 }
3095
3096 impl MockBackend {
3097 fn new() -> Self {
3098 Self {
3099 blocked_tasks: vec![],
3100 sub_issues: std::collections::HashMap::new(),
3101 tasks_by_id: std::collections::HashMap::new(),
3102 get_sub_issues_calls: Arc::new(Mutex::new(vec![])),
3103 status_updates: Arc::new(Mutex::new(vec![])),
3104 removed_labels: Arc::new(Mutex::new(vec![])),
3105 posted_comments: Arc::new(Mutex::new(vec![])),
3106 }
3107 }
3108 }
3109
3110 fn make_task(id: &str, labels: &[&str]) -> ExternalTask {
3111 ExternalTask {
3112 id: ExternalId(id.to_string()),
3113 title: format!("Task {id}"),
3114 body: String::new(),
3115 state: "open".to_string(),
3116 labels: labels
3117 .iter()
3118 .map(std::string::ToString::to_string)
3119 .collect(),
3120 author: "bot".to_string(),
3121 created_at: "2026-01-01T00:00:00Z".to_string(),
3122 updated_at: "2026-01-01T00:00:00Z".to_string(),
3123 url: format!("https://github.com/test/test/issues/{id}"),
3124 }
3125 }
3126
3127 #[async_trait]
3128 impl ExternalBackend for MockBackend {
3129 fn name(&self) -> &'static str {
3130 "mock"
3131 }
3132 async fn create_task(
3133 &self,
3134 _title: &str,
3135 _body: &str,
3136 _labels: &[String],
3137 ) -> anyhow::Result<ExternalId> {
3138 Ok(ExternalId("new".to_string()))
3139 }
3140 async fn get_task(&self, id: &ExternalId) -> anyhow::Result<ExternalTask> {
3141 self.tasks_by_id
3142 .get(&id.0)
3143 .cloned()
3144 .ok_or_else(|| anyhow::anyhow!("task not found: {}", id.0))
3145 }
3146 async fn list_by_status(&self, status: Status) -> anyhow::Result<Vec<ExternalTask>> {
3147 if status == Status::Blocked {
3148 Ok(self.blocked_tasks.clone())
3149 } else {
3150 Ok(vec![])
3151 }
3152 }
3153 async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
3154 Ok(vec![])
3155 }
3156 async fn post_comment(&self, _id: &ExternalId, _body: &str) -> anyhow::Result<()> {
3157 self.posted_comments
3158 .lock()
3159 .unwrap()
3160 .push((_id.0.clone(), _body.to_string()));
3161 Ok(())
3162 }
3163 async fn set_labels(&self, _id: &ExternalId, _labels: &[String]) -> anyhow::Result<()> {
3164 Ok(())
3165 }
3166 async fn remove_label(&self, _id: &ExternalId, _label: &str) -> anyhow::Result<()> {
3167 self.removed_labels
3168 .lock()
3169 .unwrap()
3170 .push((_id.0.clone(), _label.to_string()));
3171 Ok(())
3172 }
3173 async fn get_sub_issues(&self, id: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
3174 self.get_sub_issues_calls.lock().unwrap().push(id.0.clone());
3175 Ok(self.sub_issues.get(&id.0).cloned().unwrap_or_default())
3176 }
3177 async fn create_sub_task(
3178 &self,
3179 _parent: &ExternalId,
3180 _title: &str,
3181 _body: &str,
3182 _labels: &[String],
3183 ) -> anyhow::Result<ExternalId> {
3184 Ok(ExternalId("child".to_string()))
3185 }
3186 async fn ensure_status_label(&self, _label: &str) -> anyhow::Result<()> {
3187 Ok(())
3188 }
3189 async fn has_open_issue_with_title(
3190 &self,
3191 _title: &str,
3192 _label: &str,
3193 ) -> anyhow::Result<bool> {
3194 Ok(false)
3195 }
3196 async fn health_check(&self) -> anyhow::Result<()> {
3197 Ok(())
3198 }
3199 async fn is_pr_merged(&self, _branch: &str) -> anyhow::Result<bool> {
3200 Ok(false)
3201 }
3202 async fn get_authenticated_user(&self) -> anyhow::Result<Option<String>> {
3203 Ok(Some("testbot".to_string()))
3204 }
3205 async fn get_mentions(&self, _since: &str) -> anyhow::Result<Vec<Mention>> {
3206 Ok(vec![])
3207 }
3208 async fn update_status(&self, id: &ExternalId, status: Status) -> anyhow::Result<()> {
3209 self.status_updates
3210 .lock()
3211 .unwrap()
3212 .push((id.0.clone(), status));
3213 Ok(())
3214 }
3215 }
3216
3217 fn make_task_manager(backend: Arc<dyn ExternalBackend>) -> Arc<TaskManager> {
3218 Arc::new(TaskManager::new(backend))
3219 }
3220
3221 #[tokio::test]
3224 async fn unblock_parents_unblocks_when_all_children_done() {
3225 let mut mock = MockBackend::new();
3226
3227 let parent = make_task("10", &["status:blocked"]);
3229 mock.blocked_tasks.push(parent.clone());
3230 mock.sub_issues.insert(
3231 "10".to_string(),
3232 vec![ExternalId("11".to_string()), ExternalId("12".to_string())],
3233 );
3234 mock.tasks_by_id
3235 .insert("11".to_string(), make_task("11", &["status:done"]));
3236 mock.tasks_by_id
3237 .insert("12".to_string(), make_task("12", &["status:done"]));
3238
3239 let status_updates = std::sync::Arc::clone(&mock.status_updates);
3240 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3241 let task_manager = make_task_manager(std::sync::Arc::clone(&backend));
3242 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3243
3244 tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3245 .await
3246 .unwrap();
3247
3248 let updates = status_updates.lock().unwrap();
3249 assert_eq!(updates.len(), 1, "parent should be unblocked");
3250 assert_eq!(updates[0], ("10".to_string(), Status::New));
3251 }
3252
3253 #[tokio::test]
3254 async fn unblock_parents_skips_when_child_not_done() {
3255 let mut mock = MockBackend::new();
3256
3257 let parent = make_task("20", &["status:blocked"]);
3258 mock.blocked_tasks.push(parent.clone());
3259 mock.sub_issues.insert(
3260 "20".to_string(),
3261 vec![ExternalId("21".to_string()), ExternalId("22".to_string())],
3262 );
3263 mock.tasks_by_id
3265 .insert("21".to_string(), make_task("21", &["status:done"]));
3266 mock.tasks_by_id
3267 .insert("22".to_string(), make_task("22", &["status:in_progress"]));
3268
3269 let status_updates = std::sync::Arc::clone(&mock.status_updates);
3270 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3271 let task_manager = make_task_manager(std::sync::Arc::clone(&backend));
3272 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3273
3274 tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3275 .await
3276 .unwrap();
3277
3278 let updates = status_updates.lock().unwrap();
3279 assert!(
3280 updates.is_empty(),
3281 "should not unblock when a child is still running"
3282 );
3283 }
3284
3285 #[tokio::test]
3286 async fn unblock_parents_skips_task_with_no_children() {
3287 let mut mock = MockBackend::new();
3288
3289 let parent = make_task("30", &["status:blocked"]);
3291 mock.blocked_tasks.push(parent.clone());
3292 let status_updates = std::sync::Arc::clone(&mock.status_updates);
3295 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3296 let task_manager = make_task_manager(std::sync::Arc::clone(&backend));
3297 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3298
3299 tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3300 .await
3301 .unwrap();
3302
3303 let updates = status_updates.lock().unwrap();
3304 assert!(
3305 updates.is_empty(),
3306 "should not unblock tasks with no sub-issues"
3307 );
3308 }
3309
3310 #[tokio::test]
3311 async fn unblock_parents_skips_task_with_block_reason_in_store() {
3312 let mut mock = MockBackend::new();
3315
3316 let parent = make_task("60", &["status:blocked"]);
3318 mock.blocked_tasks.push(parent.clone());
3319 mock.sub_issues
3322 .insert("60".to_string(), vec![ExternalId("61".to_string())]);
3323 mock.tasks_by_id
3324 .insert("61".to_string(), make_task("61", &["status:done"]));
3325
3326 let status_updates = std::sync::Arc::clone(&mock.status_updates);
3327 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3328 let task_manager = make_task_manager(std::sync::Arc::clone(&backend));
3329
3330 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3332 let task_id = store
3333 .upsert_external(&crate::store::UpsertExternal {
3334 repo: "test/repo",
3335 ext_id: "60",
3336 title: "Task 60",
3337 body: "",
3338 author: "bot",
3339 url: "https://github.com/test/test/issues/60",
3340 labels: &[],
3341 origin: "github",
3342 })
3343 .await
3344 .unwrap();
3345 store
3346 .update_status(task_id, crate::store::TaskStatus::Blocked)
3347 .await
3348 .unwrap();
3349 store
3350 .set_block_reason(task_id, Some("CI failure limit reached"))
3351 .await
3352 .unwrap();
3353
3354 tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3355 .await
3356 .unwrap();
3357
3358 let updates = status_updates.lock().unwrap();
3359 assert!(
3360 updates.is_empty(),
3361 "task with block_reason should be skipped — no API call, no unblock"
3362 );
3363 }
3364
3365 #[tokio::test]
3366 async fn unblock_parents_unblocks_internal_parent_from_store_children() {
3367 let mock = MockBackend::new();
3368 let get_sub_issues_calls = std::sync::Arc::clone(&mock.get_sub_issues_calls);
3369 let status_updates = std::sync::Arc::clone(&mock.status_updates);
3370 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3371 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3372 let task_manager = Arc::new(TaskManager::with_store(
3373 std::sync::Arc::clone(&backend),
3374 std::sync::Arc::clone(&store),
3375 "test/repo".to_string(),
3376 ));
3377
3378 let parent_id = store
3379 .create_internal("test/repo", "Parent", "", "job", "daily", None)
3380 .await
3381 .unwrap();
3382 store
3383 .update_status(parent_id, crate::store::TaskStatus::Blocked)
3384 .await
3385 .unwrap();
3386 let child_id = store
3387 .create(&crate::store::NewTask {
3388 external_id: Some("91".to_string()),
3389 repo: "test/repo".to_string(),
3390 origin: "github".to_string(),
3391 title: "Child".to_string(),
3392 body: String::new(),
3393 source: String::new(),
3394 source_id: String::new(),
3395 author: String::new(),
3396 url: String::new(),
3397 labels: vec![],
3398 parent_id: Some(parent_id),
3399 })
3400 .await
3401 .unwrap();
3402 store
3403 .update_status(child_id, crate::store::TaskStatus::Done)
3404 .await
3405 .unwrap();
3406
3407 tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3408 .await
3409 .unwrap();
3410
3411 let parent = store.get(parent_id).await.unwrap();
3412 assert_eq!(parent.status, crate::store::TaskStatus::New);
3413 assert!(
3414 status_updates.lock().unwrap().is_empty(),
3415 "internal parent should be updated in store without backend mirroring"
3416 );
3417 assert!(
3418 get_sub_issues_calls.lock().unwrap().is_empty(),
3419 "internal parent should not query backend sub-issues"
3420 );
3421 }
3422
3423 #[cfg(test)]
3427 async fn set_task_updated_at_past(store: &TaskStore, task_id: i64) {
3428 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
3429 .bind(task_id)
3430 .execute(store.pool())
3431 .await
3432 .unwrap();
3433 }
3434
3435 async fn set_in_review_entered_at_past(store: &TaskStore, task_id: i64) {
3437 sqlx::query(
3438 "UPDATE task_activity SET timestamp = '2020-01-01T00:00:00Z'
3439 WHERE task_id = ? AND to_status = 'in_review'",
3440 )
3441 .bind(task_id)
3442 .execute(store.pool())
3443 .await
3444 .unwrap();
3445 }
3446
3447 #[tokio::test]
3448 async fn recover_stuck_tasks_resets_external_in_review_to_needs_review() {
3449 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3450 let mock = MockBackend::new();
3451 let status_updates = std::sync::Arc::clone(&mock.status_updates);
3452 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3453 let task_manager = Arc::new(TaskManager::with_store(
3454 std::sync::Arc::clone(&backend),
3455 std::sync::Arc::clone(&store),
3456 "owner/repo".to_string(),
3457 ));
3458 let tmux = Arc::new(TmuxManager::new());
3459 let config = EngineConfig {
3460 no_session_stuck_timeout: 600,
3461 stuck_timeout: 1800,
3462 ..EngineConfig::default()
3463 };
3464
3465 let id = store
3467 .upsert_external(&crate::store::UpsertExternal {
3468 repo: "owner/repo",
3469 ext_id: "99",
3470 title: "Stuck InReview",
3471 body: "",
3472 author: "",
3473 url: "",
3474 labels: &[],
3475 origin: "github",
3476 })
3477 .await
3478 .unwrap();
3479 store
3480 .update_status(id, crate::store::TaskStatus::InReview)
3481 .await
3482 .unwrap();
3483 set_in_review_entered_at_past(&store, id).await;
3484 set_task_updated_at_past(&store, id).await;
3485
3486 tick_recover_stuck_tasks(
3487 &backend,
3488 &tmux,
3489 "owner/repo",
3490 &task_manager,
3491 &config,
3492 &store,
3493 &std::collections::HashMap::new(),
3494 &Arc::new(DashMap::new()),
3495 )
3496 .await
3497 .unwrap();
3498
3499 let updates = status_updates.lock().unwrap();
3500 assert_eq!(updates.len(), 1, "stuck InReview task should be recovered");
3501 assert_eq!(updates[0].0, "99");
3502 assert_eq!(updates[0].1, Status::NeedsReview);
3503 }
3504
3505 #[tokio::test]
3506 async fn recover_stuck_tasks_skips_recent_in_review() {
3507 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3508 let mock = MockBackend::new();
3509 let status_updates = std::sync::Arc::clone(&mock.status_updates);
3510 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3511 let task_manager = Arc::new(TaskManager::with_store(
3512 std::sync::Arc::clone(&backend),
3513 std::sync::Arc::clone(&store),
3514 "owner/repo".to_string(),
3515 ));
3516 let tmux = Arc::new(TmuxManager::new());
3517 let config = EngineConfig {
3518 no_session_stuck_timeout: 600, stuck_timeout: 1800,
3520 ..EngineConfig::default()
3521 };
3522
3523 let id = store
3525 .upsert_external(&crate::store::UpsertExternal {
3526 repo: "owner/repo",
3527 ext_id: "100",
3528 title: "Recent InReview",
3529 body: "",
3530 author: "",
3531 url: "",
3532 labels: &[],
3533 origin: "github",
3534 })
3535 .await
3536 .unwrap();
3537 store
3538 .update_status(id, crate::store::TaskStatus::InReview)
3539 .await
3540 .unwrap();
3541 set_task_updated_at_past(&store, id).await;
3542
3543 tick_recover_stuck_tasks(
3544 &backend,
3545 &tmux,
3546 "owner/repo",
3547 &task_manager,
3548 &config,
3549 &store,
3550 &std::collections::HashMap::new(),
3551 &Arc::new(DashMap::new()),
3552 )
3553 .await
3554 .unwrap();
3555
3556 let updates = status_updates.lock().unwrap();
3557 assert!(
3558 updates.is_empty(),
3559 "recent InReview task should not be recovered"
3560 );
3561 }
3562
3563 #[tokio::test]
3564 async fn recover_stuck_tasks_resets_internal_in_review_to_needs_review() {
3565 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3566 let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
3567 let task_manager = Arc::new(TaskManager::with_store(
3568 std::sync::Arc::clone(&backend),
3569 std::sync::Arc::clone(&store),
3570 "owner/repo".to_string(),
3571 ));
3572 let tmux = Arc::new(TmuxManager::new());
3573 let config = EngineConfig {
3574 no_session_stuck_timeout: 600,
3575 stuck_timeout: 1800,
3576 ..EngineConfig::default()
3577 };
3578
3579 let id = store
3581 .create_internal("owner/repo", "Internal InReview", "", "cron", "1", None)
3582 .await
3583 .unwrap();
3584 store
3585 .update_status(id, crate::store::TaskStatus::InReview)
3586 .await
3587 .unwrap();
3588 set_in_review_entered_at_past(&store, id).await;
3589 set_task_updated_at_past(&store, id).await;
3590
3591 tick_recover_stuck_tasks(
3592 &backend,
3593 &tmux,
3594 "owner/repo",
3595 &task_manager,
3596 &config,
3597 &store,
3598 &std::collections::HashMap::new(),
3599 &Arc::new(DashMap::new()),
3600 )
3601 .await
3602 .unwrap();
3603
3604 let task = store.get(id).await.unwrap();
3605 assert_eq!(
3606 task.status,
3607 crate::store::TaskStatus::NeedsReview,
3608 "stuck internal InReview task should be reset to NeedsReview"
3609 );
3610 }
3611
3612 #[tokio::test]
3613 async fn recover_stuck_tasks_ages_in_review_from_status_entry_not_updated_at() {
3614 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3615 let mock = MockBackend::new();
3616 let status_updates = std::sync::Arc::clone(&mock.status_updates);
3617 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3618 let task_manager = Arc::new(TaskManager::with_store(
3619 std::sync::Arc::clone(&backend),
3620 std::sync::Arc::clone(&store),
3621 "owner/repo".to_string(),
3622 ));
3623 let tmux = Arc::new(TmuxManager::new());
3624 let config = EngineConfig {
3625 no_session_stuck_timeout: 600,
3626 stuck_timeout: 1800,
3627 ..EngineConfig::default()
3628 };
3629
3630 let id = store
3631 .upsert_external(&crate::store::UpsertExternal {
3632 repo: "owner/repo",
3633 ext_id: "102",
3634 title: "Fresh updated_at InReview",
3635 body: "",
3636 author: "",
3637 url: "",
3638 labels: &[],
3639 origin: "github",
3640 })
3641 .await
3642 .unwrap();
3643 store
3644 .update_status(id, crate::store::TaskStatus::InReview)
3645 .await
3646 .unwrap();
3647 set_in_review_entered_at_past(&store, id).await;
3648 store
3649 .set_fields(id, &[("pr_number", serde_json::json!(42))])
3650 .await
3651 .unwrap();
3652
3653 tick_recover_stuck_tasks(
3654 &backend,
3655 &tmux,
3656 "owner/repo",
3657 &task_manager,
3658 &config,
3659 &store,
3660 &std::collections::HashMap::new(),
3661 &Arc::new(DashMap::new()),
3662 )
3663 .await
3664 .unwrap();
3665
3666 let updates = status_updates.lock().unwrap();
3667 assert_eq!(
3668 updates.len(),
3669 1,
3670 "in_review stuck recovery must age from the status entry even when updated_at is fresh"
3671 );
3672 assert_eq!(updates[0].0, "102");
3673 assert_eq!(updates[0].1, Status::NeedsReview);
3674 }
3675
3676 async fn orphaned_review_status_after_recovery(
3679 with_decision: bool,
3680 ) -> crate::store::TaskStatus {
3681 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3682 let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
3683 let task_manager = Arc::new(TaskManager::with_store(
3684 std::sync::Arc::clone(&backend),
3685 std::sync::Arc::clone(&store),
3686 "owner/repo".to_string(),
3687 ));
3688 let tmux = Arc::new(TmuxManager::new());
3689
3690 let id = store
3691 .upsert_external(&crate::store::UpsertExternal {
3692 repo: "owner/repo",
3693 ext_id: "55",
3694 title: "Orphaned review",
3695 body: "",
3696 author: "",
3697 url: "",
3698 labels: &[],
3699 origin: "github",
3700 })
3701 .await
3702 .unwrap();
3703 store
3704 .update_status(id, crate::store::TaskStatus::InReview)
3705 .await
3706 .unwrap();
3707 set_in_review_entered_at_past(&store, id).await;
3709 if with_decision {
3710 crate::store::store_log_activity(
3711 &Some(std::sync::Arc::clone(&store)),
3712 "owner/repo",
3713 "55",
3714 "review_decision",
3715 None,
3716 None,
3717 None::<&str>,
3718 None::<&str>,
3719 Some(&serde_json::json!({"decision": "approve"})),
3720 )
3721 .await;
3722 }
3723
3724 tick_recover_stuck_tasks(
3725 &backend,
3726 &tmux,
3727 "owner/repo",
3728 &task_manager,
3729 &EngineConfig::default(),
3730 &store,
3731 &std::collections::HashMap::new(),
3732 &Arc::new(DashMap::new()),
3733 )
3734 .await
3735 .unwrap();
3736
3737 store.get(id).await.unwrap().status
3738 }
3739
3740 #[tokio::test]
3741 async fn stale_in_review_recovery_resets_orphaned_task_without_session() {
3742 assert_eq!(
3743 orphaned_review_status_after_recovery(false).await,
3744 crate::store::TaskStatus::NeedsReview
3745 );
3746 }
3747
3748 #[tokio::test]
3749 async fn stale_in_review_recovery_leaves_decided_task_waiting_on_merge() {
3750 assert_eq!(
3751 orphaned_review_status_after_recovery(true).await,
3752 crate::store::TaskStatus::InReview
3753 );
3754 }
3755
3756 #[tokio::test]
3757 async fn recover_stuck_tasks_resets_review_agent_failures() {
3758 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3759 let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
3760 let task_manager = Arc::new(TaskManager::with_store(
3761 std::sync::Arc::clone(&backend),
3762 std::sync::Arc::clone(&store),
3763 "owner/repo".to_string(),
3764 ));
3765 let tmux = Arc::new(TmuxManager::new());
3766
3767 let id = store
3768 .upsert_external(&crate::store::UpsertExternal {
3769 repo: "owner/repo",
3770 ext_id: "104",
3771 title: "Orphaned review with failures",
3772 body: "",
3773 author: "",
3774 url: "",
3775 labels: &[],
3776 origin: "github",
3777 })
3778 .await
3779 .unwrap();
3780 store
3781 .update_status(id, crate::store::TaskStatus::InReview)
3782 .await
3783 .unwrap();
3784 store
3785 .set_fields(id, &[("review_agent_failures", serde_json::json!(2))])
3786 .await
3787 .unwrap();
3788 set_in_review_entered_at_past(&store, id).await;
3789
3790 tick_recover_stuck_tasks(
3791 &backend,
3792 &tmux,
3793 "owner/repo",
3794 &task_manager,
3795 &EngineConfig::default(),
3796 &store,
3797 &std::collections::HashMap::new(),
3798 &Arc::new(DashMap::new()),
3799 )
3800 .await
3801 .unwrap();
3802
3803 let task = store.get(id).await.unwrap();
3804 assert_eq!(task.status, crate::store::TaskStatus::NeedsReview);
3805 assert_eq!(
3806 task.review_agent_failures, 0,
3807 "a dead review session is an infrastructure event, not an agent failure"
3808 );
3809 }
3810
3811 #[tokio::test]
3812 async fn recover_stuck_tasks_reclaims_hung_in_review_session() {
3813 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3814 let mock = MockBackend::new();
3815 let status_updates = std::sync::Arc::clone(&mock.status_updates);
3816 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3817 let task_manager = Arc::new(TaskManager::with_store(
3818 std::sync::Arc::clone(&backend),
3819 std::sync::Arc::clone(&store),
3820 "owner/repo".to_string(),
3821 ));
3822 let tmux = Arc::new(TmuxManager::new());
3823 let config = EngineConfig {
3824 no_session_stuck_timeout: 600,
3825 stuck_timeout: 1800,
3826 ..EngineConfig::default()
3827 };
3828
3829 let id = store
3830 .upsert_external(&crate::store::UpsertExternal {
3831 repo: "owner/repo",
3832 ext_id: "103",
3833 title: "Hung review session",
3834 body: "",
3835 author: "",
3836 url: "",
3837 labels: &[],
3838 origin: "github",
3839 })
3840 .await
3841 .unwrap();
3842 store
3843 .update_status(id, crate::store::TaskStatus::InReview)
3844 .await
3845 .unwrap();
3846 set_in_review_entered_at_past(&store, id).await;
3847 store
3848 .set_fields(id, &[("pr_number", serde_json::json!(43))])
3849 .await
3850 .unwrap();
3851
3852 let session_name = tmux.session_name("owner/repo", "103-review");
3853 let mut session_map = std::collections::HashMap::new();
3854 session_map.insert(session_name, true);
3855
3856 tick_recover_stuck_tasks(
3857 &backend,
3858 &tmux,
3859 "owner/repo",
3860 &task_manager,
3861 &config,
3862 &store,
3863 &session_map,
3864 &Arc::new(DashMap::new()),
3865 )
3866 .await
3867 .unwrap();
3868
3869 let updates = status_updates.lock().unwrap();
3870 assert_eq!(
3871 updates.len(),
3872 1,
3873 "a live hung review session must be reclaimed once the in_review entry ages past stuck_timeout"
3874 );
3875 assert_eq!(updates[0].0, "103");
3876 assert_eq!(updates[0].1, Status::NeedsReview);
3877 }
3878
3879 #[tokio::test]
3880 async fn untracked_repo_recovery_resets_stale_in_progress_to_new() {
3881 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3882 let tmux = Arc::new(TmuxManager::new());
3883 let config = EngineConfig {
3884 no_session_stuck_timeout: 600,
3885 stuck_timeout: 1800,
3886 ..EngineConfig::default()
3887 };
3888
3889 let id = store
3891 .upsert_external(&crate::store::UpsertExternal {
3892 repo: "owner/other",
3893 ext_id: "777",
3894 title: "Cross-repo stuck task",
3895 body: "",
3896 author: "",
3897 url: "",
3898 labels: &[],
3899 origin: "github",
3900 })
3901 .await
3902 .unwrap();
3903 store
3904 .update_status(id, crate::store::TaskStatus::InProgress)
3905 .await
3906 .unwrap();
3907 set_task_updated_at_past(&store, id).await;
3908
3909 let active = std::collections::HashSet::from(["owner/repo".to_string()]);
3910 recover_untracked_repo_stuck_tasks(
3911 &tmux,
3912 &config,
3913 &store,
3914 &std::collections::HashMap::new(),
3915 &Arc::new(DashMap::new()),
3916 &active,
3917 )
3918 .await;
3919
3920 let task = store.get(id).await.unwrap();
3921 assert_eq!(
3922 task.status,
3923 crate::store::TaskStatus::New,
3924 "cross-repo stale in_progress task should be reset to New"
3925 );
3926 assert_eq!(task.route_attempts, 0, "route attempts should be cleared");
3927 assert!(task.agent.is_none(), "agent should be cleared");
3928 assert!(task.model.is_none(), "model should be cleared");
3929 }
3930
3931 #[tokio::test]
3932 async fn untracked_repo_recovery_leaves_active_repos_to_their_own_pass() {
3933 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3934 let tmux = Arc::new(TmuxManager::new());
3935 let config = EngineConfig {
3936 no_session_stuck_timeout: 600,
3937 stuck_timeout: 1800,
3938 ..EngineConfig::default()
3939 };
3940 let id = store
3941 .upsert_external(&crate::store::UpsertExternal {
3942 repo: "owner/other",
3943 ext_id: "778",
3944 title: "Stuck task in an active repo",
3945 body: "",
3946 author: "",
3947 url: "",
3948 labels: &[],
3949 origin: "github",
3950 })
3951 .await
3952 .unwrap();
3953 store
3954 .update_status(id, crate::store::TaskStatus::InProgress)
3955 .await
3956 .unwrap();
3957 set_task_updated_at_past(&store, id).await;
3958
3959 let active =
3961 std::collections::HashSet::from(["owner/repo".to_string(), "owner/other".to_string()]);
3962 recover_untracked_repo_stuck_tasks(
3963 &tmux,
3964 &config,
3965 &store,
3966 &std::collections::HashMap::new(),
3967 &Arc::new(DashMap::new()),
3968 &active,
3969 )
3970 .await;
3971
3972 assert_eq!(
3973 store.get(id).await.unwrap().status,
3974 crate::store::TaskStatus::InProgress
3975 );
3976 }
3977
3978 #[tokio::test]
3979 async fn recover_stuck_tasks_finalizes_open_task_runs_rows() {
3980 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3981 let mock = MockBackend::new();
3982 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3983 let task_manager = Arc::new(TaskManager::with_store(
3984 std::sync::Arc::clone(&backend),
3985 std::sync::Arc::clone(&store),
3986 "owner/repo".to_string(),
3987 ));
3988 let tmux = Arc::new(TmuxManager::new());
3989 let config = EngineConfig {
3990 no_session_stuck_timeout: 600,
3991 stuck_timeout: 1800,
3992 ..EngineConfig::default()
3993 };
3994
3995 let id = store
3996 .create_internal("owner/repo", "Stuck with open run", "", "cron", "1", None)
3997 .await
3998 .unwrap();
3999 store
4000 .set_fields(
4001 id,
4002 &[
4003 ("agent", serde_json::json!("claude")),
4004 ("model", serde_json::json!("sonnet")),
4005 ("route_attempts", serde_json::json!(1)),
4006 ],
4007 )
4008 .await
4009 .unwrap();
4010 store
4011 .update_status(id, crate::store::TaskStatus::InProgress)
4012 .await
4013 .unwrap();
4014 set_task_updated_at_past(&store, id).await;
4015
4016 store
4018 .start_run(&crate::store::StartRun {
4019 task_id: id,
4020 attempt: 1,
4021 run_type: "agent",
4022 agent: "claude",
4023 model: "sonnet",
4024 command: "claude --model sonnet",
4025 prompt: "fix it",
4026 })
4027 .await
4028 .unwrap();
4029
4030 tick_recover_stuck_tasks(
4031 &backend,
4032 &tmux,
4033 "owner/repo",
4034 &task_manager,
4035 &config,
4036 &store,
4037 &std::collections::HashMap::new(),
4038 &Arc::new(DashMap::new()),
4039 )
4040 .await
4041 .unwrap();
4042
4043 let task = store.get(id).await.unwrap();
4044 assert_eq!(task.status, crate::store::TaskStatus::New);
4045
4046 let runs = store.get_runs(id).await.unwrap();
4047 assert_eq!(runs.len(), 1);
4048 assert_eq!(runs[0].outcome, "aborted");
4049 assert!(runs[0].completed_at.is_some());
4050 assert_eq!(
4051 runs[0].error,
4052 "stuck-task recovery: internal no session found"
4053 );
4054 }
4055
4056 #[tokio::test]
4073 async fn recover_stuck_tasks_skips_reclaim_while_dispatch_guard_held() {
4074 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4075 let mock = MockBackend::new();
4076 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4077 let task_manager = Arc::new(TaskManager::with_store(
4078 std::sync::Arc::clone(&backend),
4079 std::sync::Arc::clone(&store),
4080 "owner/repo".to_string(),
4081 ));
4082 let tmux = Arc::new(TmuxManager::new());
4083 let config = EngineConfig {
4084 no_session_stuck_timeout: 600,
4085 stuck_timeout: 1800,
4086 ..EngineConfig::default()
4087 };
4088
4089 let id = store
4090 .create_internal(
4091 "owner/repo",
4092 "Long-running daily job",
4093 "",
4094 "cron",
4095 "1",
4096 None,
4097 )
4098 .await
4099 .unwrap();
4100 store
4101 .set_fields(
4102 id,
4103 &[
4104 ("agent", serde_json::json!("claude")),
4105 ("model", serde_json::json!("sonnet")),
4106 ("route_attempts", serde_json::json!(1)),
4107 ],
4108 )
4109 .await
4110 .unwrap();
4111 store
4112 .update_status(id, crate::store::TaskStatus::InProgress)
4113 .await
4114 .unwrap();
4115 set_task_updated_at_past(&store, id).await;
4118
4119 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4122 dispatching.insert(
4123 format!("owner/repo/internal:{id}"),
4124 format!("internal:{id}"),
4125 );
4126
4127 tick_recover_stuck_tasks(
4128 &backend,
4129 &tmux,
4130 "owner/repo",
4131 &task_manager,
4132 &config,
4133 &store,
4134 &std::collections::HashMap::new(),
4135 &dispatching,
4136 )
4137 .await
4138 .unwrap();
4139
4140 let task = store.get(id).await.unwrap();
4141 assert_eq!(
4142 task.status,
4143 crate::store::TaskStatus::InProgress,
4144 "task should not be reclaimed while its dispatch guard is still held"
4145 );
4146
4147 let runs = store.get_runs(id).await.unwrap();
4149 assert!(runs.is_empty());
4150
4151 dispatching.clear();
4154 tick_recover_stuck_tasks(
4155 &backend,
4156 &tmux,
4157 "owner/repo",
4158 &task_manager,
4159 &config,
4160 &store,
4161 &std::collections::HashMap::new(),
4162 &dispatching,
4163 )
4164 .await
4165 .unwrap();
4166 let task = store.get(id).await.unwrap();
4167 assert_eq!(
4168 task.status,
4169 crate::store::TaskStatus::New,
4170 "task should still be reclaimed once the dispatch guard is released"
4171 );
4172 }
4173
4174 #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
4190 async fn recover_stuck_tasks_never_reclaims_while_guard_held_under_real_concurrency() {
4191 const HAMMER_ITERATIONS: u32 = 300;
4192
4193 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4194 let mock = MockBackend::new();
4195 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4196 let task_manager = Arc::new(TaskManager::with_store(
4197 std::sync::Arc::clone(&backend),
4198 std::sync::Arc::clone(&store),
4199 "owner/repo".to_string(),
4200 ));
4201 let tmux = Arc::new(TmuxManager::new());
4202 let config = EngineConfig {
4203 no_session_stuck_timeout: 600,
4204 stuck_timeout: 1800,
4205 ..EngineConfig::default()
4206 };
4207
4208 let id = store
4209 .create_internal(
4210 "owner/repo",
4211 "Long-running daily job",
4212 "",
4213 "cron",
4214 "1",
4215 None,
4216 )
4217 .await
4218 .unwrap();
4219 store
4220 .set_fields(
4221 id,
4222 &[
4223 ("agent", serde_json::json!("claude")),
4224 ("model", serde_json::json!("sonnet")),
4225 ("route_attempts", serde_json::json!(1)),
4226 ],
4227 )
4228 .await
4229 .unwrap();
4230 store
4231 .update_status(id, crate::store::TaskStatus::InProgress)
4232 .await
4233 .unwrap();
4234 set_task_updated_at_past(&store, id).await;
4235
4236 let dispatch_key = format!("owner/repo/internal:{id}");
4237 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4238
4239 let claim_task_id = format!("internal:{id}");
4246 {
4247 use dashmap::mapref::entry::Entry;
4248 match dispatching.entry(dispatch_key.clone()) {
4249 Entry::Vacant(slot) => {
4250 slot.insert(claim_task_id);
4251 }
4252 Entry::Occupied(_) => panic!("dispatch key already claimed"),
4253 }
4254 }
4255 let guard = DispatchGuard::new(Arc::clone(&dispatching), dispatch_key.clone());
4256
4257 let (release_tx, release_rx) = tokio::sync::oneshot::channel::<()>();
4268 let holder = tokio::spawn(async move {
4269 let _ = release_rx.await;
4270 drop(guard);
4271 });
4272
4273 for i in 0..HAMMER_ITERATIONS {
4281 tick_recover_stuck_tasks(
4282 &backend,
4283 &tmux,
4284 "owner/repo",
4285 &task_manager,
4286 &config,
4287 &store,
4288 &std::collections::HashMap::new(),
4289 &dispatching,
4290 )
4291 .await
4292 .unwrap();
4293
4294 let task = store.get(id).await.unwrap();
4295 assert_eq!(
4296 task.status,
4297 crate::store::TaskStatus::InProgress,
4298 "task must never be reclaimed while the dispatch guard is deterministically \
4299 still held, even under real cross-thread concurrency (dispatch_key={dispatch_key}, \
4300 iteration={i}, contains_key_now={}, map_len_now={})",
4301 dispatching.contains_key(&dispatch_key),
4302 dispatching.len(),
4303 );
4304 tokio::task::yield_now().await;
4305 }
4306
4307 let _ = release_tx.send(());
4309 holder.await.unwrap();
4310
4311 tick_recover_stuck_tasks(
4313 &backend,
4314 &tmux,
4315 "owner/repo",
4316 &task_manager,
4317 &config,
4318 &store,
4319 &std::collections::HashMap::new(),
4320 &dispatching,
4321 )
4322 .await
4323 .unwrap();
4324 let task = store.get(id).await.unwrap();
4325 assert_eq!(
4326 task.status,
4327 crate::store::TaskStatus::New,
4328 "task should be reclaimed once the dispatch guard is released"
4329 );
4330 }
4331
4332 #[tokio::test]
4333 async fn recover_stuck_tasks_skips_external_in_review_waiting_for_human_review() {
4334 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4335 let mock = MockBackend::new();
4336 let status_updates = std::sync::Arc::clone(&mock.status_updates);
4337 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4338 let task_manager = Arc::new(TaskManager::with_store(
4339 std::sync::Arc::clone(&backend),
4340 std::sync::Arc::clone(&store),
4341 "owner/repo".to_string(),
4342 ));
4343 let tmux = Arc::new(TmuxManager::new());
4344 let config = EngineConfig {
4345 no_session_stuck_timeout: 600,
4346 stuck_timeout: 1800,
4347 ..EngineConfig::default()
4348 };
4349
4350 let id = store
4351 .upsert_external(&crate::store::UpsertExternal {
4352 repo: "owner/repo",
4353 ext_id: "101",
4354 title: "Waiting For Human Review",
4355 body: "",
4356 author: "",
4357 url: "",
4358 labels: &[],
4359 origin: "github",
4360 })
4361 .await
4362 .unwrap();
4363 store
4364 .update_status(id, crate::store::TaskStatus::InReview)
4365 .await
4366 .unwrap();
4367 set_task_updated_at_past(&store, id).await;
4368
4369 tick_recover_stuck_tasks(
4370 &backend,
4371 &tmux,
4372 "owner/repo",
4373 &task_manager,
4374 &config,
4375 &store,
4376 &std::collections::HashMap::new(),
4377 &Arc::new(DashMap::new()),
4378 )
4379 .await
4380 .unwrap();
4381
4382 let updates = status_updates.lock().unwrap();
4383 assert!(
4384 updates.is_empty(),
4385 "InReview task waiting for human review should not be reset"
4386 );
4387 }
4388
4389 #[tokio::test]
4390 async fn unblock_parents_no_blocked_tasks() {
4391 let mock = MockBackend::new(); let status_updates = std::sync::Arc::clone(&mock.status_updates);
4393 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4394 let task_manager = make_task_manager(std::sync::Arc::clone(&backend));
4395 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4396
4397 tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
4398 .await
4399 .unwrap();
4400
4401 assert!(status_updates.lock().unwrap().is_empty());
4402 }
4403
4404 #[tokio::test]
4405 async fn unblock_parents_handles_multiple_blocked_tasks() {
4406 let mut mock = MockBackend::new();
4407
4408 let t40 = make_task("40", &["status:blocked"]);
4410 mock.blocked_tasks.push(t40);
4411 mock.sub_issues
4412 .insert("40".to_string(), vec![ExternalId("41".to_string())]);
4413 mock.tasks_by_id
4414 .insert("41".to_string(), make_task("41", &["status:done"]));
4415
4416 let t50 = make_task("50", &["status:blocked"]);
4418 mock.blocked_tasks.push(t50);
4419 mock.sub_issues
4420 .insert("50".to_string(), vec![ExternalId("51".to_string())]);
4421 mock.tasks_by_id
4422 .insert("51".to_string(), make_task("51", &["status:in_progress"]));
4423
4424 let status_updates = std::sync::Arc::clone(&mock.status_updates);
4425 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4426 let task_manager = make_task_manager(std::sync::Arc::clone(&backend));
4427 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4428
4429 tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
4430 .await
4431 .unwrap();
4432
4433 let updates = status_updates.lock().unwrap();
4434 assert_eq!(updates.len(), 1, "only one parent should be unblocked");
4435 assert_eq!(updates[0].0, "40", "task 40 should be unblocked");
4436 assert_eq!(updates[0].1, Status::New);
4437 }
4438
4439 #[tokio::test]
4442 async fn tick_all_is_a_no_op_without_projects() {
4443 let transport = Arc::new(Transport::new());
4444 let capture = Arc::new(CaptureService::new(std::sync::Arc::clone(&transport)));
4445 let tmux = Arc::new(TmuxManager::new());
4446 let semaphore = Arc::new(Semaphore::new(1));
4447 let router = Arc::new(tokio::sync::RwLock::new(cooled_review_router_for_test()));
4448 let (weight_tx, mut weight_rx) = mpsc::channel(8);
4449 let dispatching = Arc::new(DashMap::new());
4450 let auto_merge_in_flight = Arc::new(DashSet::new());
4451
4452 tick_all(
4453 &[],
4454 &tmux,
4455 &capture,
4456 &semaphore,
4457 &EngineConfig::default(),
4458 &router,
4459 &weight_tx,
4460 &dispatching,
4461 Some(&transport),
4462 &std::collections::HashSet::new(),
4463 &auto_merge_in_flight,
4464 )
4465 .await;
4466
4467 assert!(dispatching.is_empty(), "no project, so no dispatch claimed");
4468 assert!(
4469 weight_rx.try_recv().is_err(),
4470 "no project, so no weight signal"
4471 );
4472 }
4473
4474 #[serial_test::serial(cooldown_state)]
4479 #[tokio::test]
4480 async fn tick_all_runs_the_global_needs_review_sweep_once_for_two_projects() {
4481 use crate::engine::router::{Router, RouterConfig};
4482 use crate::store::UpsertExternal;
4483
4484 crate::engine::cooldown::reset_global_state().await;
4485
4486 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4487 let id = store
4488 .upsert_external(&UpsertExternal {
4489 repo: "owner/one",
4490 ext_id: "7",
4491 title: "Stale needs_review task",
4492 body: "",
4493 author: "user",
4494 url: "https://github.com/owner/one/issues/7",
4495 labels: &[],
4496 origin: "github",
4497 })
4498 .await
4499 .unwrap();
4500 store
4501 .update_status(id, crate::store::TaskStatus::NeedsReview)
4502 .await
4503 .unwrap();
4504 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4505 .bind(id)
4506 .execute(store.pool())
4507 .await
4508 .unwrap();
4509
4510 let engines: Vec<ProjectEngine> = ["owner/one", "owner/two"]
4511 .iter()
4512 .map(|repo| {
4513 let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
4514 ProjectEngine {
4515 repo: repo.to_string(),
4516 project_dir: std::path::PathBuf::from("/nonexistent"),
4517 backend: std::sync::Arc::clone(&backend),
4518 task_manager: Arc::new(TaskManager::with_store(
4519 backend,
4520 std::sync::Arc::clone(&store),
4521 repo.to_string(),
4522 )),
4523 runner: Arc::new(crate::engine::runner::TaskRunner::new(repo.to_string())),
4524 store: std::sync::Arc::clone(&store),
4525 }
4526 })
4527 .collect();
4528
4529 let mut config = RouterConfig::default();
4531 config
4532 .model_map
4533 .entry("review".to_string())
4534 .or_default()
4535 .insert("claude".to_string(), vec!["sonnet".to_string()]);
4536 let router = Arc::new(tokio::sync::RwLock::new(Router::new_for_test(
4537 config,
4538 vec!["claude".to_string()],
4539 )));
4540
4541 let transport = Arc::new(Transport::new());
4542 let capture = Arc::new(CaptureService::new(std::sync::Arc::clone(&transport)));
4543 let (weight_tx, _weight_rx) = mpsc::channel(8);
4544
4545 tick_all(
4546 &engines,
4547 &Arc::new(TmuxManager::new()),
4548 &capture,
4549 &Arc::new(Semaphore::new(1)),
4550 &EngineConfig::default(),
4551 &router,
4552 &weight_tx,
4553 &Arc::new(DashMap::new()),
4554 Some(&transport),
4555 &engines.iter().map(|e| e.repo.clone()).collect(),
4556 &Arc::new(DashSet::new()),
4557 )
4558 .await;
4559
4560 assert_eq!(
4561 store.get(id).await.unwrap().needs_review_refires,
4562 1,
4563 "two projects must share one pass of the global NeedsReview sweep"
4564 );
4565 }
4566
4567 #[tokio::test]
4570 async fn tick_all_does_not_kill_sessions_it_did_not_create() {
4571 let tmux = Arc::new(TmuxManager::new());
4572 let Ok(session_name) = tmux
4573 .create_session("other/foreign", "3889", "/tmp", "sleep 30", &[])
4574 .await
4575 else {
4576 eprintln!("Skipping test: tmux not available or failed to create test session");
4577 return;
4578 };
4579 let _guard = SessionGuard(session_name.clone());
4580
4581 tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
4584
4585 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4586 let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
4587 let engines = vec![ProjectEngine {
4588 repo: "owner/one".to_string(),
4589 project_dir: std::path::PathBuf::from("/nonexistent"),
4590 backend: std::sync::Arc::clone(&backend),
4591 task_manager: Arc::new(TaskManager::with_store(
4592 backend,
4593 std::sync::Arc::clone(&store),
4594 "owner/one".to_string(),
4595 )),
4596 runner: Arc::new(crate::engine::runner::TaskRunner::new(
4597 "owner/one".to_string(),
4598 )),
4599 store,
4600 }];
4601
4602 let transport = Arc::new(Transport::new());
4603 let capture = Arc::new(CaptureService::new(std::sync::Arc::clone(&transport)));
4604 let (weight_tx, _weight_rx) = mpsc::channel(8);
4605 let router = Arc::new(tokio::sync::RwLock::new(cooled_review_router_for_test()));
4606
4607 tick_all(
4608 &engines,
4609 &tmux,
4610 &capture,
4611 &Arc::new(Semaphore::new(1)),
4612 &EngineConfig::default(),
4613 &router,
4614 &weight_tx,
4615 &Arc::new(DashMap::new()),
4616 Some(&transport),
4617 &engines.iter().map(|e| e.repo.clone()).collect(),
4618 &Arc::new(DashSet::new()),
4619 )
4620 .await;
4621
4622 assert!(
4623 tmux.session_exists(&session_name).await,
4624 "tick_all must not kill a tmux session it did not start"
4625 );
4626 }
4627
4628 #[tokio::test]
4629 async fn detect_silent_agents_skips_backend_for_internal_tasks() {
4630 let transport = Arc::new(Transport::new());
4631 let capture = Arc::new(CaptureService::new(transport));
4632 let mock = Arc::new(MockBackend::new());
4633 let backend: Arc<dyn ExternalBackend> = Arc::<MockBackend>::clone(&mock);
4634 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4635 let task_manager = Arc::new(TaskManager::with_store(
4636 std::sync::Arc::clone(&backend),
4637 std::sync::Arc::clone(&store),
4638 "owner/repo".to_string(),
4639 ));
4640 let tmux = Arc::new(TmuxManager::new());
4641 let config = EngineConfig {
4642 silence_grace_period: 0,
4643 ..EngineConfig::default()
4644 };
4645
4646 let internal_id = store
4647 .create_internal("owner/repo", "Silent internal", "", "cron", "1", None)
4648 .await
4649 .unwrap();
4650 store
4651 .update_status(internal_id, crate::store::TaskStatus::InProgress)
4652 .await
4653 .unwrap();
4654
4655 let task_id = format!("internal:{internal_id}");
4656 capture
4657 .register_session("owner/repo", &task_id, "orch-test-internal")
4658 .await;
4659 tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
4660
4661 tick_detect_silent_agents(
4662 &tmux,
4663 "owner/repo",
4664 &capture,
4665 &backend,
4666 &task_manager,
4667 &config,
4668 &store,
4669 )
4670 .await
4671 .unwrap();
4672
4673 assert!(
4674 mock.removed_labels.lock().unwrap().is_empty(),
4675 "internal tasks should skip backend label removal"
4676 );
4677 assert!(
4678 mock.posted_comments.lock().unwrap().is_empty(),
4679 "internal tasks should skip backend comments"
4680 );
4681
4682 let task = store.get(internal_id).await.unwrap();
4683 assert!(
4687 matches!(
4688 task.status,
4689 crate::store::TaskStatus::Routed | crate::store::TaskStatus::New
4690 ),
4691 "internal task should be routed to fallback or reset to new, got {:?}",
4692 task.status
4693 );
4694 }
4695
4696 #[tokio::test]
4697 async fn detect_silent_agents_finalizes_open_task_runs_rows() {
4698 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4699 let mock = MockBackend::new();
4700 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4701 let task_manager = Arc::new(TaskManager::with_store(
4702 std::sync::Arc::clone(&backend),
4703 std::sync::Arc::clone(&store),
4704 "owner/repo".to_string(),
4705 ));
4706 let transport = Arc::new(Transport::new());
4707 let capture = Arc::new(CaptureService::new(transport));
4708 let tmux = Arc::new(TmuxManager::new());
4709 let config = EngineConfig {
4710 silence_grace_period: 60,
4711 silence_cooldown: 300,
4712 ..EngineConfig::default()
4713 };
4714
4715 let internal_id = store
4716 .create_internal("owner/repo", "Silent with open run", "", "cron", "1", None)
4717 .await
4718 .unwrap();
4719 store
4720 .set_fields(
4721 internal_id,
4722 &[
4723 ("agent", serde_json::json!("claude")),
4724 ("model", serde_json::json!("sonnet")),
4725 ],
4726 )
4727 .await
4728 .unwrap();
4729 store
4730 .update_status(internal_id, crate::store::TaskStatus::InProgress)
4731 .await
4732 .unwrap();
4733
4734 let task_id = format!("internal:{internal_id}");
4735 capture
4736 .register_session("owner/repo", &task_id, "orch-test-silent-run")
4737 .await;
4738 capture
4739 .set_buffer_state_for_test(
4740 "owner/repo",
4741 &task_id,
4742 false,
4743 false,
4744 chrono::Utc::now() - chrono::Duration::seconds(120),
4745 )
4746 .await;
4747
4748 store
4750 .start_run(&crate::store::StartRun {
4751 task_id: internal_id,
4752 attempt: 1,
4753 run_type: "agent",
4754 agent: "claude",
4755 model: "sonnet",
4756 command: "claude --model sonnet",
4757 prompt: "fix it",
4758 })
4759 .await
4760 .unwrap();
4761
4762 tick_detect_silent_agents(
4763 &tmux,
4764 "owner/repo",
4765 &capture,
4766 &backend,
4767 &task_manager,
4768 &config,
4769 &store,
4770 )
4771 .await
4772 .unwrap();
4773
4774 let runs = store.get_runs(internal_id).await.unwrap();
4775 assert_eq!(runs.len(), 1);
4776 assert_eq!(runs[0].outcome, "aborted");
4777 assert!(runs[0].completed_at.is_some());
4778 assert_eq!(runs[0].error, "silence detection: session killed");
4779 }
4780
4781 #[tokio::test]
4792 async fn phase1_session_completion_touch_prevents_phase2_reclaim() {
4793 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4794 let mock = MockBackend::new();
4795 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4796 let task_manager = Arc::new(TaskManager::with_store(
4797 std::sync::Arc::clone(&backend),
4798 std::sync::Arc::clone(&store),
4799 "owner/repo".to_string(),
4800 ));
4801 let tmux = Arc::new(TmuxManager::new());
4802 let config = EngineConfig {
4803 no_session_stuck_timeout: 600,
4804 stuck_timeout: 1800,
4805 ..EngineConfig::default()
4806 };
4807
4808 let id = store
4812 .create_internal("owner/repo", "Completed task", "", "cron", "1", None)
4813 .await
4814 .unwrap();
4815 store
4816 .update_status(id, crate::store::TaskStatus::InProgress)
4817 .await
4818 .unwrap();
4819 set_task_updated_at_past(&store, id).await;
4820
4821 let task_id = format!("internal:{id}");
4824 store_touch_updated_at(&Some(Arc::clone(&store)), "owner/repo", &task_id).await;
4825
4826 tick_recover_stuck_tasks(
4828 &backend,
4829 &tmux,
4830 "owner/repo",
4831 &task_manager,
4832 &config,
4833 &store,
4834 &std::collections::HashMap::new(),
4835 &Arc::new(DashMap::new()),
4836 )
4837 .await
4838 .unwrap();
4839
4840 let task = store.get(id).await.unwrap();
4841 assert_eq!(
4842 task.status,
4843 crate::store::TaskStatus::InProgress,
4844 "Phase 2 must not reclaim a task whose updated_at was just touched by Phase 1"
4845 );
4846 }
4847
4848 #[tokio::test]
4849 async fn stuck_task_timing_treats_dead_session_as_missing() {
4850 let tmux = Arc::new(TmuxManager::new());
4851 let repo = "owner/repo";
4852 let task_id = "internal:33498";
4853 let session_name = tmux.session_name(repo, task_id);
4854
4855 let create_result = tmux
4856 .command()
4857 .args(["new-session", "-d", "-s", &session_name, "-c", "/tmp"])
4858 .output()
4859 .await;
4860
4861 match create_result {
4862 Ok(o) if o.status.success() => {}
4863 _ => {
4864 eprintln!("Skipping test: tmux not available or failed to create test session");
4865 return;
4866 }
4867 }
4868
4869 let _guard = SessionGuard(session_name.clone());
4870
4871 let set_option_result = tmux
4872 .command()
4873 .args(["set-option", "-t", &session_name, "remain-on-exit", "on"])
4874 .output()
4875 .await;
4876 if !matches!(set_option_result, Ok(ref o) if o.status.success()) {
4877 eprintln!("Skipping test: unable to set tmux remain-on-exit option");
4878 return;
4879 }
4880
4881 let send_exit_result = tmux
4882 .command()
4883 .args(["send-keys", "-t", &session_name, "exit", "Enter"])
4884 .output()
4885 .await;
4886 if !matches!(send_exit_result, Ok(ref o) if o.status.success()) {
4887 eprintln!("Skipping test: unable to exit tmux pane");
4888 return;
4889 }
4890
4891 for _ in 0..20 {
4893 tokio::time::sleep(std::time::Duration::from_millis(100)).await;
4894 if !tmux.session_is_running(&session_name).await {
4895 break;
4896 }
4897 }
4898
4899 let mut session_map = tmux.batch_session_active().await;
4904 for _ in 0..20 {
4905 if !session_map.get(&session_name).copied().unwrap_or(false) {
4906 break;
4907 }
4908 tokio::time::sleep(std::time::Duration::from_millis(100)).await;
4909 session_map = tmux.batch_session_active().await;
4910 }
4911 if session_map.get(&session_name).copied().unwrap_or(false) {
4912 eprintln!(
4913 "Skipping test: session still alive in batch_session_active after retries (CI tmux lag)"
4914 );
4915 return;
4916 }
4917
4918 let config = EngineConfig {
4919 no_session_stuck_timeout: 600,
4920 stuck_timeout: 1800,
4921 ..EngineConfig::default()
4922 };
4923 let updated_at = (chrono::Utc::now()
4924 - chrono::Duration::seconds(
4925 i64::try_from(config.no_session_stuck_timeout).expect("timeout fits in i64") + 5,
4926 ))
4927 .to_rfc3339();
4928
4929 let timing = stuck_task_timing_from_map(
4930 &tmux,
4931 repo,
4932 task_id,
4933 &updated_at,
4934 &config,
4935 "parse failure",
4936 &session_map,
4937 )
4938 .expect("dead session should be treated as missing");
4939
4940 assert!(!timing.has_session);
4941 assert_eq!(timing.threshold, config.no_session_stuck_timeout);
4942 }
4943
4944 #[tokio::test]
4945 #[serial_test::serial(suspend_state)]
4946 async fn stuck_task_timing_discounts_host_suspend_gap() {
4947 super::super::suspend::clear_for_test();
4948
4949 let tmux = Arc::new(TmuxManager::new());
4950 let repo = "owner/repo";
4951 let task_id = "internal:99001";
4952 let session_map = std::collections::HashMap::new();
4953
4954 let config = EngineConfig {
4955 no_session_stuck_timeout: 1800,
4956 stuck_timeout: 1800,
4957 ..EngineConfig::default()
4958 };
4959
4960 let updated_at = (chrono::Utc::now() - chrono::Duration::minutes(33)).to_rfc3339();
4964 let suspend_gap_detected_at = chrono::Utc::now() - chrono::Duration::minutes(1);
4965 super::super::suspend::inject_gap_for_test(
4966 suspend_gap_detected_at,
4967 chrono::Duration::minutes(30),
4968 );
4969
4970 let timing = stuck_task_timing_from_map(
4971 &tmux,
4972 repo,
4973 task_id,
4974 &updated_at,
4975 &config,
4976 "parse failure",
4977 &session_map,
4978 );
4979
4980 assert!(
4981 timing.is_none(),
4982 "task age should be discounted below the stuck threshold once suspend time is subtracted, got {timing:?}"
4983 );
4984
4985 super::super::suspend::clear_for_test();
4986 }
4987
4988 #[tokio::test]
4989 async fn recover_stuck_tasks_reclaims_routed_task_blocked_by_stale_session() {
4990 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4991 let mock = MockBackend::new();
4992 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4993 let task_manager = Arc::new(TaskManager::with_store(
4994 std::sync::Arc::clone(&backend),
4995 std::sync::Arc::clone(&store),
4996 "owner/repo".to_string(),
4997 ));
4998 let tmux = Arc::new(TmuxManager::new());
4999 let config = EngineConfig {
5000 no_session_stuck_timeout: 600,
5001 stuck_timeout: 1,
5002 ..EngineConfig::default()
5003 };
5004
5005 let id = store
5006 .create_internal("owner/repo", "Blocked routed task", "", "cron", "1", None)
5007 .await
5008 .unwrap();
5009 store
5010 .update_status(id, crate::store::TaskStatus::Routed)
5011 .await
5012 .unwrap();
5013 set_task_updated_at_past(&store, id).await;
5014
5015 let task_id = format!("internal:{id}");
5016 let session_name = tmux.session_name("owner/repo", &task_id);
5017 let create_result = tmux
5018 .command()
5019 .args([
5020 "new-session",
5021 "-d",
5022 "-s",
5023 &session_name,
5024 "-c",
5025 "/tmp",
5026 "sleep 60",
5027 ])
5028 .output()
5029 .await;
5030
5031 match create_result {
5032 Ok(o) if o.status.success() => {}
5033 _ => {
5034 eprintln!("Skipping test: tmux not available or failed to create test session");
5035 return;
5036 }
5037 }
5038
5039 let session_map = tmux.batch_session_active().await;
5040 assert_eq!(session_map.get(&session_name).copied(), Some(true));
5041
5042 tick_recover_stuck_tasks(
5043 &backend,
5044 &tmux,
5045 "owner/repo",
5046 &task_manager,
5047 &config,
5048 &store,
5049 &session_map,
5050 &Arc::new(DashMap::new()),
5051 )
5052 .await
5053 .unwrap();
5054
5055 let task = store.get(id).await.unwrap();
5056 assert_eq!(
5057 task.status,
5058 crate::store::TaskStatus::New,
5059 "routed task blocked by a stale session should be reset for re-routing"
5060 );
5061 assert!(
5062 !tmux.session_exists(&session_name).await,
5063 "recovery should kill the stale tmux session so dispatch can proceed"
5064 );
5065 }
5066
5067 #[tokio::test]
5081 async fn dispatch_does_not_deadlock_under_write_lock() {
5082 use crate::engine::router::Router;
5083 use tokio::sync::RwLock;
5084
5085 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5086 let mock = MockBackend::new();
5087 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
5088 let task_manager = Arc::new(TaskManager::with_store(
5089 std::sync::Arc::clone(&backend),
5090 std::sync::Arc::clone(&store),
5091 "owner/repo".to_string(),
5092 ));
5093 let tmux = Arc::new(TmuxManager::new());
5094 let semaphore = Arc::new(Semaphore::new(4));
5095 let (weight_tx, _weight_rx) = mpsc::channel(16);
5096 let dispatching = Arc::new(DashMap::new());
5097 let transport = Arc::new(Transport::new());
5098 let capture = Arc::new(CaptureService::new(transport));
5099
5100 let router_config = crate::engine::router::RouterConfig::default();
5101 let router = Router::new_for_test(router_config, vec!["claude".to_string()]);
5102 let router_arc = Arc::new(RwLock::new(router));
5103
5104 let write_guard = router_arc.write().await;
5108 let dispatch_mode = dispatch_mode_from_router(&write_guard);
5109
5110 let result = tokio::time::timeout(
5111 std::time::Duration::from_secs(2),
5112 tick_dispatch_tasks(
5113 &backend,
5114 &tmux,
5115 "owner/repo",
5116 &Arc::new(crate::engine::runner::TaskRunner::new(
5117 "owner/repo".to_string(),
5118 )),
5119 &capture,
5120 &semaphore,
5121 &task_manager,
5122 &weight_tx,
5123 dispatch_mode,
5124 &dispatching,
5125 &store,
5126 &std::collections::HashMap::new(),
5127 ),
5128 )
5129 .await;
5130
5131 assert!(
5132 result.is_ok(),
5133 "tick_dispatch_tasks deadlocked! It tried to acquire a read lock \
5134 on router_arc while a write lock was already held (issue #1361)"
5135 );
5136 }
5137
5138 #[tokio::test]
5146 async fn dispatch_skips_task_whose_routed_model_entered_cooldown() {
5147 use crate::store::{StoreRoute, TaskStatus as DbStatus, UpsertExternal};
5148
5149 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5150 let mock = MockBackend::new();
5151 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
5152 let task_manager = Arc::new(TaskManager::with_store(
5153 std::sync::Arc::clone(&backend),
5154 std::sync::Arc::clone(&store),
5155 "owner/repo".to_string(),
5156 ));
5157 let tmux = Arc::new(TmuxManager::new());
5158 let runner = Arc::new(TaskRunner::new("owner/repo".to_string()));
5159 let semaphore = Arc::new(Semaphore::new(4));
5160 let (weight_tx, _weight_rx) = mpsc::channel(16);
5161 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
5162 let transport = Arc::new(Transport::new());
5163 let capture = Arc::new(CaptureService::new(transport));
5164
5165 let id = store
5166 .upsert_external(&UpsertExternal {
5167 repo: "owner/repo",
5168 ext_id: "999",
5169 title: "Stale routed task",
5170 body: "",
5171 author: "user",
5172 url: "https://github.com/owner/repo/issues/999",
5173 labels: &[],
5174 origin: "github",
5175 })
5176 .await
5177 .unwrap();
5178 store
5179 .store_route(&StoreRoute {
5180 id,
5181 agent: "minimax",
5182 model: Some("opus"),
5183 complexity: "medium",
5184 estimate: 0,
5185 reason: "test",
5186 profile: "{}",
5187 skills: "[]",
5188 })
5189 .await
5190 .unwrap();
5191 store.update_status(id, DbStatus::Routed).await.unwrap();
5192
5193 crate::engine::cooldown::set_model_cooldown(
5196 "minimax",
5197 "opus",
5198 300,
5199 "rate_limit_usage_window",
5200 )
5201 .await;
5202
5203 let dispatch_mode = DispatchMode {
5204 is_degraded: false,
5205 healthy_agents: 1,
5206 threshold: 1,
5207 };
5208
5209 tick_dispatch_tasks(
5210 &backend,
5211 &tmux,
5212 "owner/repo",
5213 &runner,
5214 &capture,
5215 &semaphore,
5216 &task_manager,
5217 &weight_tx,
5218 dispatch_mode,
5219 &dispatching,
5220 &store,
5221 &std::collections::HashMap::new(),
5222 )
5223 .await
5224 .unwrap();
5225
5226 let task = store.get(id).await.unwrap();
5227 assert_eq!(
5228 task.status,
5229 DbStatus::Routed,
5230 "task whose routed model is now cooled down must stay Routed, not be dispatched"
5231 );
5232 assert!(
5233 dispatching.is_empty(),
5234 "dispatch guard must be released when skipping a stale-cooldown task"
5235 );
5236 }
5237
5238 #[tokio::test]
5239 async fn dispatch_skips_a_task_with_an_open_dependency() {
5240 use crate::store::{
5241 DepSource, NewDependency, StoreRoute, TaskStatus as DbStatus, UpsertExternal,
5242 };
5243
5244 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5245 let mock = MockBackend::new();
5246 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
5247 let task_manager = Arc::new(TaskManager::with_store(
5248 std::sync::Arc::clone(&backend),
5249 std::sync::Arc::clone(&store),
5250 "owner/repo".to_string(),
5251 ));
5252 let tmux = Arc::new(TmuxManager::new());
5253 let runner = Arc::new(TaskRunner::new("owner/repo".to_string()));
5254 let semaphore = Arc::new(Semaphore::new(4));
5255 let (weight_tx, _weight_rx) = mpsc::channel(16);
5256 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
5257 let transport = Arc::new(Transport::new());
5258 let capture = Arc::new(CaptureService::new(transport));
5259
5260 let id = store
5261 .upsert_external(&UpsertExternal {
5262 repo: "owner/repo",
5263 ext_id: "42",
5264 title: "Waiting routed task",
5265 body: "",
5266 author: "user",
5267 url: "https://github.com/owner/repo/issues/42",
5268 labels: &[],
5269 origin: "github",
5270 })
5271 .await
5272 .unwrap();
5273 store
5274 .replace_dependencies(
5275 id,
5276 &[NewDependency {
5277 repo: "owner/repo".into(),
5278 number: 7,
5279 source: DepSource::Github,
5280 }],
5281 )
5282 .await
5283 .unwrap();
5284 store
5285 .store_route(&StoreRoute {
5286 id,
5287 agent: "claude",
5288 model: Some("haiku"),
5289 complexity: "simple",
5290 estimate: 0,
5291 reason: "test",
5292 profile: "{}",
5293 skills: "[]",
5294 })
5295 .await
5296 .unwrap();
5297 store.update_status(id, DbStatus::Routed).await.unwrap();
5298
5299 let dispatch_mode = DispatchMode {
5300 is_degraded: false,
5301 healthy_agents: 1,
5302 threshold: 1,
5303 };
5304 tick_dispatch_tasks(
5305 &backend,
5306 &tmux,
5307 "owner/repo",
5308 &runner,
5309 &capture,
5310 &semaphore,
5311 &task_manager,
5312 &weight_tx,
5313 dispatch_mode,
5314 &dispatching,
5315 &store,
5316 &std::collections::HashMap::new(),
5317 )
5318 .await
5319 .unwrap();
5320
5321 let task = store.get(id).await.unwrap();
5322 assert_eq!(task.status, DbStatus::Waiting);
5323 assert!(dispatching.is_empty());
5324 }
5325
5326 #[tokio::test]
5339 async fn completed_session_does_not_cause_stuck_reclaim_race() {
5340 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5341 let mock = MockBackend::new();
5342 let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
5343 let task_manager = Arc::new(TaskManager::with_store(
5344 std::sync::Arc::clone(&backend),
5345 std::sync::Arc::clone(&store),
5346 "owner/repo".to_string(),
5347 ));
5348 let tmux = Arc::new(TmuxManager::new());
5349 let transport = Arc::new(Transport::new());
5350 let capture = Arc::new(CaptureService::new(transport));
5351
5352 let id = store
5354 .upsert_external(&crate::store::UpsertExternal {
5355 repo: "owner/repo",
5356 ext_id: "200",
5357 title: "Race Task",
5358 body: "",
5359 author: "",
5360 url: "",
5361 labels: &[],
5362 origin: "github",
5363 })
5364 .await
5365 .unwrap();
5366 store
5367 .update_status(id, crate::store::TaskStatus::InProgress)
5368 .await
5369 .unwrap();
5370
5371 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
5373 .bind(id)
5374 .execute(store.pool())
5375 .await
5376 .unwrap();
5377
5378 let task_id = "200";
5380 let session_name = tmux.session_name("owner/repo", task_id);
5381 let create_result = tmux
5382 .command()
5383 .args(["new-session", "-d", "-s", &session_name, "-c", "/tmp"])
5384 .output()
5385 .await;
5386 match create_result {
5387 Ok(o) if o.status.success() => {}
5388 _ => {
5389 eprintln!("Skipping test: tmux not available or failed to create test session");
5390 return;
5391 }
5392 }
5393 let _guard = SessionGuard(session_name.clone());
5394
5395 let _ = tmux
5397 .command()
5398 .args(["set-option", "-t", &session_name, "remain-on-exit", "on"])
5399 .output()
5400 .await;
5401 let _ = tmux
5402 .command()
5403 .args(["send-keys", "-t", &session_name, "exit", "Enter"])
5404 .output()
5405 .await;
5406
5407 for _ in 0..20 {
5409 tokio::time::sleep(std::time::Duration::from_millis(100)).await;
5410 if !tmux.session_is_running(&session_name).await {
5411 break;
5412 }
5413 }
5414
5415 let task_key = task_id.to_string();
5417 capture
5418 .register_session("owner/repo", &task_key, &session_name)
5419 .await;
5420
5421 let config = EngineConfig {
5423 no_session_stuck_timeout: 600,
5424 stuck_timeout: 1800,
5425 ..EngineConfig::default()
5426 };
5427
5428 tick_check_session_completions(&tmux, "owner/repo", &capture, &store)
5429 .await
5430 .unwrap();
5431
5432 {
5438 let task_state = store.get(id).await.unwrap();
5439 if task_state.updated_at.starts_with("2020") {
5440 eprintln!(
5441 "Skipping test: tick_check_session_completions did not update stored \
5442 updated_at — session was not in snapshot (CI tmux environment issue)"
5443 );
5444 return;
5445 }
5446 }
5447
5448 tick_recover_stuck_tasks(
5450 &backend,
5451 &tmux,
5452 "owner/repo",
5453 &task_manager,
5454 &config,
5455 &store,
5456 &std::collections::HashMap::new(),
5457 &Arc::new(DashMap::new()),
5458 )
5459 .await
5460 .unwrap();
5461
5462 let task = store.get(id).await.unwrap();
5464 assert_eq!(
5465 task.status,
5466 crate::store::TaskStatus::InProgress,
5467 "task should not be reclaimed to New"
5468 );
5469 }
5470
5471 #[tokio::test]
5481 async fn session_completion_matches_repo_configured_with_git_suffix() {
5482 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5483 let tmux = Arc::new(TmuxManager::new());
5484 let transport = Arc::new(Transport::new());
5485 let capture = Arc::new(CaptureService::new(transport));
5486
5487 let repo = "owner/repo.git";
5488 let id = store
5489 .upsert_external(&crate::store::UpsertExternal {
5490 repo,
5491 ext_id: "201",
5492 title: "Git-suffix repo task",
5493 body: "",
5494 author: "",
5495 url: "",
5496 labels: &[],
5497 origin: "github",
5498 })
5499 .await
5500 .unwrap();
5501 store
5502 .update_status(id, crate::store::TaskStatus::InProgress)
5503 .await
5504 .unwrap();
5505 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
5506 .bind(id)
5507 .execute(store.pool())
5508 .await
5509 .unwrap();
5510
5511 let task_id = "201";
5514 let session_name = tmux.session_name(repo, task_id);
5515 let create_result = tmux
5516 .command()
5517 .args(["new-session", "-d", "-s", &session_name, "-c", "/tmp"])
5518 .output()
5519 .await;
5520 match create_result {
5521 Ok(o) if o.status.success() => {}
5522 _ => {
5523 eprintln!("Skipping test: tmux not available or failed to create test session");
5524 return;
5525 }
5526 }
5527 let _guard = SessionGuard(session_name.clone());
5528
5529 let _ = tmux
5530 .command()
5531 .args(["set-option", "-t", &session_name, "remain-on-exit", "on"])
5532 .output()
5533 .await;
5534 let _ = tmux
5535 .command()
5536 .args(["send-keys", "-t", &session_name, "exit", "Enter"])
5537 .output()
5538 .await;
5539
5540 for _ in 0..20 {
5541 tokio::time::sleep(std::time::Duration::from_millis(100)).await;
5542 if !tmux.session_is_running(&session_name).await {
5543 break;
5544 }
5545 }
5546
5547 capture.register_session(repo, task_id, &session_name).await;
5548
5549 tick_check_session_completions(&tmux, repo, &capture, &store)
5550 .await
5551 .unwrap();
5552
5553 let task_state = store.get(id).await.unwrap();
5554 if task_state.updated_at.starts_with("2020") {
5555 eprintln!(
5556 "Skipping test: tick_check_session_completions did not update stored \
5557 updated_at — session was not in snapshot (CI tmux environment issue)"
5558 );
5559 return;
5560 }
5561 assert!(
5562 !task_state.updated_at.starts_with("2020"),
5563 "tick_check_session_completions must match its own repo's live sessions \
5564 even when the repo is configured with a \".git\" suffix"
5565 );
5566 }
5567
5568 #[tokio::test]
5574 async fn job_scheduler_returns_ok_on_duplicate_job_config() {
5575 let tmp = tempfile::tempdir().unwrap();
5576 let cfg = tmp.path().join(".orch.yml");
5577 std::fs::write(
5578 &cfg,
5579 "jobs:\n - id: trading-scan\n schedule: \"0 9 * * *\"\n task:\n title: Scan\n body: do it\n",
5580 )
5581 .unwrap();
5582 let jobs_dir = tmp.path().join("prompts").join("jobs");
5583 std::fs::create_dir_all(&jobs_dir).unwrap();
5584 std::fs::write(
5585 jobs_dir.join("trading-scan.md"),
5586 "---\nid: trading-scan\nschedule: '0 9 * * *'\ntitle: Trading Scan\n---\n\nDo trading scan.\n",
5587 )
5588 .unwrap();
5589
5590 let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
5591 let result = tick_job_scheduler(&cfg, &backend, None, "test/repo", None).await;
5592 assert!(
5593 result.is_ok(),
5594 "duplicate job id must not propagate as Err: {result:?}"
5595 );
5596 }
5597
5598 #[tokio::test]
5602 async fn job_scheduler_processes_valid_project_when_other_has_duplicate() {
5603 let bad_tmp = tempfile::tempdir().unwrap();
5605 let bad_cfg = bad_tmp.path().join(".orch.yml");
5606 std::fs::write(
5607 &bad_cfg,
5608 "jobs:\n - id: dup\n schedule: \"0 9 * * *\"\n task:\n title: Dup\n body: x\n",
5609 )
5610 .unwrap();
5611 let bad_jobs_dir = bad_tmp.path().join("prompts").join("jobs");
5612 std::fs::create_dir_all(&bad_jobs_dir).unwrap();
5613 std::fs::write(
5614 bad_jobs_dir.join("dup.md"),
5615 "---\nid: dup\nschedule: '0 9 * * *'\ntitle: Dup\n---\n\nbody\n",
5616 )
5617 .unwrap();
5618
5619 let good_tmp = tempfile::tempdir().unwrap();
5621 let good_cfg = good_tmp.path().join(".orch.yml");
5622 std::fs::write(
5623 &good_cfg,
5624 "jobs:\n - id: valid-job\n schedule: \"0 9 * * *\"\n task:\n title: Valid\n body: ok\n",
5625 )
5626 .unwrap();
5627
5628 let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
5629
5630 let bad_result = tick_job_scheduler(&bad_cfg, &backend, None, "test/bad-repo", None).await;
5631 assert!(
5632 bad_result.is_ok(),
5633 "bad project must not abort scheduler: {bad_result:?}"
5634 );
5635
5636 let good_result =
5637 tick_job_scheduler(&good_cfg, &backend, None, "test/good-repo", None).await;
5638 assert!(
5639 good_result.is_ok(),
5640 "valid project must still run after bad project: {good_result:?}"
5641 );
5642 }
5643
5644 #[serial_test::serial(cooldown_state)]
5652 #[tokio::test]
5653 async fn global_sweep_routes_inactive_repo_tasks() {
5654 use crate::engine::router::{Router, RouterConfig};
5655 use crate::store::{TaskStatus, UpsertExternal};
5656
5657 crate::engine::cooldown::reset_global_state().await;
5658
5659 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5660
5661 let active_id = store
5663 .upsert_external(&UpsertExternal {
5664 repo: "active/repo",
5665 ext_id: "1",
5666 title: "Active task",
5667 body: "",
5668 author: "user",
5669 url: "https://github.com/active/repo/issues/1",
5670 labels: &[],
5671 origin: "github",
5672 })
5673 .await
5674 .unwrap();
5675
5676 let inactive_id = store
5678 .upsert_external(&UpsertExternal {
5679 repo: "inactive/repo",
5680 ext_id: "42",
5681 title: "Orphaned task",
5682 body: "",
5683 author: "user",
5684 url: "https://github.com/inactive/repo/issues/42",
5685 labels: &[],
5686 origin: "github",
5687 })
5688 .await
5689 .unwrap();
5690
5691 assert_eq!(store.get(active_id).await.unwrap().status, TaskStatus::New);
5692 assert_eq!(
5693 store.get(inactive_id).await.unwrap().status,
5694 TaskStatus::New
5695 );
5696
5697 let mut model_map: std::collections::HashMap<
5698 String,
5699 std::collections::HashMap<String, Vec<String>>,
5700 > = std::collections::HashMap::new();
5701 let mut tier = std::collections::HashMap::new();
5702 tier.insert("claude".to_string(), vec!["haiku".to_string()]);
5703 model_map.insert("medium".to_string(), tier);
5704 let config = RouterConfig {
5705 mode: "round_robin".to_string(),
5706 agents: vec!["claude".to_string()],
5707 model_map,
5708 ..Default::default()
5709 };
5710 let mut router = Router::new_for_test(config, vec!["claude".to_string()]);
5711
5712 let active_repos: std::collections::HashSet<String> =
5713 ["active/repo".to_string()].into_iter().collect();
5714 let mut routing_quota = RoutingTickQuota::new(1);
5715 route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
5716 .await
5717 .unwrap();
5718
5719 assert_eq!(
5721 store.get(active_id).await.unwrap().status,
5722 TaskStatus::New,
5723 "global sweep must not touch tasks for the active repo"
5724 );
5725
5726 let after = store.get(inactive_id).await.unwrap();
5728 assert_eq!(
5729 after.status,
5730 TaskStatus::Routed,
5731 "global sweep must route tasks from inactive repos"
5732 );
5733 assert!(
5734 matches!(&after.agent, Some(a) if !a.is_empty()),
5735 "agent must be set after routing"
5736 );
5737 }
5738
5739 #[serial_test::serial(cooldown_state)]
5747 #[tokio::test]
5748 async fn global_sweep_does_not_touch_other_active_repo_tasks() {
5749 use crate::engine::router::{Router, RouterConfig};
5750 use crate::store::{TaskStatus, UpsertExternal};
5751
5752 crate::engine::cooldown::reset_global_state().await;
5753
5754 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5755
5756 let other_active_id = store
5759 .upsert_external(&UpsertExternal {
5760 repo: "other-active/repo",
5761 ext_id: "7",
5762 title: "Other active project's task",
5763 body: "",
5764 author: "user",
5765 url: "https://github.com/other-active/repo/issues/7",
5766 labels: &[],
5767 origin: "github",
5768 })
5769 .await
5770 .unwrap();
5771
5772 let inactive_id = store
5774 .upsert_external(&UpsertExternal {
5775 repo: "inactive/repo",
5776 ext_id: "42",
5777 title: "Orphaned task",
5778 body: "",
5779 author: "user",
5780 url: "https://github.com/inactive/repo/issues/42",
5781 labels: &[],
5782 origin: "github",
5783 })
5784 .await
5785 .unwrap();
5786
5787 let mut model_map: std::collections::HashMap<
5788 String,
5789 std::collections::HashMap<String, Vec<String>>,
5790 > = std::collections::HashMap::new();
5791 let mut tier = std::collections::HashMap::new();
5792 tier.insert("claude".to_string(), vec!["haiku".to_string()]);
5793 model_map.insert("medium".to_string(), tier);
5794 let config = RouterConfig {
5795 mode: "round_robin".to_string(),
5796 agents: vec!["claude".to_string()],
5797 model_map,
5798 ..Default::default()
5799 };
5800 let mut router = Router::new_for_test(config, vec!["claude".to_string()]);
5801
5802 let active_repos: std::collections::HashSet<String> =
5806 ["active/repo".to_string(), "other-active/repo".to_string()]
5807 .into_iter()
5808 .collect();
5809 let mut routing_quota = RoutingTickQuota::new(1);
5810 route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
5811 .await
5812 .unwrap();
5813
5814 assert_eq!(
5815 store.get(other_active_id).await.unwrap().status,
5816 TaskStatus::New,
5817 "global sweep run from a different active repo's tick must not route \
5818 another active repo's new tasks — that repo's own tick will route them"
5819 );
5820 assert_eq!(
5821 store.get(inactive_id).await.unwrap().status,
5822 TaskStatus::Routed,
5823 "global sweep must still route tasks from genuinely orphaned repos"
5824 );
5825 }
5826
5827 #[serial_test::serial(cooldown_state)]
5829 #[tokio::test]
5830 async fn global_sweep_skips_no_agent_tasks() {
5831 use crate::engine::router::{Router, RouterConfig};
5832 use crate::store::{TaskStatus, UpsertExternal};
5833
5834 crate::engine::cooldown::reset_global_state().await;
5835
5836 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5837
5838 let id = store
5839 .upsert_external(&UpsertExternal {
5840 repo: "inactive/repo",
5841 ext_id: "99",
5842 title: "No-agent task",
5843 body: "",
5844 author: "user",
5845 url: "https://github.com/inactive/repo/issues/99",
5846 labels: &["no-agent".to_string()],
5847 origin: "github",
5848 })
5849 .await
5850 .unwrap();
5851
5852 let mut model_map: std::collections::HashMap<
5853 String,
5854 std::collections::HashMap<String, Vec<String>>,
5855 > = std::collections::HashMap::new();
5856 let mut tier = std::collections::HashMap::new();
5857 tier.insert("claude".to_string(), vec!["haiku".to_string()]);
5858 model_map.insert("medium".to_string(), tier);
5859 let config = RouterConfig {
5860 mode: "round_robin".to_string(),
5861 agents: vec!["claude".to_string()],
5862 model_map,
5863 ..Default::default()
5864 };
5865 let mut router = Router::new_for_test(config, vec!["claude".to_string()]);
5866
5867 let active_repos: std::collections::HashSet<String> =
5868 ["active/repo".to_string()].into_iter().collect();
5869 let mut routing_quota = RoutingTickQuota::new(1);
5870 route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
5871 .await
5872 .unwrap();
5873
5874 assert_eq!(
5875 store.get(id).await.unwrap().status,
5876 TaskStatus::New,
5877 "no-agent tasks must not be routed"
5878 );
5879 }
5880
5881 #[serial_test::serial(cooldown_state)]
5886 #[tokio::test]
5887 async fn global_sweep_honors_shared_routing_quota() {
5888 use crate::engine::router::{Router, RouterConfig};
5889 use crate::store::{TaskStatus, UpsertExternal};
5890
5891 crate::engine::cooldown::reset_global_state().await;
5892
5893 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5894 for ext_id in ["1", "2"] {
5895 store
5896 .upsert_external(&UpsertExternal {
5897 repo: "inactive/repo",
5898 ext_id,
5899 title: "Orphaned task",
5900 body: "",
5901 author: "user",
5902 url: &format!("https://github.com/inactive/repo/issues/{ext_id}"),
5903 labels: &[],
5904 origin: "github",
5905 })
5906 .await
5907 .unwrap();
5908 }
5909
5910 let mut model_map: std::collections::HashMap<
5911 String,
5912 std::collections::HashMap<String, Vec<String>>,
5913 > = std::collections::HashMap::new();
5914 let mut tier = std::collections::HashMap::new();
5915 tier.insert("claude".to_string(), vec!["haiku".to_string()]);
5916 model_map.insert("medium".to_string(), tier);
5917 let config = RouterConfig {
5918 mode: "round_robin".to_string(),
5919 agents: vec!["claude".to_string()],
5920 model_map,
5921 ..Default::default()
5922 };
5923 let mut router = Router::new_for_test(config, vec!["claude".to_string()]);
5924
5925 let active_repos: std::collections::HashSet<String> =
5926 ["active/repo".to_string()].into_iter().collect();
5927 let mut routing_quota = RoutingTickQuota::new(1);
5928 route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
5929 .await
5930 .unwrap();
5931
5932 let all_tasks = store.list_all_global().await.unwrap();
5933 let routed = all_tasks
5934 .iter()
5935 .filter(|task| task.status == TaskStatus::Routed)
5936 .count();
5937 let still_new = all_tasks
5938 .iter()
5939 .filter(|task| task.status == TaskStatus::New)
5940 .count();
5941
5942 assert_eq!(routed, 1, "only one task may consume the shared quota");
5943 assert_eq!(still_new, 1, "remaining tasks wait for the next tick");
5944
5945 route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
5946 .await
5947 .unwrap();
5948 let after_exhausted = store.list_all_global().await.unwrap();
5949 assert_eq!(
5950 after_exhausted
5951 .iter()
5952 .filter(|task| task.status == TaskStatus::Routed)
5953 .count(),
5954 1,
5955 "an exhausted shared quota must not reset within the same tick"
5956 );
5957 }
5958
5959 #[tokio::test]
5976 async fn group_orphaned_routed_tasks_selects_inactive_repo_tasks() {
5977 use crate::store::{StoreRoute, TaskStatus, UpsertExternal};
5978
5979 async fn make_routed(store: &TaskStore, repo: &str, ext_id: &str) -> i64 {
5980 let id = store
5981 .upsert_external(&UpsertExternal {
5982 repo,
5983 ext_id,
5984 title: "Routed task",
5985 body: "",
5986 author: "user",
5987 url: &format!("https://github.com/{repo}/issues/{ext_id}"),
5988 labels: &[],
5989 origin: "github",
5990 })
5991 .await
5992 .unwrap();
5993 store
5994 .store_route(&StoreRoute {
5995 id,
5996 agent: "claude",
5997 model: Some("sonnet"),
5998 complexity: "medium",
5999 estimate: 0,
6000 reason: "test",
6001 profile: "{}",
6002 skills: "[]",
6003 })
6004 .await
6005 .unwrap();
6006 store.update_status(id, TaskStatus::Routed).await.unwrap();
6007 id
6008 }
6009
6010 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6011
6012 let active_id = make_routed(&store, "active/repo", "1").await;
6013 let inactive_id_a = make_routed(&store, "inactive/repo", "2").await;
6014 let other_inactive_id = make_routed(&store, "other-inactive/repo", "3").await;
6015 let inactive_id_b = make_routed(&store, "inactive/repo", "4").await;
6016
6017 let all_routed = store
6018 .list_all_by_status_global(TaskStatus::Routed)
6019 .await
6020 .unwrap();
6021 let active_repos: std::collections::HashSet<String> =
6022 ["active/repo".to_string()].into_iter().collect();
6023 let by_repo = group_orphaned_routed_tasks(all_routed, &active_repos);
6024
6025 assert!(
6026 !by_repo.contains_key("active/repo"),
6027 "global sweep must not select tasks for the active repo"
6028 );
6029 let inactive_ids: std::collections::HashSet<i64> = by_repo
6030 .get("inactive/repo")
6031 .expect("inactive/repo must be selected")
6032 .iter()
6033 .map(|t| t.id)
6034 .collect();
6035 assert_eq!(
6036 inactive_ids,
6037 [inactive_id_a, inactive_id_b].into_iter().collect(),
6038 "global sweep must group all orphaned tasks for a repo together"
6039 );
6040 assert_eq!(
6041 by_repo
6042 .get("other-inactive/repo")
6043 .map(|tasks| tasks.iter().map(|t| t.id).collect::<Vec<_>>()),
6044 Some(vec![other_inactive_id])
6045 );
6046
6047 assert_eq!(
6050 store.get(active_id).await.unwrap().status,
6051 TaskStatus::Routed
6052 );
6053 }
6054
6055 #[tokio::test]
6060 async fn group_orphaned_routed_tasks_excludes_all_active_repos_not_just_current() {
6061 use crate::store::{StoreRoute, TaskStatus, UpsertExternal};
6062
6063 async fn make_routed(store: &TaskStore, repo: &str, ext_id: &str) -> i64 {
6064 let id = store
6065 .upsert_external(&UpsertExternal {
6066 repo,
6067 ext_id,
6068 title: "Routed task",
6069 body: "",
6070 author: "user",
6071 url: &format!("https://github.com/{repo}/issues/{ext_id}"),
6072 labels: &[],
6073 origin: "github",
6074 })
6075 .await
6076 .unwrap();
6077 store
6078 .store_route(&StoreRoute {
6079 id,
6080 agent: "claude",
6081 model: Some("sonnet"),
6082 complexity: "medium",
6083 estimate: 0,
6084 reason: "test",
6085 profile: "{}",
6086 skills: "[]",
6087 })
6088 .await
6089 .unwrap();
6090 store.update_status(id, TaskStatus::Routed).await.unwrap();
6091 id
6092 }
6093
6094 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6095
6096 let repo_a_id = make_routed(&store, "active/repo-a", "1").await;
6097 let repo_b_id = make_routed(&store, "active/repo-b", "2").await;
6098 let orphaned_id = make_routed(&store, "inactive/repo", "3").await;
6099
6100 let all_routed = store
6101 .list_all_by_status_global(TaskStatus::Routed)
6102 .await
6103 .unwrap();
6104 let active_repos: std::collections::HashSet<String> =
6107 ["active/repo-a".to_string(), "active/repo-b".to_string()]
6108 .into_iter()
6109 .collect();
6110 let by_repo = group_orphaned_routed_tasks(all_routed, &active_repos);
6111
6112 assert!(
6113 !by_repo.contains_key("active/repo-a"),
6114 "the current repo's own tasks must not be swept"
6115 );
6116 assert!(
6117 !by_repo.contains_key("active/repo-b"),
6118 "another active repo's tasks must not be swept during repo A's tick"
6119 );
6120 assert_eq!(
6121 by_repo
6122 .get("inactive/repo")
6123 .map(|tasks| tasks.iter().map(|t| t.id).collect::<Vec<_>>()),
6124 Some(vec![orphaned_id]),
6125 "a genuinely inactive repo's tasks must still be swept"
6126 );
6127
6128 assert_eq!(
6130 store.get(repo_a_id).await.unwrap().status,
6131 TaskStatus::Routed
6132 );
6133 assert_eq!(
6134 store.get(repo_b_id).await.unwrap().status,
6135 TaskStatus::Routed
6136 );
6137 }
6138
6139 #[tokio::test]
6141 async fn global_dispatch_sweep_skips_no_agent_tasks() {
6142 use crate::store::{StoreRoute, TaskStatus, UpsertExternal};
6143
6144 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6145
6146 let id = store
6147 .upsert_external(&UpsertExternal {
6148 repo: "inactive/repo",
6149 ext_id: "99",
6150 title: "No-agent routed task",
6151 body: "",
6152 author: "user",
6153 url: "https://github.com/inactive/repo/issues/99",
6154 labels: &["no-agent".to_string()],
6155 origin: "github",
6156 })
6157 .await
6158 .unwrap();
6159 store
6160 .store_route(&StoreRoute {
6161 id,
6162 agent: "claude",
6163 model: Some("sonnet"),
6164 complexity: "medium",
6165 estimate: 0,
6166 reason: "test",
6167 profile: "{}",
6168 skills: "[]",
6169 })
6170 .await
6171 .unwrap();
6172 store.update_status(id, TaskStatus::Routed).await.unwrap();
6173
6174 let tmux = Arc::new(TmuxManager::new());
6179 let transport = Arc::new(Transport::new());
6180 let capture = Arc::new(CaptureService::new(transport));
6181 let semaphore = Arc::new(Semaphore::new(4));
6182 let (weight_tx, _weight_rx) = mpsc::channel(16);
6183 let dispatching = Arc::new(DashMap::new());
6184 let dispatch_mode = DispatchMode {
6185 is_degraded: false,
6186 healthy_agents: 1,
6187 threshold: 1,
6188 };
6189
6190 let active_repos: std::collections::HashSet<String> =
6191 ["active/repo".to_string()].into_iter().collect();
6192 dispatch_routed_tasks_global(
6193 &tmux,
6194 &capture,
6195 &semaphore,
6196 &weight_tx,
6197 dispatch_mode,
6198 &dispatching,
6199 &store,
6200 &std::collections::HashMap::new(),
6201 &active_repos,
6202 )
6203 .await
6204 .unwrap();
6205
6206 assert_eq!(
6207 store.get(id).await.unwrap().status,
6208 TaskStatus::Routed,
6209 "no-agent tasks must not be dispatched"
6210 );
6211 }
6212}