1use super::EngineConfig;
13use super::cleanup::{check_merged_prs, cleanup_done_worktrees, prune_old_run_output};
14use super::commands::{CommandOutcome, CommandStoreOps, parse_command, validate_and_run_command};
15use super::review_poll::review_open_prs;
16use crate::backends::ExternalId;
17use crate::backends::ExternalTask;
18use crate::backends::{ExternalBackend, Mention, Status};
19use crate::cmd::CommandErrorContext;
20use crate::config;
21use crate::engine;
22use crate::engine::auto_merge::{is_pr_behind, required_checks_state};
23use crate::engine::cleanup::cleanup_task_worktree;
24use crate::engine::commands::CommandGh;
25use crate::engine::commands::OwnerCommand;
26use crate::engine::cooldown;
27use crate::engine::cooldown::{
28 cooldown_reason, github_circuit_remaining_secs, is_agent_in_cooldown, is_github_circuit_open,
29 is_model_in_cooldown, refresh_degraded_agents,
30};
31use crate::engine::dependencies;
32use crate::engine::router::Router;
33use crate::engine::router::RouterConfig;
34use crate::engine::router::config as router_config;
35use crate::engine::runner::agents::patterns;
36use crate::engine::runner::git_ops;
37use crate::engine::tasks;
38use crate::engine::tasks::{TaskManager, status_to_task_status};
39use crate::github::http::GhHttp;
40use crate::github::types::extract_issue_number_from_url;
41use crate::home;
42use crate::store;
43use crate::store::Task;
44use crate::store::TaskRun;
45use crate::store::TaskStatus;
46use crate::store::TaskStore;
47use crate::tmux::TmuxManager;
48use dashmap::{DashMap, DashSet};
49use futures::future;
50use std::collections::HashMap;
51use std::collections::HashSet;
52use std::path::PathBuf;
53use std::sync::Arc;
54use std::time::Duration;
55use std::time::Instant;
56use tokio::fs;
57use tokio::sync::OnceCell;
58use tokio::sync::RwLock;
59use tokio::time;
60
61async fn kv_get_prefer_store(store: Option<&Arc<TaskStore>>, key: &str) -> Option<String> {
67 match try_kv_get_prefer_store(store, key).await {
68 Ok(opt) => opt,
69 Err(e) => {
70 tracing::error!(key, err = %e, "kv_get failed");
71 None
72 }
73 }
74}
75
76async fn try_kv_get_prefer_store(
82 store: Option<&Arc<TaskStore>>,
83 key: &str,
84) -> anyhow::Result<Option<String>> {
85 if let Some(s) = store {
86 Ok(s.kv_get(key).await?)
87 } else {
88 Ok(None)
89 }
90}
91
92async fn kv_set_prefer_store(store: Option<&Arc<TaskStore>>, key: &str, value: &str) {
98 if let Err(e) = try_kv_set_prefer_store(store, key, value).await {
99 tracing::error!(key, err = %e, "kv_set failed");
100 }
101}
102
103async fn try_kv_set_prefer_store(
109 store: Option<&Arc<TaskStore>>,
110 key: &str,
111 value: &str,
112) -> anyhow::Result<()> {
113 if let Some(s) = store {
114 s.kv_set(key, value).await?;
115 }
116 Ok(())
117}
118
119#[derive(Debug, Clone)]
120pub(crate) struct ReviewTaskSnapshot {
121 pub external: ExternalTask,
122 pub stored: Task,
123}
124
125async fn prefetch_review_tasks(
126 store: &Arc<TaskStore>,
127 repo: &str,
128) -> anyhow::Result<(
129 Vec<ReviewTaskSnapshot>,
130 Vec<ReviewTaskSnapshot>,
131 Vec<ReviewTaskSnapshot>,
132)> {
133 let mut in_review = store.list_by_status(repo, TaskStatus::InReview).await?;
135 in_review.extend(store.list_by_status(repo, TaskStatus::Approved).await?);
136 let in_review = in_review
137 .into_iter()
138 .map(|stored| ReviewTaskSnapshot {
139 external: tasks::store_task_to_external(&stored),
140 stored,
141 })
142 .collect();
143 let needs_review = store
144 .list_by_status(repo, TaskStatus::NeedsReview)
145 .await?
146 .into_iter()
147 .map(|stored| ReviewTaskSnapshot {
148 external: tasks::store_task_to_external(&stored),
149 stored,
150 })
151 .collect();
152
153 let blocked_candidates = store
157 .list_by_status(repo, TaskStatus::Blocked)
158 .await?
159 .into_iter()
160 .filter(|stored| stored.pr_number.is_some() || !stored.branch.is_empty())
161 .map(|stored| ReviewTaskSnapshot {
162 external: tasks::store_task_to_external(&stored),
163 stored,
164 })
165 .collect();
166
167 Ok((in_review, needs_review, blocked_candidates))
168}
169
170async fn fetch_comments_since(
171 backend: &Arc<dyn ExternalBackend>,
172 since: &str,
173) -> anyhow::Result<Vec<Mention>> {
174 backend.get_mentions(since).await
175}
176
177fn filter_mentions_by_since(comments: &[Mention], since: &str) -> Vec<Mention> {
178 comments
179 .iter()
180 .filter(|comment| comment.created_at.as_str() > since)
181 .cloned()
182 .collect()
183}
184
185#[derive(Debug, Clone, Copy, PartialEq, Eq)]
186enum FailureCategory {
187 FalseFailure,
188 RateLimit,
189 CreditExhausted,
190 SilentExit0,
191 Timeout,
192 ConnectionError,
193 CliFlagError,
194 AllAgentsExhausted,
195 ParseError,
196 MaxAttempts,
197 PushFailed,
198 PrCreateFailed,
199 ModelUnavailable,
200 Unknown,
201}
202
203impl FailureCategory {
204 fn is_recoverable(self) -> bool {
211 matches!(
212 self,
213 FailureCategory::FalseFailure
214 | FailureCategory::RateLimit
215 | FailureCategory::CreditExhausted
216 | FailureCategory::SilentExit0
217 | FailureCategory::Timeout
218 | FailureCategory::ConnectionError
219 | FailureCategory::CliFlagError
220 | FailureCategory::AllAgentsExhausted
221 | FailureCategory::ParseError
222 | FailureCategory::ModelUnavailable
223 )
224 }
225
226 fn as_str(self) -> &'static str {
227 match self {
228 FailureCategory::FalseFailure => "FalseFailure",
229 FailureCategory::RateLimit => "RateLimit",
230 FailureCategory::CreditExhausted => "CreditExhausted",
231 FailureCategory::SilentExit0 => "SilentExit0",
232 FailureCategory::Timeout => "Timeout",
233 FailureCategory::ConnectionError => "ConnectionError",
234 FailureCategory::CliFlagError => "CliFlagError",
235 FailureCategory::AllAgentsExhausted => "AllAgentsExhausted",
236 FailureCategory::ParseError => "ParseError",
237 FailureCategory::MaxAttempts => "MaxAttempts",
238 FailureCategory::PushFailed => "PushFailed",
239 FailureCategory::PrCreateFailed => "PrCreateFailed",
240 FailureCategory::ModelUnavailable => "ModelUnavailable",
241 FailureCategory::Unknown => "Unknown",
242 }
243 }
244}
245
246fn classify_failure(error: &str, outcome: &str) -> FailureCategory {
247 let lower = error.to_lowercase();
248
249 if outcome == "timeout" || lower.contains("timeout") {
250 return FailureCategory::Timeout;
251 }
252
253 if outcome == "billing_cycle_exhausted"
264 || outcome == "credit_exhausted"
265 || outcome == "org_level_disabled"
266 || cooldown::detect_credit_exhaustion(&lower).is_some()
267 {
268 return FailureCategory::CreditExhausted;
269 }
270
271 if outcome == "rate_limit" || lower.contains("rate limit") || lower.contains("usage limit") {
272 return FailureCategory::RateLimit;
273 }
274
275 if error.trim().is_empty() {
276 return FailureCategory::FalseFailure;
277 }
278
279 if lower.contains("all agents exhausted") {
280 return FailureCategory::AllAgentsExhausted;
281 }
282
283 if lower.contains("max attempts") || lower.contains("exceeded max attempts") {
284 return FailureCategory::MaxAttempts;
285 }
286
287 if lower.contains("output-format=stream-json") && lower.contains("requires --verbose") {
288 return FailureCategory::CliFlagError;
289 }
290
291 if lower.contains("model unavailable")
292 || lower.contains("model not found")
293 || (lower.contains("does not exist") && lower.contains("model"))
294 {
295 return FailureCategory::ModelUnavailable;
296 }
297
298 if lower.contains("push failed") {
299 return FailureCategory::PushFailed;
300 }
301
302 if lower.contains("create pr failed")
303 || lower.contains("failed to create pull request")
304 || lower.contains("pull request creation failed")
305 {
306 return FailureCategory::PrCreateFailed;
307 }
308
309 if lower.contains("exit 0") {
310 return FailureCategory::SilentExit0;
311 }
312
313 if outcome == "parse_error"
314 || outcome == "truncated"
315 || lower.contains("invalid response")
316 || lower.contains("parse error")
317 || lower.contains("truncated")
318 {
319 return FailureCategory::ParseError;
320 }
321
322 if patterns::detect_network_error(&lower).is_some() {
323 return FailureCategory::ConnectionError;
324 }
325
326 FailureCategory::Unknown
327}
328
329fn classify_failure_from_run(run: &TaskRun) -> Option<FailureCategory> {
330 if run.outcome == "success" && run.error.trim().is_empty() {
332 return None;
333 }
334
335 if run.outcome.trim().is_empty()
342 && run.error.trim().is_empty()
343 && run.exit_code.is_none()
344 && run.stdout.trim().is_empty()
345 && run.stderr.trim().is_empty()
346 {
347 return None;
348 }
349
350 let has_output = !run.stdout.trim().is_empty() || !run.stderr.trim().is_empty();
351 if run.exit_code == Some(0) && !has_output && run.outcome != "success" {
352 return Some(FailureCategory::SilentExit0);
353 }
354
355 let mut category = classify_failure(&run.error, &run.outcome);
356
357 if category == FailureCategory::FalseFailure {
366 let has_nonzero_exit = run.exit_code.is_some() && run.exit_code != Some(0);
367 let has_output = !run.stdout.trim().is_empty() || !run.stderr.trim().is_empty();
368 if has_nonzero_exit || has_output {
369 category = FailureCategory::Unknown;
370 }
371 }
372
373 Some(category)
374}
375
376async fn resolve_mention_parent_id(
386 store: &TaskStore,
387 repo: &str,
388 issue_num: &str,
389 is_pr: bool,
390) -> anyhow::Result<Option<i64>> {
391 if is_pr {
392 if let Ok(pr_num) = issue_num.parse::<i32>() {
393 store.resolve_id_by_pr_number(repo, pr_num).await
394 } else {
395 Ok(None)
396 }
397 } else {
398 store.resolve_id_by_external(repo, issue_num).await
399 }
400}
401
402async fn record_mention_task(
404 store: &TaskStore,
405 repo: &str,
406 mention: &Mention,
407 title: &str,
408 body: &str,
409 parent_id: Option<i64>,
410 cursor: &mut MentionCursor,
411) -> bool {
412 match store
413 .create_internal(repo, title, body, "mention", &mention.id, parent_id)
414 .await
415 {
416 Ok(task_id) => {
417 tracing::info!(task_id, mention_id = %mention.id, "created mention task");
418 cursor.advance(&mention.created_at);
419 true
420 }
421 Err(e) => {
422 tracing::warn!(mention_id = %mention.id, err = %e, "failed to create mention task");
423 false
424 }
425 }
426}
427
428fn advance_cursor(last_safe_ts: &mut Option<String>, ts: &str) {
430 if last_safe_ts.as_deref() < Some(ts) {
431 *last_safe_ts = Some(ts.to_string());
432 }
433}
434
435#[derive(Debug, Default)]
436struct MentionCursor {
437 last_safe_ts: Option<String>,
438 blocked_by_gap: bool,
439}
440
441impl MentionCursor {
442 fn advance(&mut self, ts: &str) {
443 if self.blocked_by_gap {
444 return;
445 }
446 advance_cursor(&mut self.last_safe_ts, ts);
447 }
448
449 fn block_on_gap(&mut self) {
450 self.blocked_by_gap = true;
451 }
452
453 fn into_last_safe_ts(self) -> Option<String> {
454 self.last_safe_ts
455 }
456}
457
458#[derive(Debug, PartialEq)]
460pub(crate) enum CommentAction {
461 Skip,
463 ExecuteCommand {
465 command: OwnerCommand,
466 issue_num: String,
467 },
468 ExecuteCommandForMention {
470 command: OwnerCommand,
471 issue_num: String,
472 },
473 CreateMentionTask { issue_num: Option<String> },
475}
476
477pub(crate) fn classify_comment(
482 body: &str,
483 issue_url: Option<&str>,
484 current_user: &str,
485 already_processed: bool,
486) -> CommentAction {
487 if already_processed {
488 return CommentAction::Skip;
489 }
490
491 let is_mention =
492 body.contains(current_user) || body.contains("@orch") || body.contains("@orchestrator");
493 let command = parse_command(body);
494 let issue_num = issue_url.and_then(extract_issue_number_from_url);
495
496 match (command, is_mention, issue_num) {
497 (Some(cmd), true, Some(num)) => CommentAction::ExecuteCommandForMention {
498 command: cmd,
499 issue_num: num,
500 },
501 (Some(cmd), false, Some(num)) => CommentAction::ExecuteCommand {
502 command: cmd,
503 issue_num: num,
504 },
505 (Some(_), true, None) => CommentAction::CreateMentionTask { issue_num: None },
507 (None, true, issue_num) => CommentAction::CreateMentionTask { issue_num },
509 (Some(_), false, None) | (None, false, _) => CommentAction::Skip,
511 }
512}
513
514#[expect(
527 clippy::too_many_arguments,
528 reason = "the sync tick hands over its shared state piece by piece"
529)]
530async fn handle_slash_command(
531 backend: &Arc<dyn ExternalBackend>,
532 store: Option<&Arc<TaskStore>>,
533 repo: &str,
534 task_manager: &Arc<TaskManager>,
535 gh: &dyn CommandGh,
536 mention: &Mention,
537 command: &OwnerCommand,
538 cursor: &mut MentionCursor,
539) -> anyhow::Result<bool> {
540 let Some(issue_num) = mention
541 .issue_url
542 .as_deref()
543 .and_then(extract_issue_number_from_url)
544 else {
545 tracing::warn!(comment_id = %mention.id, "slash command without valid issue number");
546 return Ok(true);
547 };
548
549 let store_opt: Option<Arc<dyn CommandStoreOps>> =
550 store.map(|s| Arc::clone(s) as Arc<dyn CommandStoreOps>);
551 let outcome = validate_and_run_command(
552 backend,
553 gh,
554 repo,
555 &issue_num,
556 command,
557 &mention.author,
558 &store_opt,
559 task_manager,
560 )
561 .await;
562
563 let is_pr = match &outcome {
565 CommandOutcome::Executed { is_pr }
566 | CommandOutcome::NotOpen { is_pr }
567 | CommandOutcome::NotCollaborator { is_pr } => *is_pr,
568 CommandOutcome::FetchFailed | CommandOutcome::CollaboratorCheckFailed => {
569 tracing::warn!(
572 comment_id = %mention.id,
573 command = %command,
574 "deferring @orch slash command after a transient GitHub error"
575 );
576 return Ok(false);
577 }
578 };
579
580 if let Some(s) = store {
583 let parent_id = match resolve_mention_parent_id(s, repo, &issue_num, is_pr).await {
584 Ok(parent_id) => parent_id,
585 Err(e) => {
586 tracing::warn!(
587 comment_id = %mention.id,
588 err = %e,
589 "deferring mention task due to parent lookup failure"
590 );
591 return Err(e);
592 }
593 };
594
595 let (title, body) = match &outcome {
596 CommandOutcome::NotOpen { .. } => (
597 format!(
598 "Skipped @orch command on closed #{issue_num} from @{}",
599 mention.author
600 ),
601 format!(
602 "Mention by @{}:\n\n{}\n\n**Skipped:** target is closed",
603 mention.author, mention.body
604 ),
605 ),
606 CommandOutcome::NotCollaborator { .. } => (
607 format!(
608 "Skipped @orch command on #{issue_num} from non-collaborator @{}",
609 mention.author
610 ),
611 format!(
612 "Mention by @{}:\n\n{}\n\n**Skipped:** author is not a collaborator",
613 mention.author, mention.body
614 ),
615 ),
616 CommandOutcome::Executed { .. } => (
617 format!(
618 "Respond to mention by @{} on task #{issue_num} — command executed",
619 mention.author
620 ),
621 format!(
622 "Mention by @{} on #{issue_num}:\n\n{}\n\n**Status:** Command executed",
623 mention.author, mention.body
624 ),
625 ),
626 CommandOutcome::FetchFailed | CommandOutcome::CollaboratorCheckFailed => {
627 tracing::debug!("skipping mention task record due to earlier command failure");
628 return Ok(true);
629 }
630 };
631
632 let created = record_mention_task(s, repo, mention, &title, &body, parent_id, cursor).await;
633 if !created {
634 return Ok(false);
635 }
636 }
637 Ok(true)
638}
639
640fn auto_unblock_cooldown_elapsed(count: i32, last_at: &str) -> bool {
641 if count == 0 {
642 return true;
643 }
644
645 let required = match count {
646 1 => chrono::Duration::minutes(30),
647 2 => chrono::Duration::hours(2),
648 _ => return false,
649 };
650
651 if last_at.trim().is_empty() {
652 return true;
653 }
654
655 match chrono::DateTime::parse_from_rfc3339(last_at) {
656 Ok(ts) => chrono::Utc::now() - ts.with_timezone(&chrono::Utc) >= required,
657 Err(_) => true,
658 }
659}
660
661fn is_ci_failure_block(task: &Task) -> bool {
662 let reason = task.block_reason.as_deref().unwrap_or("");
663 reason.contains("CI failure limit") || reason.contains("CI checks timed out")
664}
665
666fn ci_failure_unblock_cooldown_elapsed(task: &Task) -> bool {
667 if task.auto_unblock_count == 0 {
668 return true;
669 }
670
671 let required = match task.auto_unblock_count {
675 1 => chrono::Duration::minutes(10),
676 2 => chrono::Duration::hours(1),
677 3 => chrono::Duration::hours(6),
678 _ => chrono::Duration::hours(24),
679 };
680
681 let last_at = &task.auto_unblock_last_at;
682 if last_at.trim().is_empty() {
683 return true;
684 }
685
686 match chrono::DateTime::parse_from_rfc3339(last_at) {
687 Ok(ts) => chrono::Utc::now() - ts.with_timezone(&chrono::Utc) >= required,
688 Err(_) => true,
689 }
690}
691
692async fn try_unblock_ci_failure_task(
693 gh: &GhHttp,
694 repo: &str,
695 task_manager: &Arc<TaskManager>,
696 store: &Arc<TaskStore>,
697 task: &Task,
698 auto_merge_in_flight: &Arc<DashSet<String>>,
699) -> anyhow::Result<bool> {
700 let Some(pr_number) = task.pr_number.and_then(|n| u64::try_from(n).ok()) else {
701 tracing::debug!(
702 task_id = task.id,
703 "CI failure blocked task has no PR number"
704 );
705 return Ok(false);
706 };
707
708 let pr = match gh.get_pr(repo, pr_number).await {
709 Ok(p) => p,
710 Err(e) => {
711 tracing::warn!(task_id = task.id, pr_number, err = %e, "failed to get PR status for CI failure block");
712 return Ok(false);
713 }
714 };
715
716 tracing::debug!(
717 task_id = task.id,
718 pr_number,
719 merged = pr.merged,
720 state = ?pr.state,
721 "verifying PR state for CI failure blocked task"
722 );
723
724 if pr.merged.unwrap_or(false) || pr.state.eq_ignore_ascii_case("closed") {
725 tracing::info!(
726 task_id = task.id,
727 pr_number,
728 merged = pr.merged,
729 state = %pr.state,
730 "PR merged/closed — unblocking CI failure blocked task"
731 );
732 let ext_id = task
733 .external_id
734 .clone()
735 .unwrap_or_else(|| format!("internal:{}", task.id));
736 task_manager
740 .mark_task_done_by_store_id(task.id, &task.repo, &ExternalId(ext_id))
741 .await?;
742 let fields = vec![
743 ("block_reason", serde_json::Value::Null),
744 ("auto_unblock_count", serde_json::json!(0)),
745 ("auto_unblock_last_at", serde_json::json!("")),
746 ("auto_unblock_last_reason", serde_json::json!("")),
747 ];
748 store.set_fields(task.id, &fields).await?;
749 return Ok(true);
750 }
751
752 let task_id_str = task
756 .external_id
757 .clone()
758 .unwrap_or_else(|| format!("internal:{}", task.id));
759 if auto_merge_in_flight.contains(&task_id_str) {
760 tracing::debug!(
761 task_id = task.id,
762 pr_number,
763 "CI-failure recovery merge already in flight — skipping"
764 );
765 return Ok(false);
766 }
767
768 let branch_update_result = gh.update_pr_branch(repo, pr_number).await;
774 match branch_update_result {
775 Ok(()) => {
776 tracing::info!(
777 task_id = task.id,
778 pr_number,
779 "updated CI-failure-blocked PR branch against base to retrigger CI"
780 );
781 }
782 Err(ref e) => {
783 tracing::warn!(
785 task_id = task.id,
786 pr_number,
787 err = %e,
788 outcome = "branch_update_failed",
789 "failed to update CI-failure-blocked PR branch (may already be up to date)"
790 );
791 }
792 }
793
794 if let Err(e) = gh.enable_auto_merge(repo, pr_number).await {
795 tracing::warn!(task_id = task.id, pr_number, err = %e, "failed to re-enable auto-merge after branch update");
796 }
797
798 let pr_state = pr.state.as_str();
801 let new_reason = if let Err(ref e) = branch_update_result {
802 format!(
803 "CI failure limit reached during auto-merge (PR #{pr_number} still open, state: {pr_state}, branch update failed: {e})"
804 )
805 } else {
806 format!(
807 "CI failure limit reached during auto-merge (PR #{pr_number} still open, state: {pr_state})"
808 )
809 };
810 let now = chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string();
811 let new_count = task.auto_unblock_count.saturating_add(1);
814 let fields = vec![
815 ("block_reason", serde_json::json!(&new_reason)),
816 ("auto_unblock_count", serde_json::json!(new_count)),
817 ("auto_unblock_last_at", serde_json::json!(&now)),
818 ];
819 store.set_fields_silent(task.id, &fields).await?;
823
824 auto_merge_in_flight.insert(task_id_str.clone());
828 let gh_clone = gh.clone();
829 let repo_string = repo.to_string();
830 let task_manager_clone = Arc::clone(task_manager);
831 let store_clone = Arc::clone(store);
832 let task_clone = task.clone();
833 let in_flight_clone = Arc::clone(auto_merge_in_flight);
834
835 tokio::spawn(async move {
836 let _guard = scopeguard::guard((), |()| {
837 in_flight_clone.remove(&task_id_str);
838 });
839
840 match poll_and_merge_recovered_pr(&gh_clone, &repo_string, pr_number, task_clone.id).await {
841 Ok(true) => {
842 tracing::info!(
843 task_id = task_clone.id,
844 pr_number,
845 "CI-failure-blocked PR merged during recovery sweep"
846 );
847 let ext_id = task_clone
848 .external_id
849 .clone()
850 .unwrap_or_else(|| format!("internal:{}", task_clone.id));
851 let fields = [
852 ("block_reason", serde_json::Value::Null),
853 ("auto_unblock_count", serde_json::json!(0)),
854 ("auto_unblock_last_at", serde_json::json!("")),
855 ("auto_unblock_last_reason", serde_json::json!("")),
856 ];
857 if let Err(e) = task_manager_clone
858 .update_task_status_and_result_by_store_id(
859 task_clone.id,
860 &task_clone.repo,
861 &ExternalId(ext_id),
862 Status::Done,
863 &fields,
864 )
865 .await
866 {
867 tracing::warn!(
868 task_id = task_clone.id,
869 pr_number,
870 err = %e,
871 "failed to mark CI-failure-blocked task done after recovery merge"
872 );
873 }
874 if let Err(e) =
875 cleanup_task_worktree(&format!("{}", task_clone.id), &repo_string, &store_clone)
876 .await
877 {
878 tracing::warn!(
879 task_id = task_clone.id,
880 pr_number,
881 err = %e,
882 "cleanup after CI-failure recovery merge failed"
883 );
884 }
885 }
886 Ok(false) => {
887 tracing::debug!(
888 task_id = task_clone.id,
889 pr_number,
890 "CI-failure-blocked PR not merged this recovery tick — poll completed without merge"
891 );
892 }
893 Err(e) => {
894 tracing::warn!(
895 task_id = task_clone.id,
896 pr_number,
897 err = %e,
898 "error while polling CI-failure-blocked PR for recovery"
899 );
900 }
901 }
902 });
903
904 Ok(false)
905}
906
907async fn poll_and_merge_recovered_pr(
918 gh: &GhHttp,
919 repo: &str,
920 pr_number: u64,
921 task_id: i64,
922) -> anyhow::Result<bool> {
923 const MAX_BEHIND_RETRIES: u32 = 3;
927
928 let max_wait_secs: u64 = config::get("workflow.ci_poll_max_wait_secs")
929 .ok()
930 .and_then(|s| s.parse().ok())
931 .unwrap_or(600);
932 let base_interval_secs: u64 = config::get("workflow.ci_poll_interval_secs")
933 .ok()
934 .and_then(|s| s.parse().ok())
935 .unwrap_or(15);
936 let max_wait = Duration::from_secs(max_wait_secs);
937 let start = Instant::now();
938 let mut poll_count: u32 = 0;
939 let mut behind_retries: u32 = 0;
940
941 let initial_pr = gh.get_pr(repo, pr_number).await?;
942 let base_branch = initial_pr.base.ref_.clone();
943 let required_contexts = gh
944 .get_required_status_check_contexts(repo, &base_branch)
945 .await?;
946 let repo_has_workflows = if required_contexts.is_empty() {
947 gh.has_workflows(repo).await?
948 } else {
949 true
950 };
951
952 loop {
953 let pr = gh.get_pr(repo, pr_number).await?;
954 if pr.merged.unwrap_or(false) || pr.state.eq_ignore_ascii_case("closed") {
955 tracing::info!(
956 task_id = task_id,
957 pr_number,
958 "PR merged/closed during recovery poll"
959 );
960 return Ok(true);
961 }
962
963 if pr.mergeable == Some(false) {
964 tracing::info!(
965 task_id = task_id,
966 pr_number,
967 outcome = "poll_exited_without_merge",
968 reason = "merge_conflicts",
969 "PR has merge conflicts after recovery update — leaving for next recovery cycle"
970 );
971 return Ok(false);
972 }
973
974 if is_pr_behind(&pr) {
975 if behind_retries >= MAX_BEHIND_RETRIES {
976 tracing::info!(
977 task_id = task_id,
978 pr_number,
979 behind_retries,
980 max_behind_retries = MAX_BEHIND_RETRIES,
981 outcome = "poll_exited_without_merge",
982 reason = "behind_retry_exhausted",
983 "PR still behind base after recovery retries — leaving for next recovery cycle"
984 );
985 return Ok(false);
986 }
987 behind_retries += 1;
988 match gh.update_pr_branch(repo, pr_number).await {
989 Ok(()) => {
990 tracing::info!(
991 task_id = task_id,
992 pr_number,
993 behind_retries,
994 "PR was behind base — updated branch and continuing recovery poll"
995 );
996 continue;
997 }
998 Err(e) => {
999 tracing::warn!(
1000 task_id = task_id,
1001 pr_number,
1002 behind_retries,
1003 err = %e,
1004 outcome = "branch_update_failed",
1005 "failed to update behind PR branch during recovery poll"
1006 );
1007 return Ok(false);
1008 }
1009 }
1010 }
1011
1012 let head_sha = pr.head.sha.clone();
1013 let (state, total, passing, failing, pending) = if required_contexts.is_empty() {
1014 gh.get_combined_status(
1015 repo,
1016 &head_sha,
1017 repo_has_workflows,
1018 pr.mergeable_state.as_deref(),
1019 )
1020 .await?
1021 } else {
1022 let check_runs = gh.get_check_runs(repo, &head_sha).await?;
1023 let statuses = gh.get_commit_status_contexts(repo, &head_sha).await?;
1024 required_checks_state(&required_contexts, &check_runs, &statuses)
1025 };
1026
1027 tracing::info!(
1028 task_id = task_id,
1029 pr_number,
1030 state = %state,
1031 total,
1032 passing,
1033 failing,
1034 pending,
1035 "CI recovery status check"
1036 );
1037
1038 match state.as_str() {
1039 "success" => {
1040 tracing::info!(
1041 task_id = task_id,
1042 pr_number,
1043 "CI green after recovery — attempting direct merge"
1044 );
1045 match gh.merge_pr(repo, pr_number, true, &head_sha).await {
1046 Ok(()) => {
1047 tracing::info!(
1048 task_id = task_id,
1049 pr_number,
1050 "direct merge succeeded after recovery"
1051 );
1052 return Ok(true);
1053 }
1054 Err(e) => {
1055 let err = e.to_string().to_ascii_lowercase();
1056 tracing::warn!(
1057 task_id = task_id,
1058 pr_number,
1059 err = %e,
1060 "direct merge failed after recovery"
1061 );
1062 if err.contains("already merged")
1065 || err.contains("no merge commit")
1066 || err.contains("pull request already merged")
1067 {
1068 return Ok(true);
1069 }
1070 if err.contains("head branch was modified")
1072 || err.contains("sha does not match")
1073 {
1074 tracing::info!(
1075 task_id = task_id,
1076 pr_number,
1077 outcome = "poll_exited_without_merge",
1078 reason = "head_moved",
1079 "merge rejected because the head sha changed"
1080 );
1081 return Ok(false);
1082 }
1083 if (err.contains("head branch is out of date")
1087 || err.contains("behind")
1088 || err.contains("409"))
1089 && behind_retries < MAX_BEHIND_RETRIES
1090 {
1091 behind_retries += 1;
1092 match gh.update_pr_branch(repo, pr_number).await {
1093 Ok(()) => {
1094 tracing::info!(
1095 task_id = task_id,
1096 pr_number,
1097 behind_retries,
1098 "merge rejected due to out-of-date branch — updated and continuing recovery poll"
1099 );
1100 continue;
1101 }
1102 Err(update_err) => {
1103 tracing::warn!(
1104 task_id = task_id,
1105 pr_number,
1106 behind_retries,
1107 err = %update_err,
1108 outcome = "branch_update_failed",
1109 "failed to update out-of-date PR branch during recovery poll"
1110 );
1111 }
1112 }
1113 }
1114 tracing::info!(
1115 task_id = task_id,
1116 pr_number,
1117 outcome = "poll_exited_without_merge",
1118 reason = "direct_merge_failed",
1119 "direct merge did not succeed — leaving for next recovery cycle"
1120 );
1121 return Ok(false);
1122 }
1123 }
1124 }
1125 "failure" => {
1126 tracing::info!(
1127 task_id = task_id,
1128 pr_number,
1129 outcome = "poll_exited_without_merge",
1130 reason = "ci_failure",
1131 "CI still failing after recovery — will retry later"
1132 );
1133 return Ok(false);
1134 }
1135 _ => {}
1136 }
1137
1138 if start.elapsed() >= max_wait {
1139 tracing::info!(
1140 task_id = task_id,
1141 pr_number,
1142 outcome = "poll_exited_without_merge",
1143 reason = "poll_timeout",
1144 "CI still pending after recovery poll timeout — will retry later"
1145 );
1146 return Ok(false);
1147 }
1148
1149 poll_count += 1;
1150 let multiplier = (2_u64).pow(poll_count.saturating_sub(1)).min(4);
1151 let sleep_secs = base_interval_secs.saturating_mul(multiplier);
1152 time::sleep(Duration::from_secs(sleep_secs)).await;
1153 }
1154}
1155
1156const REBROADCAST_BLOCK_REASON: &str = "review agent rebroadcast escalated after repeated retries";
1161
1162pub(crate) async fn auto_recover_rebroadcast_blocked_tasks_global(
1177 router: &Router,
1178 task_manager: &Arc<TaskManager>,
1179 store: &Arc<TaskStore>,
1180 active_repos: &HashSet<String>,
1181) -> anyhow::Result<()> {
1182 const MIN_BLOCK_AGE_MINUTES: i64 = 5;
1183
1184 let all_blocked = match store.list_all_by_status_global(TaskStatus::Blocked).await {
1185 Ok(tasks) => tasks,
1186 Err(e) => {
1187 tracing::warn!(err = %e, "global rebroadcast-blocked sweep: failed to list blocked tasks");
1188 return Ok(());
1189 }
1190 };
1191
1192 let now = chrono::Utc::now();
1193 let candidates: Vec<_> = all_blocked
1194 .into_iter()
1195 .filter(|t| {
1196 if t.block_reason.as_deref() != Some(REBROADCAST_BLOCK_REASON) {
1197 return false;
1198 }
1199 if !active_repos.contains(&t.repo) {
1201 return false;
1202 }
1203 match chrono::DateTime::parse_from_rfc3339(&t.updated_at) {
1205 Ok(updated_at) => {
1206 let age = now - updated_at.with_timezone(&chrono::Utc);
1207 age.num_minutes() >= MIN_BLOCK_AGE_MINUTES
1208 }
1209 Err(_) => true,
1210 }
1211 })
1212 .collect();
1213
1214 if candidates.is_empty() {
1215 return Ok(());
1216 }
1217
1218 let gh = match GhHttp::new() {
1219 Ok(g) => Some(g),
1220 Err(e) => {
1221 tracing::warn!(err = %e, "global rebroadcast-blocked sweep: failed to create GhHttp — skipping PR-state checks this tick");
1222 None
1223 }
1224 };
1225
1226 let any_routable =
1229 !router.available_agents.is_empty() && router.healthy_agent_count("review") > 0;
1230
1231 for task in candidates {
1232 let ext_id = task
1233 .external_id
1234 .clone()
1235 .unwrap_or_else(|| format!("internal:{}", task.id));
1236
1237 if let (Some(gh), Some(pr_number)) =
1238 (&gh, task.pr_number.and_then(|n| u64::try_from(n).ok()))
1239 {
1240 match gh.get_pr(&task.repo, pr_number).await {
1241 Ok(pr) => {
1242 if pr.merged.unwrap_or(false) {
1243 tracing::info!(
1244 task_id = task.id,
1245 repo = %task.repo,
1246 pr_number,
1247 "global rebroadcast-blocked sweep: PR already merged — marking task done"
1248 );
1249 if let Err(e) = task_manager
1250 .mark_task_done_by_store_id(
1251 task.id,
1252 &task.repo,
1253 &ExternalId(ext_id.clone()),
1254 )
1255 .await
1256 {
1257 tracing::warn!(task_id = task.id, repo = %task.repo, err = %e, "global rebroadcast-blocked sweep: failed to mark merged task done");
1258 continue;
1259 }
1260 let fields: &[(&str, serde_json::Value)] = &[
1261 ("block_reason", serde_json::Value::Null),
1262 ("needs_review_refires", serde_json::json!(0)),
1263 ];
1264 if let Err(e) = store.set_fields(task.id, fields).await {
1265 tracing::warn!(task_id = task.id, err = %e, "global rebroadcast-blocked sweep: failed to clear fields after marking done");
1266 }
1267 continue;
1268 }
1269 if pr.state.eq_ignore_ascii_case("closed") {
1270 tracing::info!(
1271 task_id = task.id,
1272 repo = %task.repo,
1273 pr_number,
1274 "global rebroadcast-blocked sweep: PR closed unmerged — leaving task blocked, not re-dispatching"
1275 );
1276 let new_reason = format!(
1277 "{REBROADCAST_BLOCK_REASON} (PR #{pr_number} closed unmerged — no further auto-recovery)"
1278 );
1279 let fields: &[(&str, serde_json::Value)] =
1280 &[("block_reason", serde_json::json!(new_reason))];
1281 if let Err(e) = store.set_fields_silent(task.id, fields).await {
1285 tracing::warn!(task_id = task.id, err = %e, "global rebroadcast-blocked sweep: failed to annotate closed-unmerged block reason");
1286 }
1287 continue;
1288 }
1289 }
1291 Err(e) => {
1292 tracing::warn!(task_id = task.id, repo = %task.repo, pr_number, err = %e, "global rebroadcast-blocked sweep: failed to get PR status — skipping this tick");
1293 continue;
1294 }
1295 }
1296 }
1297
1298 if !any_routable {
1299 continue;
1300 }
1301
1302 if let Err(e) = task_manager
1303 .update_task_status_by_store_id(
1304 task.id,
1305 &task.repo,
1306 &ExternalId(ext_id.clone()),
1307 Status::NeedsReview,
1308 )
1309 .await
1310 {
1311 tracing::warn!(task_id = task.id, repo = %task.repo, err = %e, "global rebroadcast-blocked sweep: failed to reset task to needs_review — skipping");
1312 continue;
1313 }
1314
1315 let fields: &[(&str, serde_json::Value)] = &[
1316 ("block_reason", serde_json::Value::Null),
1317 ("needs_review_refires", serde_json::json!(0)),
1318 ];
1319 if let Err(e) = store.set_fields(task.id, fields).await {
1320 tracing::warn!(task_id = task.id, err = %e, "global rebroadcast-blocked sweep: failed to clear block_reason/needs_review_refires after status reset");
1321 }
1322
1323 tracing::info!(
1324 task_id = task.id,
1325 repo = %task.repo,
1326 ext_id = %ext_id,
1327 "global rebroadcast-blocked sweep: auto-recovered rebroadcast-blocked task — review agents now available"
1328 );
1329 }
1330
1331 Ok(())
1332}
1333
1334async fn auto_unblock_blocked_tasks(
1335 repo: &str,
1336 task_manager: &Arc<TaskManager>,
1337 store: &Arc<TaskStore>,
1338 dispatching: &Arc<DashMap<String, String>>,
1339 auto_merge_in_flight: &Arc<DashSet<String>>,
1340) -> anyhow::Result<()> {
1341 let blocked = match store.list_by_status(repo, TaskStatus::Blocked).await {
1342 Ok(tasks) => tasks,
1343 Err(e) => {
1344 tracing::warn!(err = %e, "failed to list blocked tasks — skipping auto-unblock this tick");
1345 return Ok(());
1346 }
1347 };
1348
1349 if blocked.is_empty() {
1350 return Ok(());
1351 }
1352
1353 let task_ids: Vec<i64> = blocked.iter().map(|t| t.id).collect();
1355 let mut runs_by_task = match store.get_runs_batch(&task_ids).await {
1356 Ok(map) => map,
1357 Err(e) => {
1358 tracing::warn!(err = %e, "get_runs_batch failed — skipping auto-unblock this tick");
1359 return Ok(());
1360 }
1361 };
1362
1363 let (ci_failure_tasks, other_blocked): (Vec<_>, Vec<_>) =
1365 blocked.into_iter().partition(is_ci_failure_block);
1366
1367 if !ci_failure_tasks.is_empty() {
1369 let gh = match GhHttp::new() {
1370 Ok(g) => Some(g),
1371 Err(e) => {
1372 tracing::warn!(err = %e, "failed to create GhHttp — skipping CI failure unblock this tick");
1373 None
1374 }
1375 };
1376 for task in ci_failure_tasks {
1377 if ci_failure_unblock_cooldown_elapsed(&task)
1378 && let Some(ref gh) = gh
1379 && let Err(e) = try_unblock_ci_failure_task(
1380 gh,
1381 repo,
1382 task_manager,
1383 store,
1384 &task,
1385 auto_merge_in_flight,
1386 )
1387 .await
1388 {
1389 tracing::warn!(task_id = task.id, err = %e, "CI failure unblock check failed");
1390 }
1391 }
1392 }
1393
1394 for task in other_blocked {
1396 if task.block_reason.is_some() {
1397 continue;
1398 }
1399
1400 let task_id_str = task
1401 .external_id
1402 .clone()
1403 .unwrap_or_else(|| format!("internal:{}", task.id));
1404 let dispatch_key = format!("{repo}/{task_id_str}");
1405 if dispatching.contains_key(&dispatch_key) {
1406 continue;
1407 }
1408
1409 let runs = runs_by_task.remove(&task.id).unwrap_or_default();
1410 let relevant_runs: Vec<_> = runs
1411 .into_iter()
1412 .filter(|r| r.run_type == "agent" || r.run_type == "review")
1413 .collect();
1414 let mut failures = Vec::new();
1415 let mut has_review_failure = false;
1416 for run in relevant_runs.iter().rev() {
1417 if let Some(category) = classify_failure_from_run(run) {
1418 if run.run_type == "review" {
1419 has_review_failure = true;
1420 }
1421 failures.push(category);
1422 }
1423 if failures.len() >= 3 {
1424 break;
1425 }
1426 }
1427
1428 if failures.is_empty() || !failures.iter().all(|f| f.is_recoverable()) {
1429 continue;
1430 }
1431
1432 let reason_key = failures.first().map_or("", |f| f.as_str());
1434
1435 let current_reason = task.auto_unblock_last_reason.clone();
1438 let reason_changed = reason_key != current_reason;
1439
1440 let cooldown_count = if reason_changed {
1443 0
1444 } else {
1445 task.auto_unblock_count
1446 };
1447
1448 if cooldown_count >= 3 {
1452 continue;
1453 }
1454
1455 if !auto_unblock_cooldown_elapsed(cooldown_count, &task.auto_unblock_last_at) {
1456 continue;
1457 }
1458
1459 let new_count = cooldown_count + 1;
1463 let now = chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string();
1464
1465 let ext_id = task
1466 .external_id
1467 .clone()
1468 .unwrap_or_else(|| format!("internal:{}", task.id));
1469 let new_status = if has_review_failure {
1470 Status::NeedsReview
1471 } else {
1472 Status::New
1473 };
1474 let new_status_str = if has_review_failure {
1475 "needs_review"
1476 } else {
1477 "new"
1478 };
1479
1480 if let Err(e) = task_manager
1483 .update_task_status(&ExternalId(ext_id), new_status)
1484 .await
1485 {
1486 tracing::warn!(
1487 task_id = task.id,
1488 err = %e,
1489 "auto-unblock failed to update status — counter not incremented"
1490 );
1491 continue;
1492 }
1493
1494 let mut fields: Vec<(&str, serde_json::Value)> = vec![
1496 ("auto_unblock_count", serde_json::json!(new_count)),
1497 ("auto_unblock_last_at", serde_json::json!(now)),
1498 ("auto_unblock_last_reason", serde_json::json!(reason_key)),
1499 ("agent", serde_json::Value::Null),
1500 ("model", serde_json::Value::Null),
1501 ];
1502 fields.push(("no_code_reroutes", serde_json::json!(0)));
1503 fields.push(("no_code_last_agent", serde_json::json!("")));
1504 if has_review_failure {
1505 fields.push(("review_agent_failures", serde_json::json!(0)));
1506 fields.push(("review_invocations", serde_json::json!(0)));
1507 fields.push(("review_resets", serde_json::json!(0)));
1508 }
1509 if let Err(e) = store.set_fields(task.id, &fields).await {
1510 tracing::warn!(task_id = task.id, err = %e, "failed to set auto_unblock fields after successful status update");
1511 }
1514
1515 if let Err(e) = store
1516 .append_activity(
1517 task.id,
1518 "auto_unblock",
1519 Some("blocked"),
1520 Some(new_status_str),
1521 None,
1522 None,
1523 Some(&serde_json::json!({
1524 "reason": reason_key,
1525 "count": if reason_changed { 1 } else { task.auto_unblock_count + 1 },
1526 "failures": failures.iter().map(|f| format!("{f:?}")).collect::<Vec<_>>(),
1527 })),
1528 )
1529 .await
1530 {
1531 tracing::warn!(task_id = task.id, err = %e, "failed to log auto_unblock activity");
1532 }
1533
1534 tracing::info!(
1535 task_id = task.id,
1536 failures = ?failures,
1537 review_failure = has_review_failure,
1538 "auto-unblocked task with recoverable failures"
1539 );
1540 }
1541
1542 Ok(())
1543}
1544
1545pub(crate) async fn auto_unblock_ci_failure_blocked_tasks_global(
1556 task_manager: &Arc<TaskManager>,
1557 store: &Arc<TaskStore>,
1558 auto_merge_in_flight: &Arc<DashSet<String>>,
1559) -> anyhow::Result<()> {
1560 let all_blocked = match store.list_all_by_status_global(TaskStatus::Blocked).await {
1561 Ok(tasks) => tasks,
1562 Err(e) => {
1563 tracing::warn!(err = %e, "global CI-failure blocked sweep: failed to list blocked tasks");
1564 return Ok(());
1565 }
1566 };
1567
1568 let ci_failure_tasks: Vec<_> = all_blocked
1569 .into_iter()
1570 .filter(is_ci_failure_block)
1571 .collect();
1572 if ci_failure_tasks.is_empty() {
1573 return Ok(());
1574 }
1575
1576 let gh = match GhHttp::new() {
1577 Ok(g) => g,
1578 Err(e) => {
1579 tracing::warn!(err = %e, "global CI-failure blocked sweep: failed to create GhHttp");
1580 return Ok(());
1581 }
1582 };
1583
1584 for task in ci_failure_tasks {
1585 if !ci_failure_unblock_cooldown_elapsed(&task) {
1586 continue;
1587 }
1588 if let Err(e) = try_unblock_ci_failure_task(
1589 &gh,
1590 &task.repo,
1591 task_manager,
1592 store,
1593 &task,
1594 auto_merge_in_flight,
1595 )
1596 .await
1597 {
1598 tracing::warn!(
1599 task_id = task.id,
1600 repo = %task.repo,
1601 err = %e,
1602 "global CI-failure blocked sweep: unblock check failed"
1603 );
1604 }
1605 }
1606
1607 Ok(())
1608}
1609
1610pub(crate) async fn refire_and_escalate_stale_needs_review_global(
1624 task_manager: &Arc<TaskManager>,
1625 store: &Arc<TaskStore>,
1626 router: &Router,
1627 dispatching: &Arc<DashMap<String, String>>,
1628 active_repos: &HashSet<String>,
1629) -> anyhow::Result<()> {
1630 const MIN_STALE_NEEDS_REVIEW_MINUTES: i64 = 1;
1631 const MAX_NEEDS_REVIEW_REFIRE_ATTEMPTS: u64 = 5;
1632
1633 let needs_review_tasks = match store
1634 .list_all_by_status_global(TaskStatus::NeedsReview)
1635 .await
1636 {
1637 Ok(tasks) => tasks,
1638 Err(e) => {
1639 tracing::warn!(err = %e, "global NeedsReview refire sweep: failed to list needs_review tasks");
1640 return Ok(());
1641 }
1642 };
1643
1644 if needs_review_tasks.is_empty() {
1645 return Ok(());
1646 }
1647
1648 let all_review_agents_cooled = router.healthy_agent_count("review") == 0;
1651 if all_review_agents_cooled {
1652 tracing::info!(
1653 count = needs_review_tasks.len(),
1654 "global NeedsReview refire sweep: no review agent available — skipping this pass"
1655 );
1656 return Ok(());
1657 }
1658
1659 for stored in needs_review_tasks {
1660 if !active_repos.contains(&stored.repo) {
1662 continue;
1663 }
1664
1665 let external = tasks::store_task_to_external(&stored);
1666 let ext_id = external.id.clone();
1667 let repo = stored.repo.clone();
1668
1669 let age_minutes = match chrono::DateTime::parse_from_rfc3339(&external.updated_at) {
1670 Ok(updated_at) => {
1671 let age = chrono::Utc::now() - updated_at.with_timezone(&chrono::Utc);
1672 if age.num_minutes() < MIN_STALE_NEEDS_REVIEW_MINUTES {
1673 continue;
1674 }
1675 Some(age.num_minutes())
1676 }
1677 Err(_) => None,
1678 };
1679
1680 let dispatch_key = format!("{}/{}", repo, ext_id.0);
1682 if dispatching.contains_key(&dispatch_key) {
1683 continue;
1684 }
1685
1686 let current_refires = u64::try_from(stored.needs_review_refires).unwrap_or(0);
1687 let new_refires = current_refires + 1;
1688
1689 if new_refires > MAX_NEEDS_REVIEW_REFIRE_ATTEMPTS {
1690 tracing::warn!(
1691 task_id = stored.id,
1692 repo = %repo,
1693 new_refires,
1694 "global NeedsReview refire sweep: escalating to Blocked after repeated refires"
1695 );
1696 let fields = [
1697 ("block_reason", serde_json::json!(REBROADCAST_BLOCK_REASON)),
1698 (
1699 "last_error",
1700 serde_json::json!(format!("escalated after {} retries", new_refires)),
1701 ),
1702 ];
1703 if let Err(e) = task_manager
1704 .update_task_status_and_result_by_store_id(
1705 stored.id,
1706 &repo,
1707 &ext_id,
1708 Status::Blocked,
1709 &fields,
1710 )
1711 .await
1712 {
1713 tracing::error!(task_id = stored.id, repo = %repo, err = %e, "global NeedsReview refire sweep: escalation to Blocked failed — skipping to avoid silent auto-unblock loop");
1714 continue;
1715 }
1716 if let Err(e) = store::store_increment_by_id(
1717 &Some(Arc::clone(store)),
1718 stored.id,
1719 "needs_review_refires",
1720 )
1721 .await
1722 {
1723 tracing::warn!(
1724 task_id = stored.id,
1725 repo = %repo,
1726 err = %e,
1727 "global NeedsReview refire sweep: failed to increment needs_review_refires after escalation"
1728 );
1729 }
1730 continue;
1731 }
1732
1733 let required_minutes =
1735 MIN_STALE_NEEDS_REVIEW_MINUTES * (1i64 << u32::try_from(current_refires).unwrap_or(0));
1736
1737 let should_fire = match age_minutes {
1738 Some(age) => age >= required_minutes,
1739 None => true,
1740 };
1741
1742 if !should_fire {
1743 continue;
1744 }
1745
1746 if let Err(e) = task_manager
1747 .update_task_status_by_store_id(stored.id, &repo, &ext_id, Status::NeedsReview)
1748 .await
1749 {
1750 tracing::warn!(
1751 task_id = stored.id,
1752 repo = %repo,
1753 err = %e,
1754 "global NeedsReview refire sweep: failed to re-fire NeedsReview event — not incrementing counter"
1755 );
1756 continue;
1757 }
1758
1759 let fired_refires = match store::store_increment_by_id(
1760 &Some(Arc::clone(store)),
1761 stored.id,
1762 "needs_review_refires",
1763 )
1764 .await
1765 {
1766 Ok(v) => v,
1767 Err(e) => {
1768 tracing::warn!(
1769 task_id = stored.id,
1770 repo = %repo,
1771 err = %e,
1772 "global NeedsReview refire sweep: failed to increment needs_review_refires after re-fire"
1773 );
1774 current_refires + 1
1775 }
1776 };
1777
1778 tracing::info!(
1779 task_id = stored.id,
1780 repo = %repo,
1781 age_minutes,
1782 refires = fired_refires,
1783 required_minutes,
1784 "global NeedsReview refire sweep: re-firing NeedsReview event for stale task"
1785 );
1786 }
1787
1788 Ok(())
1789}
1790
1791#[expect(
1798 clippy::too_many_arguments,
1799 reason = "the engine passes its shared handles one by one"
1800)]
1801pub(crate) async fn sync_tick(
1802 backend: &Arc<dyn ExternalBackend>,
1803 tmux: &Arc<TmuxManager>,
1804 repo: &str,
1805 config: &EngineConfig,
1806 router: &Arc<RwLock<Router>>,
1807 task_manager: &Arc<TaskManager>,
1808 store: &Arc<TaskStore>,
1809 dispatching: &Arc<DashMap<String, String>>,
1810 auto_merge_in_flight: &Arc<DashSet<String>>,
1811) -> anyhow::Result<()> {
1812 if !store.is_leader() {
1814 return Ok(());
1815 }
1816
1817 tracing::debug!("sync tick");
1818
1819 let mut phases: Vec<(&str, u128)> = Vec::new();
1820 let mut at = Instant::now();
1821
1822 let gh_circuit_open = is_github_circuit_open();
1825
1826 if gh_circuit_open {
1827 let remaining = github_circuit_remaining_secs();
1828 tracing::debug!(
1829 remaining_secs = remaining,
1830 "GitHub 5xx circuit breaker open — skipping non-critical sync work"
1831 );
1832 if let Err(e) =
1835 try_kv_set_prefer_store(Some(store), "metrics:orch.github_5xx_circuit.open", "1").await
1836 {
1837 tracing::error!(err = %e, "failed to persist circuit breaker state — may cause thundering herd after restart");
1838 }
1839 } else if let Err(e) =
1840 try_kv_set_prefer_store(Some(store), "metrics:orch.github_5xx_circuit.open", "0").await
1841 {
1842 tracing::error!(err = %e, "failed to clear circuit breaker state");
1843 }
1844
1845 if !gh_circuit_open {
1848 if let Err(e) = ingest_external_tasks(backend, repo, store).await {
1849 tracing::debug!(err = %e, "external task ingest failed");
1850 }
1851 phases.push(("ingest", at.elapsed().as_millis()));
1852 at = Instant::now();
1853
1854 if let Err(e) = backend.reconcile_project_statuses().await {
1858 tracing::warn!(err = %e, "project status reconciliation failed");
1859 }
1860 phases.push(("board_reconcile", at.elapsed().as_millis()));
1861 }
1862 at = Instant::now();
1863
1864 if !gh_circuit_open {
1866 let backend = Arc::clone(backend);
1867 let repo = repo.to_string();
1868 let task_manager = Arc::clone(task_manager);
1869 let store = Arc::clone(store);
1870 tokio::spawn(async move {
1871 if let Err(e) = cleanup_done_worktrees(&backend, &repo, &task_manager, &store).await {
1872 tracing::warn!(err = %e, "worktree cleanup failed");
1873 }
1874 if let Err(e) = prune_old_run_output(&store).await {
1875 tracing::warn!(err = %e, "task run pruning failed");
1876 }
1877 });
1878 }
1879
1880 let (in_review_tasks, needs_review_tasks, blocked_review_candidates) =
1881 match prefetch_review_tasks(store, repo).await {
1882 Ok(tasks) => tasks,
1883 Err(e) => {
1884 tracing::warn!(err = %e, "failed to prefetch review tasks");
1885 (vec![], vec![], vec![])
1886 }
1887 };
1888 phases.push(("prefetch_review", at.elapsed().as_millis()));
1889 at = Instant::now();
1890
1891 if !gh_circuit_open {
1892 let mention_fallback = chrono::Utc::now() - chrono::Duration::hours(24);
1893 let mention_since = kv_get_prefer_store(Some(store), "mentions_last_checked")
1894 .await
1895 .unwrap_or_else(|| mention_fallback.format("%Y-%m-%dT%H:%M:%SZ").to_string());
1896 let shared_since = mention_since.clone();
1897
1898 let cached_router_config = router.read().await.config.clone();
1899 let mut combined_needs = needs_review_tasks.clone();
1901 combined_needs.extend(blocked_review_candidates.clone());
1902
1903 let (merge_result, comments_result, review_result) = tokio::join!(
1904 check_merged_prs(
1905 backend,
1906 repo,
1907 store,
1908 task_manager,
1909 &in_review_tasks,
1910 &combined_needs,
1911 ),
1912 fetch_comments_since(backend, &shared_since),
1913 async {
1914 let gh = match GhHttp::new() {
1915 Ok(g) => g,
1916 Err(e) => {
1917 tracing::warn!(err = %e, "failed to create GitHub HTTP client for review poll");
1918 return Ok(());
1919 }
1920 };
1921 Box::pin(review_open_prs(
1922 backend,
1923 repo,
1924 config,
1925 task_manager,
1926 store,
1927 dispatching,
1928 auto_merge_in_flight,
1929 &in_review_tasks,
1930 &gh,
1931 &cached_router_config,
1932 ))
1933 .await
1934 }
1935 );
1936
1937 if let Err(e) = merge_result {
1938 tracing::warn!(err = %e, "PR merge check failed");
1939 }
1940
1941 match comments_result {
1942 Ok(comments) => {
1943 let filtered = filter_mentions_by_since(&comments, &mention_since);
1944
1945 if let Err(e) =
1946 scan_comments(backend, Some(store), repo, task_manager, Some(&filtered)).await
1947 {
1948 tracing::warn!(err = %e, "comment scan failed");
1949 }
1950 }
1951 Err(e) => {
1952 tracing::warn!(err = %e, "comment fetch failed");
1953 }
1954 }
1955
1956 if let Err(e) = review_result {
1957 tracing::warn!(err = %e, "PR review failed");
1958 }
1959
1960 phases.push(("merge_comments_review", at.elapsed().as_millis()));
1961 }
1962 at = Instant::now();
1963
1964 let in_progress_tasks: Vec<_> = match store.list_by_status(repo, TaskStatus::InProgress).await {
1974 Ok(tasks) => tasks
1975 .into_iter()
1976 .map(|stored| {
1977 let external = tasks::store_task_to_external(&stored);
1978 (stored, external)
1979 })
1980 .collect(),
1981 Err(e) => {
1982 tracing::warn!(err = %e, "failed to list in_progress tasks — skipping stale InProgress check");
1983 vec![]
1984 }
1985 };
1986
1987 if !in_progress_tasks.is_empty() {
1988 let live_sessions: HashSet<String> = tmux
1990 .list_sessions()
1991 .await
1992 .unwrap_or_default()
1993 .into_iter()
1994 .map(|s| s.name)
1995 .collect();
1996
1997 for (_stored, external) in &in_progress_tasks {
1998 const MIN_STALE_MINUTES: i64 = 5;
2000
2001 let task_id = &external.id.0;
2002
2003 let dispatch_key = format!("{repo}/{task_id}");
2005 {
2006 if dispatching.contains_key(&dispatch_key) {
2007 tracing::debug!(
2008 task_id = %task_id,
2009 "task locked by dispatch flow, skipping stale check"
2010 );
2011 continue;
2012 }
2013 }
2014
2015 match chrono::DateTime::parse_from_rfc3339(&external.updated_at) {
2018 Ok(updated_at) => {
2019 let age = chrono::Utc::now() - updated_at.with_timezone(&chrono::Utc);
2020 if age.num_minutes() < MIN_STALE_MINUTES {
2021 tracing::debug!(
2022 task_id = %task_id,
2023 age_seconds = age.num_seconds(),
2024 "InProgress task is too young to be considered stale, skipping"
2025 );
2026 continue;
2027 }
2028 }
2029 Err(e) => {
2030 tracing::warn!(
2031 task_id = %task_id,
2032 ts = %external.updated_at,
2033 err = %e,
2034 "invalid updated_at timestamp — treating task as potentially stale"
2035 );
2036 }
2037 }
2038
2039 let agent_session = tmux.session_name(repo, task_id);
2040 if live_sessions.contains(&agent_session) {
2041 continue;
2043 }
2044
2045 {
2049 tracing::warn!(
2050 task_id = %task_id,
2051 session = %agent_session,
2052 "InProgress task has no active agent session — resetting to Routed"
2053 );
2054 match task_manager
2055 .update_task_status_if(&external.id, Status::Routed, Status::InProgress)
2056 .await
2057 {
2058 Err(e) => {
2059 tracing::error!(task_id = %task_id, err = %e, "failed to reset stale InProgress task — task may be stuck in InProgress indefinitely");
2060 }
2061 Ok(false) => {
2062 tracing::debug!(
2063 task_id = %task_id,
2064 "stale InProgress reset skipped — task already transitioned (concurrent Done/Blocked/NeedsReview)"
2065 );
2066 }
2067 Ok(true) => {
2068 }
2070 }
2071 }
2072 }
2073 }
2074
2075 phases.push(("stale_in_progress", at.elapsed().as_millis()));
2076 at = Instant::now();
2077 if let Err(e) =
2079 auto_unblock_blocked_tasks(repo, task_manager, store, dispatching, auto_merge_in_flight)
2080 .await
2081 {
2082 tracing::warn!(err = %e, "auto-unblock failed");
2083 }
2084
2085 phases.push(("auto_unblock", at.elapsed().as_millis()));
2086
2087 engine::log_phase_timings("sync", repo, &phases);
2088 Ok(())
2089}
2090
2091pub(crate) async fn sync_shared_tail(router: &Arc<RwLock<Router>>, store: Option<&Arc<TaskStore>>) {
2096 let Some(store) = store else {
2097 return;
2098 };
2099 if !store.is_leader() {
2100 return;
2101 }
2102 match skills_sync().await {
2103 Ok(()) => {
2104 let llm = router.read().await.llm_router_handle();
2107 llm.invalidate_skills_catalog().await;
2108 }
2109 Err(e) => {
2110 tracing::warn!(err = %e, "skills sync failed");
2111 }
2112 }
2113
2114 let (available_agents, config) = {
2118 let r = router.read().await;
2119 (r.available_agents.clone(), r.config.clone())
2120 };
2121 let model_checker = |agent: &str| -> bool {
2122 for comp in &["simple", "medium", "complex", "review"] {
2123 if config.has_available_model_for_complexity(agent, comp) {
2124 return true;
2125 }
2126 }
2127 false
2128 };
2129 refresh_degraded_agents(
2130 store,
2131 &available_agents,
2132 &model_checker,
2133 router_config::health_check_window_hours(),
2134 router_config::degraded_rate_limit_threshold(),
2135 )
2136 .await;
2137
2138 emit_degraded_agents_if_needed(&available_agents, &config, Some(store)).await;
2139}
2140
2141async fn scan_comments(
2151 backend: &Arc<dyn ExternalBackend>,
2152 store: Option<&Arc<TaskStore>>,
2153 repo: &str,
2154 task_manager: &Arc<TaskManager>,
2155 prefetched_comments: Option<&[Mention]>,
2156) -> anyhow::Result<()> {
2157 let current_user = match backend.get_authenticated_user().await {
2158 Ok(Some(u)) => format!("@{u}"),
2159 Ok(None) => {
2160 tracing::debug!("backend does not support user identity, skipping comment scan");
2161 return Ok(());
2162 }
2163 Err(e) => {
2164 tracing::warn!(err = %e, "failed to get current user, skipping comment scan");
2165 return Ok(());
2166 }
2167 };
2168
2169 let comments = if let Some(c) = prefetched_comments {
2170 c.to_vec()
2171 } else {
2172 let fallback = chrono::Utc::now() - chrono::Duration::hours(24);
2173 let since_str = match kv_get_prefer_store(store, "mentions_last_checked").await {
2174 Some(ts) => ts,
2175 None => fallback.format("%Y-%m-%dT%H:%M:%SZ").to_string(),
2176 };
2177 match backend.get_mentions(&since_str).await {
2178 Ok(m) => m,
2179 Err(e) => {
2180 tracing::warn!(err = %e, "failed to get comments");
2181 return Ok(());
2182 }
2183 }
2184 };
2185
2186 let existing_mentions: HashSet<String> = if let Some(s) = store {
2188 match s.list_source_ids_by_source(repo, "mention", 30).await {
2189 Ok(ids) => ids.into_iter().collect(),
2190 Err(e) => {
2191 tracing::warn!(err = %e, "failed to load mention source IDs — skipping to prevent duplicates");
2192 return Ok(());
2193 }
2194 }
2195 } else {
2196 HashSet::new()
2197 };
2198
2199 let gh_cell: OnceCell<GhHttp> = OnceCell::new();
2203 let get_gh = || async { gh_cell.get_or_try_init(|| async { GhHttp::new() }).await };
2204
2205 let actions: Vec<CommentAction> = comments
2208 .iter()
2209 .map(|c| {
2210 classify_comment(
2211 &c.body,
2212 c.issue_url.as_deref(),
2213 ¤t_user,
2214 existing_mentions.contains(&c.id),
2215 )
2216 })
2217 .collect();
2218
2219 let pr_check_inputs: Vec<(usize, String)> = actions
2223 .iter()
2224 .enumerate()
2225 .filter_map(|(i, action)| {
2226 if let CommentAction::CreateMentionTask {
2227 issue_num: Some(num),
2228 } = action
2229 {
2230 Some((i, num.clone()))
2231 } else {
2232 None
2233 }
2234 })
2235 .collect();
2236
2237 let is_pr_values: Vec<bool> = if pr_check_inputs.is_empty() {
2239 Vec::new()
2240 } else {
2241 let gh = get_gh().await?;
2242 future::join_all(
2243 pr_check_inputs
2244 .iter()
2245 .map(|(_, num)| async move { gh.is_pull_request(repo, num).await }),
2246 )
2247 .await
2248 };
2249
2250 let is_pr_map: HashMap<usize, bool> = pr_check_inputs
2252 .iter()
2253 .map(|(i, _)| *i)
2254 .zip(is_pr_values)
2255 .collect();
2256
2257 let mut cursor = MentionCursor::default();
2258
2259 for (comment_idx, (comment, action)) in comments.iter().zip(actions).enumerate() {
2260 match action {
2261 CommentAction::Skip => {
2262 cursor.advance(&comment.created_at);
2263 }
2264
2265 CommentAction::ExecuteCommand { command, issue_num } => {
2266 let store_opt: Option<Arc<dyn CommandStoreOps>> =
2267 store.map(|s| Arc::clone(s) as Arc<dyn CommandStoreOps>);
2268 let gh = get_gh().await?;
2269 let outcome = validate_and_run_command(
2270 backend,
2271 gh,
2272 repo,
2273 &issue_num,
2274 &command,
2275 &comment.author,
2276 &store_opt,
2277 task_manager,
2278 )
2279 .await;
2280 match outcome {
2281 CommandOutcome::FetchFailed | CommandOutcome::CollaboratorCheckFailed => {
2284 tracing::warn!(
2285 comment_id = %comment.id,
2286 command = %command,
2287 "deferring slash command after a transient GitHub error"
2288 );
2289 cursor.block_on_gap();
2290 break;
2291 }
2292 CommandOutcome::Executed { .. }
2295 | CommandOutcome::NotOpen { .. }
2296 | CommandOutcome::NotCollaborator { .. } => cursor.advance(&comment.created_at),
2297 }
2298 }
2299
2300 CommentAction::ExecuteCommandForMention { command, .. } => {
2301 if let Err(e) = backend.acknowledge_mention(&comment.id).await {
2302 tracing::debug!(err = %e, mention_id = %comment.id, "failed to acknowledge mention");
2303 } else {
2307 let gh = get_gh().await?;
2310 let advanced_safely = handle_slash_command(
2311 backend,
2312 store,
2313 repo,
2314 task_manager,
2315 gh,
2316 comment,
2317 &command,
2318 &mut cursor,
2319 )
2320 .await?;
2321 if !advanced_safely {
2322 cursor.block_on_gap();
2323 break;
2324 }
2325 }
2326 }
2327
2328 CommentAction::CreateMentionTask { issue_num } => {
2329 if let Err(e) = backend.acknowledge_mention(&comment.id).await {
2330 tracing::debug!(err = %e, mention_id = %comment.id, "failed to acknowledge mention");
2331 } else {
2335 let is_pr = is_pr_map.get(&comment_idx).copied().unwrap_or(false);
2340
2341 let (title, task_body) = match (&issue_num, is_pr) {
2342 (Some(num), false) => (
2343 format!("Respond to mention by @{} on task #{num}", comment.author),
2344 format!(
2345 "Mention by @{} on GitHub issue #{num}:\n\n{}\n\n---\nThis mention was posted on issue #{num}. Review the full issue to understand the context before responding.",
2346 comment.author, comment.body
2347 ),
2348 ),
2349 (Some(num), true) => (
2350 format!("Respond to mention by @{} on PR #{num}", comment.author),
2351 format!(
2352 "Mention by @{} on GitHub PR #{num}:\n\n{}\n\n---\nThis mention was posted on PR #{num}. Review the full PR (diff, description, and comments) to understand the context before responding.",
2353 comment.author, comment.body
2354 ),
2355 ),
2356 (None, _) => (
2357 format!("Respond to mention by @{}", comment.author),
2358 format!("Mention by @{}:\n\n{}", comment.author, comment.body),
2359 ),
2360 };
2361
2362 if let Some(s) = store {
2363 let parent_id = match issue_num {
2366 Some(ref num) => {
2367 match resolve_mention_parent_id(s, repo, num, is_pr).await {
2368 Ok(id) => id,
2369 Err(e) => {
2370 tracing::warn!(
2371 mention_id = %comment.id,
2372 err = %e,
2373 "deferring mention task due to parent lookup failure"
2374 );
2375 return Err(anyhow::anyhow!(
2376 "deferring mention {} due to parent lookup failure",
2377 comment.id
2378 ));
2379 }
2380 }
2381 }
2382 None => None,
2383 };
2384 if !record_mention_task(
2385 s,
2386 repo,
2387 comment,
2388 &title,
2389 &task_body,
2390 parent_id,
2391 &mut cursor,
2392 )
2393 .await
2394 {
2395 cursor.block_on_gap();
2396 break;
2397 }
2398 }
2399 }
2400 }
2401 }
2402 }
2403
2404 if let Some(ts) = cursor.into_last_safe_ts() {
2405 kv_set_prefer_store(store, "mentions_last_checked", &ts).await;
2406 }
2407
2408 Ok(())
2409}
2410
2411async fn skills_sync() -> anyhow::Result<()> {
2417 let skills = match config::get_skills() {
2418 Ok(s) => s,
2419 Err(e) => {
2420 tracing::debug!(err = %e, "no skills configured");
2421 return Ok(());
2422 }
2423 };
2424
2425 if skills.is_empty() {
2426 tracing::debug!("no skills configured, skipping sync");
2427 return Ok(());
2428 }
2429
2430 let skills_base = home::skills_dir()?;
2431 let git_timeout = Duration::from_secs(60);
2432
2433 let futs: Vec<_> = skills
2436 .into_iter()
2437 .map(|skill| sync_one_skill(skill, skills_base.clone(), git_timeout))
2438 .collect();
2439 future::join_all(futs).await;
2440
2441 Ok(())
2442}
2443
2444async fn sync_one_skill(skill: config::SkillConfig, skills_base: PathBuf, git_timeout: Duration) {
2449 if skill.repo.contains("..") || skill.repo.matches('/').count() != 1 {
2451 tracing::warn!(repo = %skill.repo, "invalid skill repo format, expected 'owner/repo'");
2452 return;
2453 }
2454
2455 let repo_dir = skills_base.join(&skill.repo);
2456 let repo_url = format!("https://github.com/{}.git", skill.repo);
2457
2458 let repo_exists = fs::metadata(&repo_dir).await.is_ok_and(|m| m.is_dir());
2461
2462 if repo_exists {
2463 tracing::debug!(repo = %skill.repo, "pulling skill repo");
2465 let pull_result = time::timeout(
2466 git_timeout,
2467 git_ops::git()
2468 .await
2469 .args(["pull", "--ff-only", "--prune"])
2470 .current_dir(&repo_dir)
2471 .output_with_context(),
2472 )
2473 .await;
2474
2475 match pull_result {
2476 Ok(Ok(output)) if !output.status.success() => {
2477 let stderr = String::from_utf8_lossy(&output.stderr);
2478 tracing::warn!(repo = %skill.repo, err = %stderr, "git pull failed");
2479 }
2480 Ok(Ok(_)) => {
2481 tracing::debug!(repo = %skill.repo, "skill repo updated");
2482 }
2483 Ok(Err(e)) => {
2484 tracing::warn!(repo = %skill.repo, err = %e, "git pull error");
2485 }
2486 Err(_) => {
2487 tracing::warn!(repo = %skill.repo, "git pull timed out after 60s");
2488 }
2489 }
2490 } else {
2491 tracing::debug!(repo = %skill.repo, "cloning skill repo");
2493 let Some(parent) = repo_dir.parent() else {
2494 tracing::warn!(repo = %skill.repo, "skill repo path has no parent directory");
2495 return;
2496 };
2497 if let Err(e) = fs::create_dir_all(parent).await {
2498 tracing::warn!(repo = %skill.repo, err = %e, "failed to create parent dir for skill");
2499 return;
2500 }
2501 let Some(repo_dir_str) = repo_dir.to_str() else {
2502 tracing::warn!(repo = %skill.repo, "skill repo path is not valid UTF-8");
2503 return;
2504 };
2505
2506 let clone_result = time::timeout(
2507 git_timeout,
2508 git_ops::git()
2509 .await
2510 .args(["clone", "--depth", "1", &repo_url, repo_dir_str])
2511 .output_with_context(),
2512 )
2513 .await;
2514
2515 match clone_result {
2516 Ok(Ok(output)) if !output.status.success() => {
2517 let stderr = String::from_utf8_lossy(&output.stderr);
2518 tracing::warn!(repo = %skill.repo, err = %stderr, "git clone failed");
2519 let _ = fs::remove_dir_all(&repo_dir).await;
2521 }
2522 Ok(Ok(_)) => {
2523 tracing::info!(repo = %skill.repo, "skill repo cloned");
2524 }
2525 Ok(Err(e)) => {
2526 tracing::warn!(repo = %skill.repo, err = %e, "git clone error");
2527 let _ = fs::remove_dir_all(&repo_dir).await;
2528 }
2529 Err(_) => {
2530 tracing::warn!(repo = %skill.repo, "git clone timed out after 60s");
2531 let _ = fs::remove_dir_all(&repo_dir).await;
2532 }
2533 }
2534 }
2535}
2536
2537struct DegradedAgentDetail {
2539 agent: String,
2541 reason: String,
2545 cooled_models: Vec<String>,
2548}
2549
2550struct AgentPoolHealth {
2552 agent: String,
2553 cooled_models: Vec<String>,
2554 persistent_failed_models: Vec<String>,
2555 total_configured_models: usize,
2556}
2557
2558pub(crate) async fn emit_degraded_agents_if_needed(
2567 available_agents: &[String],
2568 config: &RouterConfig,
2569 store: Option<&Arc<TaskStore>>,
2570) {
2571 const STALE_AGENTS_KV: &str = "metrics:orch.model_pool_stale_persistent.agents";
2574
2575 const COMPLEXITIES: &[&str] = &["simple", "medium", "complex", "review"];
2577 const HEAVILY_COOLED_RATIO_THRESHOLD: f64 = 0.5;
2578
2579 let mut details: Vec<DegradedAgentDetail> = Vec::new();
2580 let mut pool_health: Vec<AgentPoolHealth> = Vec::new();
2581
2582 for agent in available_agents {
2583 let agent_in_cd = is_agent_in_cooldown(agent);
2584
2585 let mut cooled_models: Vec<String> = Vec::new();
2587 let mut persistent_failed_models: Vec<String> = Vec::new();
2588 let mut seen_models = HashSet::new();
2589 for comp in COMPLEXITIES {
2590 if let Some(pool) = config.model_pool_for_complexity(agent, comp) {
2591 for model in pool {
2592 if seen_models.insert(model.clone()) && is_model_in_cooldown(agent, &model) {
2593 let key = format!("{agent}:{model}");
2594 if matches!(
2595 cooldown_reason(&key).as_deref(),
2596 Some("persistent_model_error" | "model_not_found")
2597 ) {
2598 persistent_failed_models.push(model.clone());
2599 }
2600 cooled_models.push(model);
2601 }
2602 }
2603 }
2604 }
2605 let total_configured_models = seen_models.len();
2606 pool_health.push(AgentPoolHealth {
2607 agent: agent.clone(),
2608 cooled_models: cooled_models.clone(),
2609 persistent_failed_models: persistent_failed_models.clone(),
2610 total_configured_models,
2611 });
2612
2613 let has_model = COMPLEXITIES
2616 .iter()
2617 .any(|comp| config.has_available_model_for_complexity(agent, comp));
2618
2619 if agent_in_cd || !has_model {
2620 let reason = if agent_in_cd {
2621 cooldown_reason(agent).unwrap_or_else(|| "agent_cooldown".to_string())
2622 } else {
2623 "no_available_model".to_string()
2624 };
2625 details.push(DegradedAgentDetail {
2626 agent: agent.clone(),
2627 reason,
2628 cooled_models,
2629 });
2630 }
2631 }
2632
2633 let count = details.len();
2634
2635 if let Err(e) = try_kv_set_prefer_store(
2638 store,
2639 "metrics:orch.agents_degraded.count",
2640 &count.to_string(),
2641 )
2642 .await
2643 {
2644 tracing::warn!(err = %e, "failed to persist degraded agents count metric");
2645 }
2646
2647 if count >= 3 {
2648 let degraded_agents: Vec<&str> = details.iter().map(|d| d.agent.as_str()).collect();
2650 let cooled_models_dim: Vec<String> = details
2651 .iter()
2652 .filter(|d| !d.cooled_models.is_empty())
2653 .map(|d| format!("{}:[{}]", d.agent, d.cooled_models.join(",")))
2654 .collect();
2655 let reasons_dim: Vec<String> = details
2656 .iter()
2657 .map(|d| format!("{}={}", d.agent, d.reason))
2658 .collect();
2659
2660 tracing::warn!(
2661 degraded_count = count,
2662 degraded_agents = ?degraded_agents,
2663 cooled_models = %cooled_models_dim.join("; "),
2664 cooldown_reasons = %reasons_dim.join("; "),
2665 "multi-agent degradation detected"
2666 );
2667
2668 if let Err(e) =
2670 try_kv_set_prefer_store(store, "metrics:orch.agents_degraded.alert", "1").await
2671 {
2672 tracing::warn!(err = %e, "failed to persist degraded agents alert metric");
2673 }
2674 } else if let Err(e) =
2675 try_kv_set_prefer_store(store, "metrics:orch.agents_degraded.alert", "0").await
2676 {
2677 tracing::warn!(err = %e, "failed to clear degraded agents alert metric");
2679 }
2680
2681 let heavily_cooled_agents: Vec<String> = pool_health
2685 .iter()
2686 .filter_map(|d| {
2687 if d.total_configured_models == 0 || d.persistent_failed_models.is_empty() {
2688 return None;
2689 }
2690 #[expect(
2691 clippy::cast_precision_loss,
2692 reason = "model counts are far below 2^52"
2693 )]
2694 let ratio = d.cooled_models.len() as f64 / d.total_configured_models as f64;
2695 (ratio >= HEAVILY_COOLED_RATIO_THRESHOLD).then(|| {
2696 format!(
2697 "{}:{}/{}:{}",
2698 d.agent,
2699 d.cooled_models.len(),
2700 d.total_configured_models,
2701 d.persistent_failed_models.join(",")
2702 )
2703 })
2704 })
2705 .collect();
2706
2707 let current_dim = heavily_cooled_agents.join(",");
2708 let prev_dim = kv_get_prefer_store(store, STALE_AGENTS_KV)
2709 .await
2710 .unwrap_or_default();
2711
2712 if heavily_cooled_agents.is_empty() {
2713 if !prev_dim.is_empty() {
2714 tracing::info!("agent model pool stale condition cleared");
2715 }
2716 if let Err(e) = try_kv_set_prefer_store(store, STALE_AGENTS_KV, "").await {
2717 tracing::warn!(err = %e, "failed to clear stale model pool agents state");
2718 }
2719 if let Err(e) =
2720 try_kv_set_prefer_store(store, "metrics:orch.model_pool_stale_persistent.alert", "0")
2721 .await
2722 {
2723 tracing::warn!(err = %e, "failed to clear stale model pool alert metric");
2724 }
2725 } else {
2726 if current_dim != prev_dim {
2727 tracing::warn!(
2728 affected_agents = ?heavily_cooled_agents,
2729 "agent model pool appears stale: persistent model failures in heavily cooled pool"
2730 );
2731 }
2732 if let Err(e) = try_kv_set_prefer_store(store, STALE_AGENTS_KV, ¤t_dim).await {
2733 tracing::warn!(err = %e, "failed to persist stale model pool agents state");
2734 }
2735 if let Err(e) =
2736 try_kv_set_prefer_store(store, "metrics:orch.model_pool_stale_persistent.alert", "1")
2737 .await
2738 {
2739 tracing::warn!(err = %e, "failed to persist stale model pool alert metric");
2740 }
2741 }
2742}
2743
2744pub(crate) async fn ingest_external_tasks(
2752 backend: &Arc<dyn ExternalBackend>,
2753 repo: &str,
2754 store: &Arc<TaskStore>,
2755) -> anyhow::Result<()> {
2756 #[derive(Clone)]
2757 struct DuplicateTarget {
2758 id: String,
2759 status_label: String,
2760 }
2761
2762 let mut existing_by_title: HashMap<String, DuplicateTarget> = HashMap::new();
2763 let mut accepted_by_title: HashMap<String, DuplicateTarget> = HashMap::new();
2764
2765 let cutoff = (chrono::Utc::now() - chrono::Duration::hours(24))
2766 .format("%Y-%m-%dT%H:%M:%SZ")
2767 .to_string();
2768 match store.list_for_doctor(repo, &cutoff).await {
2769 Ok(recent) => {
2770 let mut insert_target = |task: &Task| {
2771 if task.origin == "internal" {
2772 return;
2773 }
2774 let Some(ext_id) = task.external_id.as_ref() else {
2775 return;
2776 };
2777 let status_label = task
2778 .labels
2779 .iter()
2780 .find(|l| l.starts_with("status:"))
2781 .cloned()
2782 .unwrap_or_default();
2783 existing_by_title
2784 .entry(task.title.clone())
2785 .or_insert(DuplicateTarget {
2786 id: ext_id.clone(),
2787 status_label,
2788 });
2789 };
2790
2791 for task in recent.iter().filter(|t| t.status != TaskStatus::Done) {
2793 insert_target(task);
2794 }
2795 for task in recent.iter().filter(|t| t.status == TaskStatus::Done) {
2796 insert_target(task);
2797 }
2798 }
2799 Err(e) => {
2800 tracing::warn!(repo, err = %e, "ingest: failed to load recent tasks for dedup");
2801 }
2802 }
2803
2804 let all_tasks: Vec<(ExternalTask, Option<Status>)> =
2814 match backend.list_active_open_issues().await {
2815 Ok(tasks) => tasks,
2816 Err(e) => {
2817 tracing::warn!(
2818 repo,
2819 err = %e,
2820 "ingest: failed to list open issues — no new issue is ingested this cycle"
2821 );
2822 return Ok(());
2823 }
2824 };
2825
2826 let mut id_status_pairs: Vec<(i64, Status)> = Vec::new();
2829 let mut new_task_node_ids: Vec<(i64, String)> = Vec::new();
2832 let existing_ext_ids: HashSet<String> = match store.existing_external_ids(repo).await {
2837 Ok(s) => s,
2838 Err(e) => {
2839 tracing::warn!(repo = repo, err = %e, "ingest: failed to load existing external ids — proceeding");
2840 HashSet::new()
2841 }
2842 };
2843 for (task, status) in &all_tasks {
2844 let is_new = !existing_ext_ids.contains(&task.id.0);
2847
2848 if is_new {
2849 let duplicate_target = existing_by_title
2850 .get(&task.title)
2851 .or_else(|| accepted_by_title.get(&task.title));
2852 if let Some(target) = duplicate_target
2853 && target.id != task.id.0
2854 {
2855 let label = target.status_label.as_str();
2856 let should_dedupe =
2857 match backend.has_open_issue_with_title(&task.title, label).await {
2858 Ok(true) => true,
2859 Ok(false) => false,
2860 Err(e) => {
2861 tracing::warn!(
2862 task_id = task.id.0,
2863 err = %e,
2864 "ingest: dedup check failed — defaulting to keep issue"
2865 );
2866 false
2867 }
2868 };
2869
2870 if should_dedupe {
2871 let comment = format!(
2872 "Duplicate of #{id} — closing to avoid duplicate work.{footer}",
2873 id = target.id,
2874 footer = engine::orch_footer()
2875 );
2876 if let Err(e) = backend.post_comment(&task.id, &comment).await {
2877 tracing::warn!(
2878 task_id = task.id.0,
2879 err = %e,
2880 "ingest: failed to comment on duplicate issue"
2881 );
2882 }
2883 match backend.update_status(&task.id, Status::Done).await {
2884 Ok(()) => {
2885 tracing::info!(
2886 task_id = task.id.0,
2887 duplicate_of = target.id,
2888 "ingest: closed duplicate issue"
2889 );
2890 continue;
2891 }
2892 Err(e) => {
2893 tracing::warn!(
2894 task_id = task.id.0,
2895 err = %e,
2896 "ingest: failed to close duplicate issue — falling through to ingest normally"
2897 );
2898 }
2899 }
2900 }
2901 }
2902 }
2903
2904 match store.ensure_external_task(repo, task).await {
2905 Ok(store_id) => {
2906 if let Ok(number) = task.id.0.parse::<i64>()
2907 && let Err(err) = dependencies::sync_issue_dependencies(
2908 backend.as_ref(),
2909 store,
2910 store_id,
2911 repo,
2912 Some(number),
2913 &task.body,
2914 )
2915 .await
2916 {
2917 tracing::warn!(
2918 task_id = task.id.0,
2919 err = %err,
2920 "ingest: dependency sync failed"
2921 );
2922 }
2923 if let Some(s) = status {
2924 id_status_pairs.push((store_id, *s));
2925 }
2926 if is_new {
2928 let status_label = task
2929 .labels
2930 .iter()
2931 .find(|l| l.starts_with("status:"))
2932 .cloned()
2933 .unwrap_or_default();
2934 accepted_by_title
2935 .entry(task.title.clone())
2936 .or_insert(DuplicateTarget {
2937 id: task.id.0.clone(),
2938 status_label,
2939 });
2940 if let Err(e) = backend.acknowledge_issue(&task.id).await {
2941 tracing::debug!(
2942 task_id = task.id.0,
2943 err = %e,
2944 "ingest: acknowledgment failed"
2945 );
2946 }
2947 let task_status = status.unwrap_or(Status::New);
2950 match backend.sync_to_project(&task.id, task_status).await {
2951 Ok(Some(node_id)) => {
2952 new_task_node_ids.push((store_id, node_id));
2953 }
2954 Ok(None) => {
2955 }
2957 Err(e) => {
2958 tracing::debug!(
2959 task_id = task.id.0,
2960 err = %e,
2961 "ingest: project board sync failed"
2962 );
2963 }
2964 }
2965 }
2966 }
2967 Err(e) => {
2968 tracing::debug!(task_id = task.id.0, ?e, "ingest: upsert failed");
2969 }
2970 }
2971 }
2972
2973 if !id_status_pairs.is_empty() {
2979 let all_ids: Vec<i64> = id_status_pairs.iter().map(|(id, _)| *id).collect();
2980 let existing_map = match store.get_batch(&all_ids).await {
2981 Ok(map) => map,
2982 Err(e) => {
2983 tracing::warn!(err = %e, task_count = all_ids.len(), "ingest: get_batch failed — skipping initial status sync for newly ingested tasks this tick");
2984 return Ok(());
2985 }
2986 };
2987 for (store_id, status) in id_status_pairs {
2988 if let Some(existing) = existing_map.get(&store_id)
2989 && existing.status == TaskStatus::New
2990 {
2991 let db_status = status_to_task_status(status);
2992 if db_status != TaskStatus::New
2993 && let Err(e) = store.update_status(store_id, db_status).await
2994 {
2995 tracing::debug!(?e, "ingest: status sync failed");
2996 }
2997 }
2998 }
2999 }
3000
3001 if !new_task_node_ids.is_empty() {
3004 let node_ids: Vec<String> = new_task_node_ids.iter().map(|(_, n)| n.clone()).collect();
3005 match backend.get_project_item_estimates(&node_ids).await {
3006 Ok(estimates) => {
3007 for (store_id, node_id) in new_task_node_ids {
3008 if let Some(&estimate) = estimates.get(&node_id)
3009 && estimate > 0
3010 && let Err(e) = store.set_estimate_if_zero(store_id, estimate).await
3011 {
3012 tracing::debug!(
3013 store_id,
3014 estimate,
3015 err = %e,
3016 "ingest: failed to set estimate from project"
3017 );
3018 }
3019 }
3020 }
3021 Err(e) => {
3022 tracing::debug!(
3023 err = %e,
3024 "ingest: project estimate fetch failed — skipping estimate sync this tick"
3025 );
3026 }
3027 }
3028 }
3029
3030 Ok(())
3031}
3032
3033#[cfg(test)]
3034mod tests {
3035 use super::*;
3036 use super::{CommentAction, MentionCursor, classify_comment};
3037 use crate::backends::test_helpers::NoopBackend;
3038 use crate::backends::{ExternalId, ExternalTask, Mention, Status};
3039 use crate::engine::commands::OwnerCommand;
3040 use crate::engine::cooldown::record_persistent_model_failure;
3041 use crate::engine::cooldown::set_agent_cooldown;
3042 use crate::engine::dispatch_guard::DispatchGuard;
3043 use crate::engine::router::Router;
3044 use crate::engine::router::RouterConfig;
3045 use crate::github::types::GitHubIssue;
3046 use crate::github::types::GitHubUser;
3047 use crate::store::NewTask;
3048 use crate::store::{RunTokenUsage, TaskStore};
3049 use async_trait::async_trait;
3050 use serial_test::serial;
3051 use std::slice;
3052 use std::sync::Arc;
3053 use std::sync::atomic::AtomicUsize;
3054 use std::sync::atomic::Ordering;
3055 use tokio::sync::Mutex;
3056 use tokio::sync::oneshot;
3057
3058 struct IngestMockBackend {
3062 tasks: Vec<(String, Vec<ExternalTask>)>,
3064 dedup_result: bool,
3065 comments: Mutex<Vec<(String, String)>>,
3066 status_updates: Mutex<Vec<(String, Status)>>,
3067 project_syncs: Mutex<Vec<(String, Status)>>,
3068 project_reconciliations: AtomicUsize,
3069 list_calls: AtomicUsize,
3071 }
3072
3073 impl IngestMockBackend {
3074 fn with_tasks(tasks: Vec<(Status, ExternalTask)>) -> Arc<Self> {
3075 Self::with_tasks_and_dedup(tasks, false)
3076 }
3077
3078 fn with_tasks_and_dedup(
3079 tasks: Vec<(Status, ExternalTask)>,
3080 dedup_result: bool,
3081 ) -> Arc<Self> {
3082 let mut grouped: Vec<(String, Vec<ExternalTask>)> = Vec::new();
3083 for (status, task) in tasks {
3084 let label = status.as_label().to_string();
3085 if let Some(entry) = grouped.iter_mut().find(|(l, _)| l == &label) {
3086 entry.1.push(task);
3087 } else {
3088 grouped.push((label, vec![task]));
3089 }
3090 }
3091 Arc::new(Self {
3092 tasks: grouped,
3093 dedup_result,
3094 comments: Mutex::new(Vec::new()),
3095 status_updates: Mutex::new(Vec::new()),
3096 project_syncs: Mutex::new(Vec::new()),
3097 project_reconciliations: AtomicUsize::new(0),
3098 list_calls: AtomicUsize::new(0),
3099 })
3100 }
3101 }
3102
3103 #[async_trait]
3104 impl ExternalBackend for IngestMockBackend {
3105 fn name(&self) -> &'static str {
3106 "ingest-mock"
3107 }
3108 async fn create_task(&self, _: &str, _: &str, _: &[String]) -> anyhow::Result<ExternalId> {
3109 Ok(ExternalId("new".into()))
3110 }
3111 async fn get_task(&self, _: &ExternalId) -> anyhow::Result<ExternalTask> {
3112 anyhow::bail!("not implemented")
3113 }
3114 async fn list_by_status(&self, status: Status) -> anyhow::Result<Vec<ExternalTask>> {
3115 let label = status.as_label().to_string();
3116 Ok(self
3117 .tasks
3118 .iter()
3119 .find(|(l, _)| l == &label)
3120 .map(|(_, t)| t.clone())
3121 .unwrap_or_default())
3122 }
3123 async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
3124 Ok(vec![])
3125 }
3126 async fn list_active_open_issues(
3127 &self,
3128 ) -> anyhow::Result<Vec<(ExternalTask, Option<Status>)>> {
3129 self.list_calls.fetch_add(1, Ordering::Relaxed);
3130 let mut result = Vec::new();
3131 for (label, tasks) in &self.tasks {
3132 let status = match label.as_str() {
3133 "status:routed" => Some(Status::Routed),
3134 "status:in_progress" => Some(Status::InProgress),
3135 "status:needs_review" => Some(Status::NeedsReview),
3136 "status:in_review" => Some(Status::InReview),
3137 "status:approved" => Some(Status::Approved),
3138 "status:blocked" => Some(Status::Blocked),
3139 _ => None,
3140 };
3141 for task in tasks {
3142 result.push((task.clone(), status));
3143 }
3144 }
3145 Ok(result)
3146 }
3147 async fn post_comment(&self, id: &ExternalId, body: &str) -> anyhow::Result<()> {
3148 self.comments
3149 .lock()
3150 .await
3151 .push((id.0.clone(), body.to_string()));
3152 Ok(())
3153 }
3154 async fn set_labels(&self, _: &ExternalId, _: &[String]) -> anyhow::Result<()> {
3155 Ok(())
3156 }
3157 async fn remove_label(&self, _: &ExternalId, _: &str) -> anyhow::Result<()> {
3158 Ok(())
3159 }
3160 async fn get_sub_issues(&self, _: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
3161 Ok(vec![])
3162 }
3163 async fn health_check(&self) -> anyhow::Result<()> {
3164 Ok(())
3165 }
3166 async fn get_mentions(&self, _: &str) -> anyhow::Result<Vec<Mention>> {
3167 Ok(vec![])
3168 }
3169 async fn update_status(&self, id: &ExternalId, status: Status) -> anyhow::Result<()> {
3170 self.status_updates
3171 .lock()
3172 .await
3173 .push((id.0.clone(), status));
3174 Ok(())
3175 }
3176
3177 async fn has_open_issue_with_title(&self, _: &str, _: &str) -> anyhow::Result<bool> {
3178 Ok(self.dedup_result)
3179 }
3180
3181 async fn sync_to_project(
3182 &self,
3183 id: &ExternalId,
3184 status: Status,
3185 ) -> anyhow::Result<Option<String>> {
3186 self.project_syncs.lock().await.push((id.0.clone(), status));
3187 Ok(None) }
3189
3190 async fn reconcile_project_statuses(&self) -> anyhow::Result<()> {
3191 self.project_reconciliations.fetch_add(1, Ordering::Relaxed);
3192 Ok(())
3193 }
3194 }
3195
3196 fn make_ext_task(id: &str, title: &str) -> ExternalTask {
3197 ExternalTask {
3198 id: ExternalId(id.to_string()),
3199 title: title.to_string(),
3200 body: String::new(),
3201 state: "open".to_string(),
3202 labels: vec![],
3203 author: "user".to_string(),
3204 created_at: "2026-01-01T00:00:00Z".to_string(),
3205 updated_at: "2026-01-01T00:00:00Z".to_string(),
3206 url: format!("https://github.com/owner/repo/issues/{id}"),
3207 }
3208 }
3209
3210 #[serial(cooldown_state)]
3211 #[tokio::test]
3212 async fn ingest_upserts_tasks_into_store() {
3213 cooldown::reset_global_state().await;
3214 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![
3215 (Status::New, make_ext_task("1", "First issue")),
3216 (Status::InProgress, make_ext_task("2", "Second issue")),
3217 ]);
3218 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3219
3220 ingest_external_tasks(&backend, "owner/repo", &store)
3221 .await
3222 .unwrap();
3223
3224 let all = store.list_all("owner/repo").await.unwrap();
3226 assert_eq!(all.len(), 2);
3227
3228 let task1 = store
3229 .get_by_external_id("owner/repo", "1")
3230 .await
3231 .unwrap()
3232 .unwrap();
3233 assert_eq!(task1.title, "First issue");
3234 assert_eq!(task1.status, TaskStatus::New);
3235
3236 let task2 = store
3237 .get_by_external_id("owner/repo", "2")
3238 .await
3239 .unwrap()
3240 .unwrap();
3241 assert_eq!(task2.title, "Second issue");
3242 assert_eq!(task2.status, TaskStatus::InProgress);
3243 }
3244
3245 #[serial(cooldown_state)]
3246 #[tokio::test]
3247 async fn ingest_takes_every_open_issue_whatever_its_age_or_the_last_ingest() {
3248 cooldown::reset_global_state().await;
3252 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3253 store
3254 .kv_set("issues_last_ingested:owner/repo", "2026-10-26T00:00:00Z")
3255 .await
3256 .unwrap();
3257
3258 let mut old = make_ext_task("73", "Filed weeks ago, never ingested");
3259 old.updated_at = "2026-01-01T00:00:00Z".to_string();
3260 let backend = IngestMockBackend::with_tasks(vec![(Status::New, old)]);
3261 let backend_dyn: Arc<dyn ExternalBackend> = Arc::<IngestMockBackend>::clone(&backend);
3262 ingest_external_tasks(&backend_dyn, "owner/repo", &store)
3263 .await
3264 .unwrap();
3265 assert!(
3266 store
3267 .get_by_external_id("owner/repo", "73")
3268 .await
3269 .unwrap()
3270 .is_some()
3271 );
3272
3273 let backend = IngestMockBackend::with_tasks(vec![(
3275 Status::New,
3276 make_ext_task("78", "Opened after the restart"),
3277 )]);
3278 let backend_dyn: Arc<dyn ExternalBackend> = Arc::<IngestMockBackend>::clone(&backend);
3279 ingest_external_tasks(&backend_dyn, "owner/repo", &store)
3280 .await
3281 .unwrap();
3282 assert_eq!(
3283 backend.list_calls.load(Ordering::Relaxed),
3284 1,
3285 "one sync is one listing"
3286 );
3287 assert!(
3288 store
3289 .get_by_external_id("owner/repo", "78")
3290 .await
3291 .unwrap()
3292 .is_some()
3293 );
3294 assert_eq!(
3295 store
3296 .kv_get("issues_last_ingested:owner/repo")
3297 .await
3298 .unwrap()
3299 .as_deref(),
3300 Some("2026-10-26T00:00:00Z"),
3301 "the ingest must keep no cursor, so a stale one is never read or advanced"
3302 );
3303 }
3304
3305 #[serial(cooldown_state)]
3306 #[tokio::test]
3307 async fn ingest_updates_existing_tasks() {
3308 cooldown::reset_global_state().await;
3309 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![(
3310 Status::Routed,
3311 make_ext_task("42", "Updated title"),
3312 )]);
3313 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3314
3315 store
3317 .create(&NewTask {
3318 external_id: Some("42".to_string()),
3319 repo: "owner/repo".to_string(),
3320 origin: "github".to_string(),
3321 title: "Original title".to_string(),
3322 ..Default::default()
3323 })
3324 .await
3325 .unwrap();
3326
3327 ingest_external_tasks(&backend, "owner/repo", &store)
3328 .await
3329 .unwrap();
3330
3331 let task = store
3333 .get_by_external_id("owner/repo", "42")
3334 .await
3335 .unwrap()
3336 .unwrap();
3337 assert_eq!(task.title, "Updated title");
3338 assert_eq!(task.status, TaskStatus::Routed);
3339 }
3340
3341 #[serial(cooldown_state)]
3342 #[tokio::test]
3343 async fn ingest_closes_duplicate_titles() {
3344 cooldown::reset_global_state().await;
3345
3346 let backend = IngestMockBackend::with_tasks_and_dedup(
3347 vec![(Status::New, make_ext_task("2", "Duplicate issue"))],
3348 true,
3349 );
3350 let backend_dyn: Arc<dyn ExternalBackend> = Arc::<IngestMockBackend>::clone(&backend);
3351 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3352
3353 store
3354 .create(&NewTask {
3355 external_id: Some("1".to_string()),
3356 repo: "owner/repo".to_string(),
3357 origin: "github".to_string(),
3358 title: "Duplicate issue".to_string(),
3359 labels: vec!["status:new".to_string()],
3360 ..Default::default()
3361 })
3362 .await
3363 .unwrap();
3364
3365 ingest_external_tasks(&backend_dyn, "owner/repo", &store)
3366 .await
3367 .unwrap();
3368
3369 let all = store.list_all("owner/repo").await.unwrap();
3370 assert_eq!(all.len(), 1);
3371
3372 let comments = backend.comments.lock().await;
3373 assert_eq!(comments.len(), 1);
3374 assert!(comments[0].1.contains("Duplicate of #1"));
3375
3376 let status_updates = backend.status_updates.lock().await;
3377 assert_eq!(status_updates.len(), 1);
3378 assert_eq!(status_updates[0].0, "2");
3379 assert_eq!(status_updates[0].1, Status::Done);
3380 }
3381
3382 #[serial(cooldown_state)]
3383 #[tokio::test]
3384 async fn ingest_handles_empty_backend() {
3385 cooldown::reset_global_state().await;
3386 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
3387 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3388
3389 let result = ingest_external_tasks(&backend, "owner/repo", &store).await;
3391 assert!(result.is_ok());
3392
3393 let all = store.list_all("owner/repo").await.unwrap();
3394 assert!(all.is_empty());
3395 }
3396
3397 #[serial(cooldown_state)]
3398 #[tokio::test]
3399 async fn ingest_syncs_status_correctly_across_statuses() {
3400 cooldown::reset_global_state().await;
3401 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![
3402 (Status::New, make_ext_task("1", "New")),
3403 (Status::Routed, make_ext_task("2", "Routed")),
3404 (Status::InProgress, make_ext_task("3", "InProgress")),
3405 (Status::NeedsReview, make_ext_task("4", "NeedsReview")),
3406 (Status::InReview, make_ext_task("5", "InReview")),
3407 (Status::Blocked, make_ext_task("6", "Blocked")),
3408 ]);
3409 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3410
3411 ingest_external_tasks(&backend, "owner/repo", &store)
3412 .await
3413 .unwrap();
3414
3415 let counts = store.status_counts("owner/repo").await.unwrap();
3416 assert_eq!(counts.get("new"), Some(&1));
3417 assert_eq!(counts.get("routed"), Some(&1));
3418 assert_eq!(counts.get("in_progress"), Some(&1));
3419 assert_eq!(counts.get("needs_review"), Some(&1));
3420 assert_eq!(counts.get("in_review"), Some(&1));
3421 assert_eq!(counts.get("blocked"), Some(&1));
3422 }
3423
3424 #[serial(cooldown_state)]
3425 #[tokio::test]
3426 async fn ingest_does_not_overwrite_store_authoritative_status() {
3427 cooldown::reset_global_state().await;
3428 let backend: Arc<dyn ExternalBackend> =
3430 IngestMockBackend::with_tasks(vec![(Status::New, make_ext_task("99", "My task"))]);
3431 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3432
3433 let id = store
3435 .create(&NewTask {
3436 external_id: Some("99".to_string()),
3437 repo: "owner/repo".to_string(),
3438 origin: "github".to_string(),
3439 title: "My task".to_string(),
3440 ..Default::default()
3441 })
3442 .await
3443 .unwrap();
3444 store.update_status(id, TaskStatus::Routed).await.unwrap();
3445
3446 ingest_external_tasks(&backend, "owner/repo", &store)
3448 .await
3449 .unwrap();
3450
3451 let task = store.get(id).await.unwrap();
3452 assert_eq!(
3453 task.status,
3454 TaskStatus::Routed,
3455 "ingest must not overwrite store-authoritative status"
3456 );
3457 }
3458
3459 #[serial(cooldown_state)]
3460 #[tokio::test]
3461 async fn ingest_sync_to_project_called_only_for_new_tasks() {
3462 cooldown::reset_global_state().await;
3463
3464 let backend = IngestMockBackend::with_tasks(vec![
3466 (Status::New, make_ext_task("10", "Existing task")),
3467 (Status::New, make_ext_task("11", "New task")),
3468 ]);
3469 let backend_dyn: Arc<dyn ExternalBackend> = Arc::<IngestMockBackend>::clone(&backend);
3470 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3471
3472 store
3474 .create(&NewTask {
3475 external_id: Some("10".to_string()),
3476 repo: "owner/repo".to_string(),
3477 origin: "github".to_string(),
3478 title: "Existing task".to_string(),
3479 ..Default::default()
3480 })
3481 .await
3482 .unwrap();
3483
3484 ingest_external_tasks(&backend_dyn, "owner/repo", &store)
3485 .await
3486 .unwrap();
3487
3488 let syncs = backend.project_syncs.lock().await;
3489 assert_eq!(
3491 syncs.len(),
3492 1,
3493 "sync_to_project must be called exactly once (for new task only), got: {syncs:?}"
3494 );
3495 assert_eq!(syncs[0].0, "11", "sync_to_project must target the new task");
3496 }
3497
3498 #[serial(cooldown_state)]
3499 #[tokio::test]
3500 async fn ingest_handles_mixed_labeled_and_unlabeled_open_issues() {
3501 struct MixedBackend;
3505
3506 #[async_trait]
3507 impl ExternalBackend for MixedBackend {
3508 fn name(&self) -> &'static str {
3509 "mixed"
3510 }
3511 async fn create_task(
3512 &self,
3513 _: &str,
3514 _: &str,
3515 _: &[String],
3516 ) -> anyhow::Result<ExternalId> {
3517 Ok(ExternalId("x".into()))
3518 }
3519 async fn get_task(&self, _: &ExternalId) -> anyhow::Result<ExternalTask> {
3520 anyhow::bail!("not implemented")
3521 }
3522 async fn list_by_status(&self, _: Status) -> anyhow::Result<Vec<ExternalTask>> {
3523 Ok(vec![])
3524 }
3525 async fn list_active_open_issues(
3527 &self,
3528 ) -> anyhow::Result<Vec<(ExternalTask, Option<Status>)>> {
3529 Ok(vec![
3530 (make_ext_task("100", "Unlabeled routable"), None),
3531 (
3532 make_ext_task("101", "In progress issue"),
3533 Some(Status::InProgress),
3534 ),
3535 ])
3536 }
3537 async fn post_comment(&self, _: &ExternalId, _: &str) -> anyhow::Result<()> {
3538 Ok(())
3539 }
3540 async fn set_labels(&self, _: &ExternalId, _: &[String]) -> anyhow::Result<()> {
3541 Ok(())
3542 }
3543 async fn remove_label(&self, _: &ExternalId, _: &str) -> anyhow::Result<()> {
3544 Ok(())
3545 }
3546 async fn get_sub_issues(&self, _: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
3547 Ok(vec![])
3548 }
3549 async fn health_check(&self) -> anyhow::Result<()> {
3550 Ok(())
3551 }
3552 async fn get_mentions(&self, _: &str) -> anyhow::Result<Vec<Mention>> {
3553 Ok(vec![])
3554 }
3555 async fn update_status(&self, _: &ExternalId, _: Status) -> anyhow::Result<()> {
3556 Ok(())
3557 }
3558 async fn has_open_issue_with_title(&self, _: &str, _: &str) -> anyhow::Result<bool> {
3559 Ok(false)
3560 }
3561 async fn sync_to_project(
3562 &self,
3563 _: &ExternalId,
3564 _: Status,
3565 ) -> anyhow::Result<Option<String>> {
3566 Ok(None)
3567 }
3568 }
3569
3570 cooldown::reset_global_state().await;
3571
3572 let backend: Arc<dyn ExternalBackend> = Arc::new(MixedBackend);
3573 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3574
3575 ingest_external_tasks(&backend, "owner/repo", &store)
3576 .await
3577 .unwrap();
3578
3579 let all = store.list_all("owner/repo").await.unwrap();
3580 assert_eq!(all.len(), 2, "both issues must be ingested");
3581
3582 let unlabeled = store
3583 .get_by_external_id("owner/repo", "100")
3584 .await
3585 .unwrap()
3586 .unwrap();
3587 assert_eq!(
3588 unlabeled.status,
3589 TaskStatus::New,
3590 "unlabeled issue should start as New"
3591 );
3592
3593 let in_progress = store
3594 .get_by_external_id("owner/repo", "101")
3595 .await
3596 .unwrap()
3597 .unwrap();
3598 assert_eq!(
3599 in_progress.status,
3600 TaskStatus::InProgress,
3601 "labeled in_progress issue must be synced to InProgress"
3602 );
3603 }
3604
3605 #[serial(cooldown_state)]
3608 #[tokio::test]
3609 async fn kv_get_prefer_store_reads_from_store() {
3610 cooldown::reset_global_state().await;
3611 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3612
3613 store.kv_set("k1", "store_val").await.unwrap();
3614
3615 let opt = Some(&store);
3616 let val = kv_get_prefer_store(opt, "k1").await;
3617 assert_eq!(val.as_deref(), Some("store_val"));
3618 }
3619
3620 #[serial(cooldown_state)]
3621 #[tokio::test]
3622 async fn kv_get_prefer_store_returns_none_without_store() {
3623 cooldown::reset_global_state().await;
3624 let opt: Option<&Arc<TaskStore>> = None;
3625 let val = kv_get_prefer_store(opt, "k2").await;
3626 assert_eq!(val, None);
3627 }
3628
3629 #[serial(cooldown_state)]
3630 #[tokio::test]
3631 async fn kv_set_prefer_store_writes_to_store() {
3632 cooldown::reset_global_state().await;
3633 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3634
3635 let opt = Some(&store);
3636 kv_set_prefer_store(opt, "k3", "val3").await;
3637
3638 assert_eq!(store.kv_get("k3").await.unwrap().as_deref(), Some("val3"));
3639 }
3640
3641 #[serial(cooldown_state)]
3642 #[tokio::test]
3643 async fn kv_set_prefer_store_noop_without_store() {
3644 cooldown::reset_global_state().await;
3645 let opt: Option<&Arc<TaskStore>> = None;
3646 kv_set_prefer_store(opt, "k4", "val4").await;
3648 }
3649
3650 #[serial(cooldown_state)]
3653 #[tokio::test]
3654 async fn try_kv_get_prefer_store_returns_value_from_store() {
3655 cooldown::reset_global_state().await;
3656 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3657 store.kv_set("try_k1", "try_val1").await.unwrap();
3658
3659 let opt = Some(&store);
3660 let result = try_kv_get_prefer_store(opt, "try_k1").await;
3661 assert_eq!(result.unwrap().as_deref(), Some("try_val1"));
3662 }
3663
3664 #[serial(cooldown_state)]
3665 #[tokio::test]
3666 async fn try_kv_get_prefer_store_returns_none_for_missing_key() {
3667 cooldown::reset_global_state().await;
3668 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3669 let opt = Some(&store);
3670
3671 let result = try_kv_get_prefer_store(opt, "nonexistent_key").await;
3672 assert_eq!(result.unwrap(), None); }
3674
3675 #[serial(cooldown_state)]
3676 #[tokio::test]
3677 async fn try_kv_get_prefer_store_returns_none_without_store() {
3678 cooldown::reset_global_state().await;
3679 let opt: Option<&Arc<TaskStore>> = None;
3680 let result = try_kv_get_prefer_store(opt, "try_k2").await;
3681 assert_eq!(result.unwrap(), None);
3682 }
3683
3684 #[serial(cooldown_state)]
3685 #[tokio::test]
3686 async fn try_kv_set_prefer_store_writes_to_store() {
3687 cooldown::reset_global_state().await;
3688 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3689 let opt = Some(&store);
3690
3691 let result = try_kv_set_prefer_store(opt, "try_k3", "try_val3").await;
3692 assert!(result.is_ok());
3693 assert_eq!(
3694 store.kv_get("try_k3").await.unwrap().as_deref(),
3695 Some("try_val3")
3696 );
3697 }
3698
3699 #[serial(cooldown_state)]
3700 #[tokio::test]
3701 async fn try_kv_set_prefer_store_ok_without_store() {
3702 cooldown::reset_global_state().await;
3703 let opt: Option<&Arc<TaskStore>> = None;
3704 let result = try_kv_set_prefer_store(opt, "try_k4", "try_val4").await;
3706 assert!(result.is_ok());
3707 }
3708
3709 #[serial(cooldown_state)]
3712 #[test]
3713 fn dispatching_set_blocks_duplicate_processing() {
3714 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3715
3716 let repo = "owner/repo";
3717 let task_id = "42";
3718 let dispatch_key = format!("{repo}/{task_id}");
3719
3720 assert!(!dispatching.contains_key(&dispatch_key));
3722
3723 dispatching.insert(dispatch_key.clone(), task_id.to_string());
3725
3726 assert!(
3728 dispatching.contains_key(&dispatch_key),
3729 "task should be locked while dispatching"
3730 );
3731
3732 dispatching.remove(&dispatch_key);
3734
3735 assert!(
3737 !dispatching.contains_key(&dispatch_key),
3738 "task should be unlocked after review completes"
3739 );
3740 }
3741
3742 #[serial(cooldown_state)]
3743 #[test]
3744 fn dispatching_set_does_not_block_other_tasks() {
3745 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3746
3747 let repo = "owner/repo";
3748
3749 let key_42 = format!("{repo}/42");
3751 dispatching.insert(key_42.clone(), "42".to_string());
3752
3753 let key_43 = format!("{repo}/43");
3755 assert!(
3756 dispatching.contains_key(&key_42),
3757 "task 42 should be locked"
3758 );
3759 assert!(
3760 !dispatching.contains_key(&key_43),
3761 "task 43 should not be locked"
3762 );
3763 }
3764
3765 #[serial(cooldown_state)]
3778 #[tokio::test]
3779 async fn dispatch_key_held_during_review_agent_execution() {
3780 cooldown::reset_global_state().await;
3781
3782 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3783
3784 let dispatch_key = "owner/repo/42".to_string();
3785
3786 dispatching.insert(dispatch_key.clone(), "42".to_string());
3788
3789 let (agent_started_tx, agent_started_rx) = oneshot::channel::<()>();
3790 let (check_done_tx, check_done_rx) = oneshot::channel::<()>();
3791
3792 let dispatching_agent = Arc::clone(&dispatching);
3793 let key_agent = dispatch_key.clone();
3794
3795 let review_task = tokio::spawn(async move {
3797 agent_started_tx.send(()).unwrap();
3798 check_done_rx.await.unwrap();
3800 dispatching_agent.remove(&key_agent);
3802 });
3803
3804 agent_started_rx.await.unwrap();
3807 assert!(
3808 dispatching.contains_key(&dispatch_key),
3809 "dispatch_key must be visible in the dispatching set while the review \
3810 agent is running — without the a6d8b9a fix, review_open_prs would see \
3811 the key absent and re-dispatch the task, silently dropping \
3812 CHANGES_REQUESTED feedback"
3813 );
3814
3815 check_done_tx.send(()).unwrap();
3817 review_task.await.unwrap();
3818
3819 assert!(
3821 !dispatching.contains_key(&dispatch_key),
3822 "dispatch_key must be removed from the dispatching set after the \
3823 review agent completes so subsequent sync ticks can process the task"
3824 );
3825 }
3826
3827 #[serial(cooldown_state)]
3840 #[tokio::test]
3841 async fn dispatch_guard_releases_key_on_panic() {
3842 cooldown::reset_global_state().await;
3843
3844 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3845 let key = "owner/repo/42".to_string();
3846
3847 dispatching.insert(key.clone(), "42".to_string());
3851
3852 let guard = DispatchGuard::new(Arc::clone(&dispatching), key.clone());
3854
3855 let handle = tokio::spawn(async move {
3857 let _guard = guard; panic!("simulated panic inside review_and_merge");
3859 });
3860
3861 let _ = handle.await;
3863
3864 assert!(
3866 !dispatching.contains_key(&key),
3867 "dispatch key must be removed from the dispatching set even when the \
3868 spawned review task panics — without DispatchGuard the key leaks and \
3869 the task gets stuck in a permanent review loop"
3870 );
3871 }
3872
3873 #[serial(cooldown_state)]
3874 #[tokio::test]
3875 async fn sync_tick_reconciles_board_without_ingesting_or_transitioning_tasks() {
3876 cooldown::reset_global_state().await;
3877 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3878 let mock = IngestMockBackend::with_tasks(vec![]);
3879 let backend: Arc<dyn ExternalBackend> = Arc::<IngestMockBackend>::clone(&mock);
3880 let task_manager = Arc::new(TaskManager::with_store(
3881 Arc::clone(&backend),
3882 Arc::clone(&store),
3883 "owner/repo".to_string(),
3884 ));
3885 let dispatching = Arc::new(DashMap::new());
3886 Box::pin(sync_tick(
3887 &backend,
3888 &Arc::new(TmuxManager::new()),
3889 "owner/repo",
3890 &EngineConfig::default(),
3891 &Arc::new(RwLock::new(Router::new_for_test(
3892 RouterConfig::default(),
3893 vec!["claude".to_string()],
3894 ))),
3895 &task_manager,
3896 &store,
3897 &dispatching,
3898 &Arc::new(DashSet::new()),
3899 ))
3900 .await
3901 .unwrap();
3902 assert_eq!(mock.project_reconciliations.load(Ordering::Relaxed), 1);
3903 assert!(mock.project_syncs.lock().await.is_empty());
3904 assert!(mock.status_updates.lock().await.is_empty());
3905 assert!(dispatching.is_empty());
3906 assert!(
3907 store
3908 .existing_external_ids("owner/repo")
3909 .await
3910 .unwrap()
3911 .is_empty()
3912 );
3913 }
3914
3915 #[serial(cooldown_state)]
3916 #[tokio::test]
3917 async fn needs_review_refire_increments_and_fires_when_old_enough() {
3918 cooldown::reset_global_state().await;
3919 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3920 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
3921 let task_manager = Arc::new(TaskManager::with_store(
3922 backend,
3923 Arc::clone(&store),
3924 "owner/repo".to_string(),
3925 ));
3926 let router = reviewer_router();
3927 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3928
3929 let id = store
3930 .upsert_external(&crate::store::UpsertExternal {
3931 repo: "owner/repo",
3932 ext_id: "201",
3933 title: "Needs review old",
3934 body: "",
3935 author: "",
3936 url: "",
3937 labels: &[],
3938 origin: "github",
3939 })
3940 .await
3941 .unwrap();
3942 store
3943 .update_status(id, TaskStatus::NeedsReview)
3944 .await
3945 .unwrap();
3946 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
3947 .bind(id)
3948 .execute(store.pool())
3949 .await
3950 .unwrap();
3951
3952 let t = store.get(id).await.unwrap();
3953 assert_eq!(t.needs_review_refires, 0);
3954
3955 refire_and_escalate_stale_needs_review_global(
3956 &task_manager,
3957 &store,
3958 &router,
3959 &dispatching,
3960 &sweep_repos(),
3961 )
3962 .await
3963 .unwrap();
3964
3965 let t2 = store.get(id).await.unwrap();
3966 assert_eq!(t2.needs_review_refires, 1);
3967 assert_eq!(t2.status, TaskStatus::NeedsReview);
3968 }
3969
3970 #[serial(cooldown_state)]
3971 #[tokio::test]
3972 async fn needs_review_backoff_delays_fire_but_increments_counter() {
3973 cooldown::reset_global_state().await;
3974 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3975 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
3976 let task_manager = Arc::new(TaskManager::with_store(
3977 backend,
3978 Arc::clone(&store),
3979 "owner/repo".to_string(),
3980 ));
3981 let router = reviewer_router();
3982 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3983
3984 let id = store
3985 .upsert_external(&crate::store::UpsertExternal {
3986 repo: "owner/repo",
3987 ext_id: "202",
3988 title: "Needs review backoff",
3989 body: "",
3990 author: "",
3991 url: "",
3992 labels: &[],
3993 origin: "github",
3994 })
3995 .await
3996 .unwrap();
3997 store
3998 .update_status(id, TaskStatus::NeedsReview)
3999 .await
4000 .unwrap();
4001
4002 store
4003 .set_fields(id, &[("needs_review_refires", serde_json::json!(1))])
4004 .await
4005 .unwrap();
4006
4007 let now_minus_1 = (chrono::Utc::now() - chrono::Duration::minutes(1))
4009 .format("%Y-%m-%dT%H:%M:%SZ")
4010 .to_string();
4011 sqlx::query("UPDATE tasks SET updated_at = ? WHERE id = ?")
4012 .bind(now_minus_1.clone())
4013 .bind(id)
4014 .execute(store.pool())
4015 .await
4016 .unwrap();
4017
4018 let before = store.get(id).await.unwrap();
4019 assert_eq!(before.needs_review_refires, 1);
4020
4021 refire_and_escalate_stale_needs_review_global(
4022 &task_manager,
4023 &store,
4024 &router,
4025 &dispatching,
4026 &sweep_repos(),
4027 )
4028 .await
4029 .unwrap();
4030
4031 let after = store.get(id).await.unwrap();
4032 assert_eq!(after.needs_review_refires, 1);
4034 assert_eq!(after.updated_at, now_minus_1);
4035 assert_eq!(after.status, TaskStatus::NeedsReview);
4036 }
4037
4038 fn reviewer_router() -> Router {
4040 let mut config = RouterConfig::default();
4041 config.model_map.insert(
4042 "review".to_string(),
4043 HashMap::from([("claude".to_string(), vec!["sonnet".to_string()])]),
4044 );
4045 Router::new_for_test(config, vec!["claude".to_string()])
4046 }
4047
4048 #[serial(cooldown_state)]
4049 #[tokio::test]
4050 async fn needs_review_escalates_to_blocked_after_max_refires() {
4051 cooldown::reset_global_state().await;
4052 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4053 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4054 let task_manager = Arc::new(TaskManager::with_store(
4055 backend,
4056 Arc::clone(&store),
4057 "owner/repo".to_string(),
4058 ));
4059 let router = reviewer_router();
4060 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4061
4062 let id = store
4063 .upsert_external(&crate::store::UpsertExternal {
4064 repo: "owner/repo",
4065 ext_id: "203",
4066 title: "Needs review escalate",
4067 body: "",
4068 author: "",
4069 url: "",
4070 labels: &[],
4071 origin: "github",
4072 })
4073 .await
4074 .unwrap();
4075 store
4076 .update_status(id, TaskStatus::NeedsReview)
4077 .await
4078 .unwrap();
4079
4080 store
4082 .set_fields(id, &[("needs_review_refires", serde_json::json!(5))])
4083 .await
4084 .unwrap();
4085 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4086 .bind(id)
4087 .execute(store.pool())
4088 .await
4089 .unwrap();
4090
4091 refire_and_escalate_stale_needs_review_global(
4092 &task_manager,
4093 &store,
4094 &router,
4095 &dispatching,
4096 &sweep_repos(),
4097 )
4098 .await
4099 .unwrap();
4100
4101 let after = store.get(id).await.unwrap();
4102 assert_eq!(after.status, TaskStatus::Blocked);
4103 assert!(after.block_reason.is_some());
4104 }
4105
4106 #[serial(cooldown_state)]
4109 #[tokio::test]
4110 async fn needs_review_allows_max_refires_before_escalation() {
4111 cooldown::reset_global_state().await;
4112 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4113 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4114 let task_manager = Arc::new(TaskManager::with_store(
4115 backend,
4116 Arc::clone(&store),
4117 "owner/repo".to_string(),
4118 ));
4119 let router = reviewer_router();
4120 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4121
4122 let id = store
4123 .upsert_external(&crate::store::UpsertExternal {
4124 repo: "owner/repo",
4125 ext_id: "204",
4126 title: "Max refires test",
4127 body: "",
4128 author: "",
4129 url: "",
4130 labels: &[],
4131 origin: "github",
4132 })
4133 .await
4134 .unwrap();
4135 store
4136 .update_status(id, TaskStatus::NeedsReview)
4137 .await
4138 .unwrap();
4139
4140 store
4142 .set_fields(id, &[("needs_review_refires", serde_json::json!(4))])
4143 .await
4144 .unwrap();
4145 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4146 .bind(id)
4147 .execute(store.pool())
4148 .await
4149 .unwrap();
4150
4151 refire_and_escalate_stale_needs_review_global(
4152 &task_manager,
4153 &store,
4154 &router,
4155 &dispatching,
4156 &sweep_repos(),
4157 )
4158 .await
4159 .unwrap();
4160
4161 let after = store.get(id).await.unwrap();
4162 assert_eq!(after.status, TaskStatus::NeedsReview);
4163 assert_eq!(after.needs_review_refires, 5);
4164 assert!(after.block_reason.is_none());
4165 }
4166
4167 fn sweep_repos() -> HashSet<String> {
4169 ["owner/repo", "inactive/repo", "some/repo", "active/repo"]
4170 .into_iter()
4171 .map(String::from)
4172 .collect()
4173 }
4174
4175 #[serial(cooldown_state)]
4177 #[tokio::test]
4178 async fn global_sweeps_leave_tasks_of_a_removed_repo_alone() {
4179 cooldown::reset_global_state().await;
4180 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4181 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4182 let task_manager = Arc::new(TaskManager::with_store(
4183 backend,
4184 Arc::clone(&store),
4185 "active/repo".to_string(),
4186 ));
4187 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4188 let router = router_with_healthy_review_agent();
4189 let active: HashSet<String> = ["active/repo".to_string()].into_iter().collect();
4190
4191 let review_id = store
4192 .create_internal("removed/repo", "Needs review", "", "cron", "1", None)
4193 .await
4194 .unwrap();
4195 store
4196 .update_status(review_id, TaskStatus::NeedsReview)
4197 .await
4198 .unwrap();
4199 let blocked_id = store
4200 .create_internal("removed/repo", "Rebroadcast blocked", "", "cron", "2", None)
4201 .await
4202 .unwrap();
4203 store
4204 .update_status(blocked_id, TaskStatus::Blocked)
4205 .await
4206 .unwrap();
4207 store
4208 .set_block_reason(blocked_id, Some(REBROADCAST_BLOCK_REASON))
4209 .await
4210 .unwrap();
4211 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z'")
4212 .execute(store.pool())
4213 .await
4214 .unwrap();
4215
4216 refire_and_escalate_stale_needs_review_global(
4217 &task_manager,
4218 &store,
4219 &router,
4220 &dispatching,
4221 &active,
4222 )
4223 .await
4224 .unwrap();
4225 auto_recover_rebroadcast_blocked_tasks_global(&router, &task_manager, &store, &active)
4226 .await
4227 .unwrap();
4228
4229 let review = store.get(review_id).await.unwrap();
4230 assert_eq!(review.status, TaskStatus::NeedsReview);
4231 assert_eq!(review.needs_review_refires, 0);
4232 let blocked = store.get(blocked_id).await.unwrap();
4233 assert_eq!(blocked.status, TaskStatus::Blocked);
4234 assert_eq!(
4235 blocked.block_reason.as_deref(),
4236 Some(REBROADCAST_BLOCK_REASON)
4237 );
4238 }
4239
4240 fn router_with_healthy_review_agent() -> Router {
4243 let mut config = RouterConfig::default();
4244 config
4245 .model_map
4246 .entry("review".to_string())
4247 .or_default()
4248 .insert("claude".to_string(), vec!["sonnet".to_string()]);
4249 let mut router = Router::new(config);
4250 router.available_agents = vec!["claude".to_string()];
4251 router
4252 }
4253
4254 #[serial(cooldown_state)]
4259 #[tokio::test]
4260 async fn global_refire_sweep_refires_stale_task_from_inactive_repo() {
4261 cooldown::reset_global_state().await;
4262 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4263 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4264 let task_manager = Arc::new(TaskManager::with_store(
4266 backend,
4267 Arc::clone(&store),
4268 "active/repo".to_string(),
4269 ));
4270 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4271 let router = router_with_healthy_review_agent();
4272
4273 let id = store
4274 .upsert_external(&crate::store::UpsertExternal {
4275 repo: "inactive/repo",
4276 ext_id: "490",
4277 title: "Stale needs_review from inactive repo",
4278 body: "",
4279 author: "",
4280 url: "",
4281 labels: &[],
4282 origin: "github",
4283 })
4284 .await
4285 .unwrap();
4286 store
4287 .update_status(id, TaskStatus::NeedsReview)
4288 .await
4289 .unwrap();
4290 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4291 .bind(id)
4292 .execute(store.pool())
4293 .await
4294 .unwrap();
4295
4296 refire_and_escalate_stale_needs_review_global(
4297 &task_manager,
4298 &store,
4299 &router,
4300 &dispatching,
4301 &sweep_repos(),
4302 )
4303 .await
4304 .unwrap();
4305
4306 let task = store.get(id).await.unwrap();
4307 assert_eq!(
4308 task.needs_review_refires, 1,
4309 "cross-repo task must be refired"
4310 );
4311 assert_eq!(task.status, TaskStatus::NeedsReview);
4312 }
4313
4314 #[serial(cooldown_state)]
4317 #[tokio::test]
4318 async fn global_refire_sweep_escalates_cross_repo_task_to_blocked() {
4319 cooldown::reset_global_state().await;
4320 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4321 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4322 let task_manager = Arc::new(TaskManager::with_store(
4323 backend,
4324 Arc::clone(&store),
4325 "active/repo".to_string(),
4326 ));
4327 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4328 let router = router_with_healthy_review_agent();
4329
4330 let id = store
4331 .upsert_external(&crate::store::UpsertExternal {
4332 repo: "inactive/repo",
4333 ext_id: "493",
4334 title: "Exhausted refires from inactive repo",
4335 body: "",
4336 author: "",
4337 url: "",
4338 labels: &[],
4339 origin: "github",
4340 })
4341 .await
4342 .unwrap();
4343 store
4344 .update_status(id, TaskStatus::NeedsReview)
4345 .await
4346 .unwrap();
4347 store
4348 .set_fields(id, &[("needs_review_refires", serde_json::json!(5))])
4349 .await
4350 .unwrap();
4351 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4352 .bind(id)
4353 .execute(store.pool())
4354 .await
4355 .unwrap();
4356
4357 refire_and_escalate_stale_needs_review_global(
4358 &task_manager,
4359 &store,
4360 &router,
4361 &dispatching,
4362 &sweep_repos(),
4363 )
4364 .await
4365 .unwrap();
4366
4367 let task = store.get(id).await.unwrap();
4368 assert_eq!(task.status, TaskStatus::Blocked);
4369 assert_eq!(
4370 task.block_reason.as_deref(),
4371 Some("review agent rebroadcast escalated after repeated retries")
4372 );
4373 }
4374
4375 #[serial(cooldown_state)]
4380 #[tokio::test]
4381 async fn global_refire_sweep_skips_when_all_review_agents_cooled() {
4382 cooldown::reset_global_state().await;
4383 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4384 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4385 let task_manager = Arc::new(TaskManager::with_store(
4386 backend,
4387 Arc::clone(&store),
4388 "active/repo".to_string(),
4389 ));
4390 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4391 let router = Router::new_for_test(RouterConfig::default(), vec!["claude".to_string()]);
4394
4395 let id = store
4396 .upsert_external(&crate::store::UpsertExternal {
4397 repo: "inactive/repo",
4398 ext_id: "494",
4399 title: "Stale needs_review while agents cooled",
4400 body: "",
4401 author: "",
4402 url: "",
4403 labels: &[],
4404 origin: "github",
4405 })
4406 .await
4407 .unwrap();
4408 store
4409 .update_status(id, TaskStatus::NeedsReview)
4410 .await
4411 .unwrap();
4412 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4413 .bind(id)
4414 .execute(store.pool())
4415 .await
4416 .unwrap();
4417
4418 refire_and_escalate_stale_needs_review_global(
4419 &task_manager,
4420 &store,
4421 &router,
4422 &dispatching,
4423 &sweep_repos(),
4424 )
4425 .await
4426 .unwrap();
4427
4428 let task = store.get(id).await.unwrap();
4429 assert_eq!(
4430 task.needs_review_refires, 0,
4431 "counter must not be touched while cooled"
4432 );
4433 assert_eq!(task.status, TaskStatus::NeedsReview);
4434 }
4435
4436 #[serial(cooldown_state)]
4439 #[tokio::test]
4440 async fn global_refire_sweep_skips_when_no_reviewer_available() {
4441 cooldown::reset_global_state().await;
4442 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4443 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4444 let task_manager = Arc::new(TaskManager::with_store(
4445 backend,
4446 Arc::clone(&store),
4447 "active/repo".to_string(),
4448 ));
4449 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4450 let router = Router::new_for_test(RouterConfig::default(), vec![]);
4451
4452 let id = store
4453 .upsert_external(&crate::store::UpsertExternal {
4454 repo: "some/repo",
4455 ext_id: "495",
4456 title: "needs_review with no reviewer",
4457 body: "",
4458 author: "",
4459 url: "",
4460 labels: &[],
4461 origin: "github",
4462 })
4463 .await
4464 .unwrap();
4465 store
4466 .update_status(id, TaskStatus::NeedsReview)
4467 .await
4468 .unwrap();
4469 store
4470 .set_fields(id, &[("needs_review_refires", serde_json::json!(5))])
4471 .await
4472 .unwrap();
4473 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4474 .bind(id)
4475 .execute(store.pool())
4476 .await
4477 .unwrap();
4478
4479 refire_and_escalate_stale_needs_review_global(
4480 &task_manager,
4481 &store,
4482 &router,
4483 &dispatching,
4484 &sweep_repos(),
4485 )
4486 .await
4487 .unwrap();
4488
4489 let task = store.get(id).await.unwrap();
4490 assert_eq!(task.status, TaskStatus::NeedsReview);
4491 assert_eq!(task.needs_review_refires, 5);
4492 }
4493
4494 #[serial(cooldown_state)]
4500 #[tokio::test]
4501 async fn auto_recover_rebroadcast_blocked_task_after_review_cycle() {
4502 cooldown::reset_global_state().await;
4503 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4504 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4505 let task_manager = Arc::new(TaskManager::with_store(
4506 backend,
4507 Arc::clone(&store),
4508 "owner/repo".to_string(),
4509 ));
4510 let router = router_with_healthy_review_agent();
4511
4512 let id = store
4513 .upsert_external(&crate::store::UpsertExternal {
4514 repo: "owner/repo",
4515 ext_id: "511",
4516 title: "Rebroadcast blocked after review",
4517 body: "",
4518 author: "",
4519 url: "",
4520 labels: &[],
4521 origin: "github",
4522 })
4523 .await
4524 .unwrap();
4525 store.update_status(id, TaskStatus::Blocked).await.unwrap();
4526 store
4527 .set_block_reason(
4528 id,
4529 Some("review agent rebroadcast escalated after repeated retries"),
4530 )
4531 .await
4532 .unwrap();
4533 store
4534 .set_fields(id, &[("review_cycles", serde_json::json!(1))])
4535 .await
4536 .unwrap();
4537 store
4538 .set_fields(id, &[("needs_review_refires", serde_json::json!(6))])
4539 .await
4540 .unwrap();
4541 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4542 .bind(id)
4543 .execute(store.pool())
4544 .await
4545 .unwrap();
4546
4547 auto_recover_rebroadcast_blocked_tasks_global(
4548 &router,
4549 &task_manager,
4550 &store,
4551 &sweep_repos(),
4552 )
4553 .await
4554 .unwrap();
4555
4556 let task = store.get(id).await.unwrap();
4557 assert_eq!(
4558 task.status,
4559 TaskStatus::NeedsReview,
4560 "blocked rebroadcast task must be recovered to needs_review"
4561 );
4562 assert!(
4563 task.block_reason.is_none(),
4564 "block_reason must be cleared on recovery"
4565 );
4566 assert_eq!(
4567 task.needs_review_refires, 0,
4568 "needs_review_refires must be reset to 0 on recovery"
4569 );
4570 }
4571
4572 #[serial(cooldown_state)]
4575 #[tokio::test]
4576 async fn stale_in_review_recovery_skips_fresh_tasks() {
4577 cooldown::reset_global_state().await;
4578 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4579 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4580 let task_manager = Arc::new(TaskManager::with_store(
4581 Arc::clone(&backend),
4582 Arc::clone(&store),
4583 "owner/repo".to_string(),
4584 ));
4585 let tmux = Arc::new(TmuxManager::new());
4586 let router = Arc::new(RwLock::new(Router::new_for_test(
4587 RouterConfig::default(),
4588 vec!["claude".to_string()],
4589 )));
4590 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4591 let auto_merge_in_flight: Arc<DashSet<String>> = Arc::new(DashSet::new());
4592
4593 let id = store
4594 .upsert_external(&crate::store::UpsertExternal {
4595 repo: "owner/repo",
4596 ext_id: "66",
4597 title: "Fresh review",
4598 body: "",
4599 author: "",
4600 url: "",
4601 labels: &[],
4602 origin: "github",
4603 })
4604 .await
4605 .unwrap();
4606 store.update_status(id, TaskStatus::InReview).await.unwrap();
4607 store
4608 .set_fields(id, &[("branch", serde_json::json!("feature/66"))])
4609 .await
4610 .unwrap();
4611 Box::pin(sync_tick(
4614 &backend,
4615 &tmux,
4616 "owner/repo",
4617 &EngineConfig::default(),
4618 &router,
4619 &task_manager,
4620 &store,
4621 &dispatching,
4622 &auto_merge_in_flight,
4623 ))
4624 .await
4625 .unwrap();
4626
4627 let task = store.get(id).await.unwrap();
4629 assert_eq!(task.status, TaskStatus::InReview);
4630 }
4631
4632 #[serial(cooldown_state)]
4633 #[tokio::test]
4634 async fn auto_unblock_routes_recoverable_failure() {
4635 cooldown::reset_global_state().await;
4636 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4637 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4638 let task_manager = Arc::new(TaskManager::with_store(
4639 backend,
4640 Arc::clone(&store),
4641 "owner/repo".to_string(),
4642 ));
4643 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4644
4645 let id = store
4646 .upsert_external(&crate::store::UpsertExternal {
4647 repo: "owner/repo",
4648 ext_id: "77",
4649 title: "Blocked task",
4650 body: "",
4651 author: "",
4652 url: "",
4653 labels: &[],
4654 origin: "github",
4655 })
4656 .await
4657 .unwrap();
4658 store.update_status(id, TaskStatus::Blocked).await.unwrap();
4659 store
4660 .set_fields(
4661 id,
4662 &[
4663 ("agent", serde_json::json!("codex")),
4664 ("model", serde_json::json!("gpt-5")),
4665 ],
4666 )
4667 .await
4668 .unwrap();
4669
4670 let run_id = store
4671 .start_run(&crate::store::StartRun {
4672 task_id: id,
4673 attempt: 1,
4674 run_type: "agent",
4675 agent: "codex",
4676 model: "gpt-5",
4677 command: "codex --model gpt-5",
4678 prompt: "do thing",
4679 })
4680 .await
4681 .unwrap();
4682 store
4683 .complete_run(&crate::store::CompleteRun {
4684 run_id,
4685 exit_code: Some(1),
4686 stdout: "",
4687 stderr: "",
4688 parsed: "",
4689 outcome: "rate_limit",
4690 error: "rate limit exceeded",
4691 tokens: RunTokenUsage::default(),
4692 })
4693 .await
4694 .unwrap();
4695
4696 auto_unblock_blocked_tasks(
4697 "owner/repo",
4698 &task_manager,
4699 &store,
4700 &dispatching,
4701 &Arc::new(DashSet::new()),
4702 )
4703 .await
4704 .unwrap();
4705
4706 let task = store.get(id).await.unwrap();
4707 assert_eq!(task.status, TaskStatus::New);
4708 assert!(task.agent.is_none());
4709 assert!(task.model.is_none());
4710 assert_eq!(task.auto_unblock_count, 1);
4711 assert_ne!(task.auto_unblock_last_at, "");
4712 assert_eq!(task.auto_unblock_last_reason, "RateLimit");
4713 }
4714
4715 #[serial(cooldown_state)]
4716 #[tokio::test]
4717 async fn auto_unblock_skips_manual_block() {
4718 cooldown::reset_global_state().await;
4719 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4720 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4721 let task_manager = Arc::new(TaskManager::with_store(
4722 backend,
4723 Arc::clone(&store),
4724 "owner/repo".to_string(),
4725 ));
4726 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4727
4728 let id = store
4729 .upsert_external(&crate::store::UpsertExternal {
4730 repo: "owner/repo",
4731 ext_id: "88",
4732 title: "Manually blocked",
4733 body: "",
4734 author: "",
4735 url: "",
4736 labels: &[],
4737 origin: "github",
4738 })
4739 .await
4740 .unwrap();
4741 store.update_status(id, TaskStatus::Blocked).await.unwrap();
4742 store
4743 .set_block_reason(id, Some("waiting on input"))
4744 .await
4745 .unwrap();
4746
4747 let run_id = store
4748 .start_run(&crate::store::StartRun {
4749 task_id: id,
4750 attempt: 1,
4751 run_type: "agent",
4752 agent: "codex",
4753 model: "gpt-5",
4754 command: "codex --model gpt-5",
4755 prompt: "do thing",
4756 })
4757 .await
4758 .unwrap();
4759 store
4760 .complete_run(&crate::store::CompleteRun {
4761 run_id,
4762 exit_code: Some(1),
4763 stdout: "",
4764 stderr: "",
4765 parsed: "",
4766 outcome: "rate_limit",
4767 error: "rate limit exceeded",
4768 tokens: RunTokenUsage::default(),
4769 })
4770 .await
4771 .unwrap();
4772
4773 auto_unblock_blocked_tasks(
4774 "owner/repo",
4775 &task_manager,
4776 &store,
4777 &dispatching,
4778 &Arc::new(DashSet::new()),
4779 )
4780 .await
4781 .unwrap();
4782
4783 let task = store.get(id).await.unwrap();
4784 assert_eq!(task.status, TaskStatus::Blocked);
4785 assert_eq!(task.auto_unblock_count, 0);
4786 }
4787
4788 #[serial(cooldown_state)]
4791 #[test]
4792 fn failure_category_as_str() {
4793 assert_eq!(FailureCategory::RateLimit.as_str(), "RateLimit");
4796 assert_eq!(FailureCategory::Timeout.as_str(), "Timeout");
4797 assert_eq!(
4798 FailureCategory::ModelUnavailable.as_str(),
4799 "ModelUnavailable"
4800 );
4801 assert_eq!(FailureCategory::MaxAttempts.as_str(), "MaxAttempts");
4802 assert_eq!(FailureCategory::Unknown.as_str(), "Unknown");
4803 assert_eq!(FailureCategory::SilentExit0.as_str(), "SilentExit0");
4804 assert_eq!(FailureCategory::ConnectionError.as_str(), "ConnectionError");
4805 assert_eq!(FailureCategory::CliFlagError.as_str(), "CliFlagError");
4806 assert_eq!(
4807 FailureCategory::AllAgentsExhausted.as_str(),
4808 "AllAgentsExhausted"
4809 );
4810 assert_eq!(FailureCategory::ParseError.as_str(), "ParseError");
4811 assert_eq!(FailureCategory::FalseFailure.as_str(), "FalseFailure");
4812 assert_eq!(FailureCategory::PushFailed.as_str(), "PushFailed");
4813 assert_eq!(FailureCategory::PrCreateFailed.as_str(), "PrCreateFailed");
4814 }
4815
4816 #[serial(cooldown_state)]
4819 #[test]
4820 fn classify_failure_model_unavailable_phrase_is_recoverable() {
4821 let category = classify_failure(
4822 "model unavailable (anthropic/claude-sonnet-4-6): Model not found: anthropic/claude-sonnet-4-6",
4823 "error",
4824 );
4825 assert_eq!(
4826 category,
4827 FailureCategory::ModelUnavailable,
4828 "\"model unavailable\" error must classify as ModelUnavailable"
4829 );
4830 assert!(
4831 category.is_recoverable(),
4832 "ModelUnavailable must be recoverable so auto-unblock can re-route"
4833 );
4834 }
4835
4836 #[serial(cooldown_state)]
4838 #[test]
4839 fn classify_failure_kimi_billing_cycle_is_credit_exhausted_not_rate_limit() {
4840 let error = "kimi rate limit: Failed to authenticate. API Error: 403 You've reached \
4847 your usage limit for this billing cycle. Your quota will be refreshed soon.";
4848 let category = classify_failure(error, "rate_limit");
4849 assert_eq!(
4850 category,
4851 FailureCategory::CreditExhausted,
4852 "billing-cycle exhaustion must classify as CreditExhausted, not RateLimit"
4853 );
4854 assert!(category.is_recoverable());
4855 }
4856
4857 #[serial(cooldown_state)]
4858 #[test]
4859 fn classify_failure_billing_cycle_exhausted_outcome_is_credit_exhausted() {
4860 let category = classify_failure("some billing message", "billing_cycle_exhausted");
4863 assert_eq!(category, FailureCategory::CreditExhausted);
4864 }
4865
4866 #[serial(cooldown_state)]
4867 #[test]
4868 fn classify_failure_generic_rate_limit_is_still_rate_limit() {
4869 let category = classify_failure("429 Too Many Requests", "rate_limit");
4872 assert_eq!(category, FailureCategory::RateLimit);
4873 }
4874
4875 #[serial(cooldown_state)]
4877 #[test]
4878 fn classify_failure_pr_create_failed_specific_patterns() {
4879 assert_eq!(
4881 classify_failure("create pr failed", ""),
4882 FailureCategory::PrCreateFailed
4883 );
4884 assert_eq!(
4885 classify_failure("failed to create pull request", ""),
4886 FailureCategory::PrCreateFailed
4887 );
4888 assert_eq!(
4889 classify_failure("pull request creation failed", ""),
4890 FailureCategory::PrCreateFailed
4891 );
4892
4893 assert_ne!(
4895 classify_failure("network failed while fetching pull request status", ""),
4896 FailureCategory::PrCreateFailed
4897 );
4898 assert_ne!(
4899 classify_failure("connection failed, retrying pull request check", ""),
4900 FailureCategory::PrCreateFailed
4901 );
4902 assert_ne!(
4903 classify_failure("pull request created but failed to merge", ""),
4904 FailureCategory::PrCreateFailed
4905 );
4906 }
4907
4908 #[serial(cooldown_state)]
4909 #[test]
4910 fn classify_failure_model_not_found_is_recoverable() {
4911 let category = classify_failure("Model not found: gpt-5-ultra", "error");
4912 assert_eq!(
4913 category,
4914 FailureCategory::ModelUnavailable,
4915 "\"model not found\" error must classify as ModelUnavailable"
4916 );
4917 assert!(category.is_recoverable());
4918 }
4919
4920 #[serial(cooldown_state)]
4921 #[test]
4922 fn classify_failure_model_does_not_exist_is_recoverable() {
4923 let category = classify_failure("The model `claude-opus-99` does not exist", "error");
4924 assert_eq!(category, FailureCategory::ModelUnavailable);
4925 assert!(category.is_recoverable());
4926 }
4927
4928 #[serial(cooldown_state)]
4929 #[tokio::test]
4930 async fn auto_unblock_routes_task_blocked_by_model_unavailable() {
4931 cooldown::reset_global_state().await;
4932 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4933 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4934 let task_manager = Arc::new(TaskManager::with_store(
4935 backend,
4936 Arc::clone(&store),
4937 "owner/repo".to_string(),
4938 ));
4939 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4940
4941 let id = store
4942 .upsert_external(&crate::store::UpsertExternal {
4943 repo: "owner/repo",
4944 ext_id: "1202",
4945 title: "Task blocked by model unavailable",
4946 body: "",
4947 author: "",
4948 url: "",
4949 labels: &[],
4950 origin: "github",
4951 })
4952 .await
4953 .unwrap();
4954 store.update_status(id, TaskStatus::Blocked).await.unwrap();
4955
4956 let run_id = store
4957 .start_run(&crate::store::StartRun {
4958 task_id: id,
4959 attempt: 1,
4960 run_type: "agent",
4961 agent: "opencode",
4962 model: "anthropic/claude-sonnet-4-6",
4963 command: "opencode",
4964 prompt: "fix the bug",
4965 })
4966 .await
4967 .unwrap();
4968 store
4969 .complete_run(&crate::store::CompleteRun {
4970 run_id,
4971 exit_code: Some(1),
4972 stdout: "",
4973 stderr: "",
4974 parsed: "",
4975 outcome: "error",
4976 error: "model unavailable (anthropic/claude-sonnet-4-6): Model not found: anthropic/claude-sonnet-4-6",
4977 tokens: RunTokenUsage::default(),
4978 })
4979 .await
4980 .unwrap();
4981
4982 auto_unblock_blocked_tasks(
4983 "owner/repo",
4984 &task_manager,
4985 &store,
4986 &dispatching,
4987 &Arc::new(DashSet::new()),
4988 )
4989 .await
4990 .unwrap();
4991
4992 let task = store.get(id).await.unwrap();
4993 assert_eq!(
4994 task.status,
4995 TaskStatus::New,
4996 "task blocked by model unavailable must be auto-unblocked and re-routed"
4997 );
4998 assert_eq!(task.auto_unblock_count, 1);
4999 assert_eq!(task.auto_unblock_last_reason, "ModelUnavailable");
5000 }
5001
5002 #[serial(cooldown_state)]
5005 #[tokio::test]
5006 async fn auto_unblock_resets_count_when_reason_changes() {
5007 cooldown::reset_global_state().await;
5008 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5012 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
5013 let task_manager = Arc::new(TaskManager::with_store(
5014 backend,
5015 Arc::clone(&store),
5016 "owner/repo".to_string(),
5017 ));
5018 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
5019
5020 let id = store
5021 .upsert_external(&crate::store::UpsertExternal {
5022 repo: "owner/repo",
5023 ext_id: "1227",
5024 title: "Task with different failure reason",
5025 body: "",
5026 author: "",
5027 url: "",
5028 labels: &[],
5029 origin: "github",
5030 })
5031 .await
5032 .unwrap();
5033 store.update_status(id, TaskStatus::Blocked).await.unwrap();
5034 store
5036 .set_fields(
5037 id,
5038 &[
5039 ("auto_unblock_count", serde_json::json!(1)),
5040 (
5041 "auto_unblock_last_reason",
5042 serde_json::json!("ModelUnavailable"),
5043 ),
5044 ],
5045 )
5046 .await
5047 .unwrap();
5048
5049 let run_id = store
5051 .start_run(&crate::store::StartRun {
5052 task_id: id,
5053 attempt: 2,
5054 run_type: "agent",
5055 agent: "claude",
5056 model: "opus",
5057 command: "claude",
5058 prompt: "fix the bug",
5059 })
5060 .await
5061 .unwrap();
5062 store
5063 .complete_run(&crate::store::CompleteRun {
5064 run_id,
5065 exit_code: Some(1),
5066 stdout: "",
5067 stderr: "",
5068 parsed: "",
5069 outcome: "error",
5070 error: "rate limit reached for model opus",
5071 tokens: RunTokenUsage::default(),
5072 })
5073 .await
5074 .unwrap();
5075
5076 auto_unblock_blocked_tasks(
5077 "owner/repo",
5078 &task_manager,
5079 &store,
5080 &dispatching,
5081 &Arc::new(DashSet::new()),
5082 )
5083 .await
5084 .unwrap();
5085
5086 let task = store.get(id).await.unwrap();
5087 assert_eq!(
5088 task.status,
5089 TaskStatus::New,
5090 "task must be auto-unblocked even with count=1 from a different reason"
5091 );
5092 assert_eq!(task.auto_unblock_count, 1);
5094 assert_eq!(
5095 task.auto_unblock_last_reason, "RateLimit",
5096 "reason must be updated to the new failure category"
5097 );
5098 }
5099
5100 #[serial(cooldown_state)]
5106 #[tokio::test]
5107 async fn auto_unblock_leaves_max_attempts_blocked_task_alone() {
5108 cooldown::reset_global_state().await;
5109 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5110 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
5111 let task_manager = Arc::new(TaskManager::with_store(
5112 backend,
5113 Arc::clone(&store),
5114 "owner/repo".to_string(),
5115 ));
5116 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
5117
5118 let id = store
5119 .upsert_external(&crate::store::UpsertExternal {
5120 repo: "owner/repo",
5121 ext_id: "3783",
5122 title: "Approved work sent back to the agent",
5123 body: "",
5124 author: "",
5125 url: "",
5126 labels: &[],
5127 origin: "github",
5128 })
5129 .await
5130 .unwrap();
5131 store.update_status(id, TaskStatus::Blocked).await.unwrap();
5132
5133 let run_id = store
5134 .start_run(&crate::store::StartRun {
5135 task_id: id,
5136 attempt: 11,
5137 run_type: "agent",
5138 agent: "claude",
5139 model: "opus",
5140 command: "claude",
5141 prompt: "fix the bug",
5142 })
5143 .await
5144 .unwrap();
5145 store
5146 .complete_run(&crate::store::CompleteRun {
5147 run_id,
5148 exit_code: Some(1),
5149 stdout: "",
5150 stderr: "",
5151 parsed: "",
5152 outcome: "error",
5153 error: "exceeded max attempts (11/10). Use `/retry` to reset.",
5154 tokens: RunTokenUsage::default(),
5155 })
5156 .await
5157 .unwrap();
5158
5159 auto_unblock_blocked_tasks(
5160 "owner/repo",
5161 &task_manager,
5162 &store,
5163 &dispatching,
5164 &Arc::new(DashSet::new()),
5165 )
5166 .await
5167 .unwrap();
5168
5169 let task = store.get(id).await.unwrap();
5170 assert_eq!(
5171 task.status,
5172 TaskStatus::Blocked,
5173 "max-attempts block must survive the next tick until a person acts"
5174 );
5175 assert_eq!(
5176 task.auto_unblock_count, 0,
5177 "no auto-unblock attempt should be recorded"
5178 );
5179 }
5180
5181 #[serial(cooldown_state)]
5182 #[tokio::test]
5183 async fn auto_unblock_stores_reason_on_first_unblock() {
5184 cooldown::reset_global_state().await;
5185 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5188 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
5189 let task_manager = Arc::new(TaskManager::with_store(
5190 backend,
5191 Arc::clone(&store),
5192 "owner/repo".to_string(),
5193 ));
5194 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
5195
5196 let id = store
5197 .upsert_external(&crate::store::UpsertExternal {
5198 repo: "owner/repo",
5199 ext_id: "1227b",
5200 title: "Task for first-time reason storage",
5201 body: "",
5202 author: "",
5203 url: "",
5204 labels: &[],
5205 origin: "github",
5206 })
5207 .await
5208 .unwrap();
5209 store.update_status(id, TaskStatus::Blocked).await.unwrap();
5210
5211 let run_id = store
5212 .start_run(&crate::store::StartRun {
5213 task_id: id,
5214 attempt: 1,
5215 run_type: "agent",
5216 agent: "claude",
5217 model: "opus",
5218 command: "claude",
5219 prompt: "fix the bug",
5220 })
5221 .await
5222 .unwrap();
5223 store
5224 .complete_run(&crate::store::CompleteRun {
5225 run_id,
5226 exit_code: Some(1),
5227 stdout: "",
5228 stderr: "",
5229 parsed: "",
5230 outcome: "timeout",
5231 error: "task timed out",
5232 tokens: RunTokenUsage::default(),
5233 })
5234 .await
5235 .unwrap();
5236
5237 auto_unblock_blocked_tasks(
5238 "owner/repo",
5239 &task_manager,
5240 &store,
5241 &dispatching,
5242 &Arc::new(DashSet::new()),
5243 )
5244 .await
5245 .unwrap();
5246
5247 let task = store.get(id).await.unwrap();
5248 assert_eq!(task.status, TaskStatus::New);
5249 assert_eq!(task.auto_unblock_count, 1);
5250 assert_eq!(
5251 task.auto_unblock_last_reason, "Timeout",
5252 "reason must be stored on first auto-unblock"
5253 );
5254 }
5255
5256 #[serial(cooldown_state)]
5259 #[test]
5260 fn classify_failure_sparse_checkout_not_parse_error() {
5261 let category = classify_failure("error: cannot reset paths in a sparse checkout", "error");
5264 assert_ne!(
5265 category,
5266 FailureCategory::ParseError,
5267 "sparse checkout errors must NOT be classified as ParseError"
5268 );
5269 assert_eq!(
5270 category,
5271 FailureCategory::Unknown,
5272 "sparse checkout errors should be Unknown (not recoverable auto-unblock)"
5273 );
5274 }
5275
5276 #[serial(cooldown_state)]
5277 #[test]
5278 fn classify_failure_sparse_index_not_parse_error() {
5279 let category = classify_failure("failed due to sparse index configuration", "error");
5281 assert_ne!(category, FailureCategory::ParseError);
5282 assert_eq!(category, FailureCategory::Unknown);
5283 }
5284
5285 #[serial(cooldown_state)]
5292 #[test]
5293 fn classify_failure_truncated_is_recoverable() {
5294 let category = classify_failure(
5295 "truncated: review output was cut off before completion \
5296 (output/reasoning token budget exceeded)",
5297 "truncated",
5298 );
5299 assert_eq!(
5300 category,
5301 FailureCategory::ParseError,
5302 "truncated outcome must classify as a recoverable parse-style failure"
5303 );
5304 assert!(
5305 category.is_recoverable(),
5306 "truncated failures must be recoverable so auto-unblock can retry"
5307 );
5308 }
5309
5310 #[serial(cooldown_state)]
5311 #[test]
5312 fn dispatching_key_includes_repo_for_cross_project_isolation() {
5313 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
5314
5315 let key_a = "owner/repo-a/42".to_string();
5317 dispatching.insert(key_a.clone(), "42".to_string());
5318
5319 let key_b = "owner/repo-b/42".to_string();
5321 assert!(dispatching.contains_key(&key_a));
5322 assert!(
5323 !dispatching.contains_key(&key_b),
5324 "same task ID in different repo should not be blocked"
5325 );
5326 }
5327
5328 #[serial(cooldown_state)]
5329 #[tokio::test]
5330 async fn emit_degraded_agents_writes_metric_when_three_or_more_agents_degraded() {
5331 cooldown::reset_global_state().await;
5332
5333 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5334
5335 let mut config = RouterConfig::default();
5337 config
5338 .model_map
5339 .entry("medium".to_string())
5340 .or_default()
5341 .insert(
5342 "test-agent-3a".to_string(),
5343 vec!["test-model-a".to_string()],
5344 );
5345 config
5346 .model_map
5347 .entry("medium".to_string())
5348 .or_default()
5349 .insert(
5350 "test-agent-3b".to_string(),
5351 vec!["test-model-b".to_string()],
5352 );
5353 config
5354 .model_map
5355 .entry("medium".to_string())
5356 .or_default()
5357 .insert(
5358 "test-agent-3c".to_string(),
5359 vec!["test-model-c".to_string()],
5360 );
5361 config
5362 .model_map
5363 .entry("medium".to_string())
5364 .or_default()
5365 .insert(
5366 "test-agent-3d".to_string(),
5367 vec!["test-model-d".to_string()],
5368 );
5369 let mut router = Router::new(config);
5370 router.available_agents = vec![
5371 "test-agent-3a".to_string(),
5372 "test-agent-3b".to_string(),
5373 "test-agent-3c".to_string(),
5374 "test-agent-3d".to_string(),
5375 ];
5376
5377 set_agent_cooldown("test-agent-3a", 3600).await;
5379 set_agent_cooldown("test-agent-3b", 3600).await;
5380 set_agent_cooldown("test-agent-3c", 3600).await;
5381
5382 emit_degraded_agents_if_needed(&router.available_agents, &router.config, Some(&store))
5384 .await;
5385
5386 let count_val = store
5388 .kv_get("metrics:orch.agents_degraded.count")
5389 .await
5390 .unwrap();
5391 assert_eq!(count_val.as_deref(), Some("3"));
5392
5393 let alert_val = store
5395 .kv_get("metrics:orch.agents_degraded.alert")
5396 .await
5397 .unwrap();
5398 assert_eq!(alert_val.as_deref(), Some("1"));
5399 }
5400
5401 #[serial(cooldown_state)]
5402 #[tokio::test]
5403 async fn emit_degraded_agents_clears_alert_when_below_threshold() {
5404 cooldown::reset_global_state().await;
5405
5406 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5407
5408 let mut config = RouterConfig::default();
5409 config
5410 .model_map
5411 .entry("medium".to_string())
5412 .or_default()
5413 .insert(
5414 "test-agent-2a".to_string(),
5415 vec!["test-model-a".to_string()],
5416 );
5417 config
5418 .model_map
5419 .entry("medium".to_string())
5420 .or_default()
5421 .insert(
5422 "test-agent-2b".to_string(),
5423 vec!["test-model-b".to_string()],
5424 );
5425 let mut router = Router::new(config);
5426 router.available_agents = vec!["test-agent-2a".to_string(), "test-agent-2b".to_string()];
5427
5428 set_agent_cooldown("test-agent-2a", 3600).await;
5430 set_agent_cooldown("test-agent-2b", 3600).await;
5431
5432 emit_degraded_agents_if_needed(&router.available_agents, &router.config, Some(&store))
5433 .await;
5434
5435 let count_val = store
5437 .kv_get("metrics:orch.agents_degraded.count")
5438 .await
5439 .unwrap();
5440 assert_eq!(count_val.as_deref(), Some("2"));
5441
5442 let alert_val = store
5444 .kv_get("metrics:orch.agents_degraded.alert")
5445 .await
5446 .unwrap();
5447 assert_eq!(alert_val.as_deref(), Some("0"));
5448
5449 let stale_pool_alert = store
5450 .kv_get("metrics:orch.model_pool_stale_persistent.alert")
5451 .await
5452 .unwrap();
5453 assert_eq!(stale_pool_alert.as_deref(), Some("0"));
5454 }
5455
5456 #[serial(cooldown_state)]
5457 #[tokio::test]
5458 async fn emit_degraded_agents_sets_stale_pool_alert_for_persistent_model_failures() {
5459 cooldown::reset_global_state().await;
5460
5461 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5462
5463 let mut config = RouterConfig::default();
5464 config
5465 .model_map
5466 .entry("medium".to_string())
5467 .or_default()
5468 .insert(
5469 "test-agent-stale".to_string(),
5470 vec!["model-a".to_string(), "model-b".to_string()],
5471 );
5472 let mut router = Router::new(config);
5473 router.available_agents = vec!["test-agent-stale".to_string()];
5474
5475 record_persistent_model_failure("test-agent-stale", "model-a").await;
5477
5478 emit_degraded_agents_if_needed(&router.available_agents, &router.config, Some(&store))
5479 .await;
5480
5481 let stale_pool_alert = store
5482 .kv_get("metrics:orch.model_pool_stale_persistent.alert")
5483 .await
5484 .unwrap();
5485 assert_eq!(stale_pool_alert.as_deref(), Some("1"));
5486 }
5487
5488 #[serial(cooldown_state)]
5491 #[test]
5492 fn classify_skip_already_processed() {
5493 let action = classify_comment(
5494 "@orch /retry",
5495 Some("https://api.github.com/repos/o/r/issues/1"),
5496 "@bot",
5497 true,
5498 );
5499 assert_eq!(action, CommentAction::Skip);
5500 }
5501
5502 #[serial(cooldown_state)]
5503 #[test]
5504 fn classify_skip_irrelevant_comment() {
5505 let action = classify_comment("just a regular comment", None, "@bot", false);
5506 assert_eq!(action, CommentAction::Skip);
5507 }
5508
5509 #[serial(cooldown_state)]
5510 #[test]
5511 fn classify_command_without_mention() {
5512 let action = classify_comment(
5513 "/retry",
5514 Some("https://api.github.com/repos/o/r/issues/42"),
5515 "@bot",
5516 false,
5517 );
5518 assert_eq!(
5519 action,
5520 CommentAction::ExecuteCommand {
5521 command: OwnerCommand::Retry,
5522 issue_num: "42".to_string(),
5523 }
5524 );
5525 }
5526
5527 #[serial(cooldown_state)]
5528 #[test]
5529 fn classify_command_with_mention() {
5530 let action = classify_comment(
5531 "@orch /close",
5532 Some("https://api.github.com/repos/o/r/issues/7"),
5533 "@bot",
5534 false,
5535 );
5536 assert_eq!(
5537 action,
5538 CommentAction::ExecuteCommandForMention {
5539 command: OwnerCommand::Close,
5540 issue_num: "7".to_string(),
5541 }
5542 );
5543 }
5544
5545 #[serial(cooldown_state)]
5546 #[test]
5547 fn classify_mention_without_command() {
5548 let action = classify_comment(
5549 "@orch can you look at this?",
5550 Some("https://api.github.com/repos/o/r/issues/99"),
5551 "@bot",
5552 false,
5553 );
5554 assert_eq!(
5555 action,
5556 CommentAction::CreateMentionTask {
5557 issue_num: Some("99".to_string()),
5558 }
5559 );
5560 }
5561
5562 #[serial(cooldown_state)]
5563 #[test]
5564 fn classify_mention_without_issue_url() {
5565 let action = classify_comment("@orch help me", None, "@bot", false);
5566 assert_eq!(action, CommentAction::CreateMentionTask { issue_num: None });
5567 }
5568
5569 #[serial(cooldown_state)]
5570 #[test]
5571 fn classify_command_no_mention_no_url_skips() {
5572 let action = classify_comment("/retry", None, "@bot", false);
5573 assert_eq!(action, CommentAction::Skip);
5574 }
5575
5576 #[serial(cooldown_state)]
5577 #[test]
5578 fn classify_mention_via_current_user() {
5579 let action = classify_comment(
5580 "@bot please help",
5581 Some("https://api.github.com/repos/o/r/issues/5"),
5582 "@bot",
5583 false,
5584 );
5585 assert_eq!(
5586 action,
5587 CommentAction::CreateMentionTask {
5588 issue_num: Some("5".to_string()),
5589 }
5590 );
5591 }
5592
5593 #[serial(cooldown_state)]
5594 #[test]
5595 fn classify_command_with_mention_no_url_creates_task() {
5596 let action = classify_comment("@orch /retry", None, "@bot", false);
5598 assert_eq!(action, CommentAction::CreateMentionTask { issue_num: None });
5599 }
5600
5601 #[serial(cooldown_state)]
5602 #[test]
5603 fn mention_cursor_stops_advancing_after_gap() {
5604 let mut cursor = MentionCursor::default();
5605 cursor.advance("2026-01-01T00:00:00Z");
5606 cursor.block_on_gap();
5607 cursor.advance("2026-01-02T00:00:00Z");
5608
5609 assert_eq!(
5610 cursor.into_last_safe_ts().as_deref(),
5611 Some("2026-01-01T00:00:00Z")
5612 );
5613 }
5614
5615 #[serial(cooldown_state)]
5616 #[test]
5617 fn mention_cursor_tracks_max_timestamp_until_gap() {
5618 let mut cursor = MentionCursor::default();
5619 cursor.advance("2026-01-01T00:00:00Z");
5620 cursor.advance("2026-01-01T00:05:00Z");
5621 cursor.block_on_gap();
5622 cursor.advance("2026-01-01T00:10:00Z");
5623
5624 assert_eq!(
5625 cursor.into_last_safe_ts().as_deref(),
5626 Some("2026-01-01T00:05:00Z")
5627 );
5628 }
5629
5630 struct AckTrackingBackend {
5634 ack_calls: Mutex<Vec<String>>,
5635 posted: Mutex<Vec<String>>,
5636 ack_failure: bool,
5637 inner: NoopBackend,
5638 }
5639
5640 impl AckTrackingBackend {
5641 fn new(ack_failure: bool) -> Arc<Self> {
5642 Arc::new(Self {
5643 ack_calls: Mutex::new(Vec::new()),
5644 posted: Mutex::new(Vec::new()),
5645 ack_failure,
5646 inner: NoopBackend,
5647 })
5648 }
5649
5650 fn fail_on_ack(&self) -> bool {
5651 self.ack_failure
5652 }
5653 }
5654
5655 #[async_trait]
5656 impl ExternalBackend for AckTrackingBackend {
5657 fn name(&self) -> &'static str {
5658 "ack-tracking"
5659 }
5660
5661 async fn acknowledge_mention(&self, comment_id: &str) -> anyhow::Result<()> {
5662 self.ack_calls.lock().await.push(comment_id.to_string());
5663 if self.fail_on_ack() {
5664 anyhow::bail!("network error: temporary failure posting reaction");
5666 }
5667 Ok(())
5668 }
5669
5670 async fn get_authenticated_user(&self) -> anyhow::Result<Option<String>> {
5672 Ok(Some("bot".into()))
5673 }
5674
5675 async fn get_mentions(&self, _since: &str) -> anyhow::Result<Vec<Mention>> {
5677 Ok(vec![])
5678 }
5679
5680 async fn create_task(&self, t: &str, b: &str, l: &[String]) -> anyhow::Result<ExternalId> {
5682 self.inner.create_task(t, b, l).await
5683 }
5684 async fn get_task(&self, id: &ExternalId) -> anyhow::Result<ExternalTask> {
5685 self.inner.get_task(id).await
5686 }
5687 async fn list_by_status(&self, s: Status) -> anyhow::Result<Vec<ExternalTask>> {
5688 self.inner.list_by_status(s).await
5689 }
5690 async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
5691 self.inner.list_routable().await
5692 }
5693 async fn post_comment(&self, id: &ExternalId, b: &str) -> anyhow::Result<()> {
5694 self.posted.lock().await.push(b.to_string());
5695 self.inner.post_comment(id, b).await
5696 }
5697 async fn set_labels(&self, id: &ExternalId, l: &[String]) -> anyhow::Result<()> {
5698 self.inner.set_labels(id, l).await
5699 }
5700 async fn remove_label(&self, id: &ExternalId, l: &str) -> anyhow::Result<()> {
5701 self.inner.remove_label(id, l).await
5702 }
5703 async fn get_sub_issues(&self, id: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
5704 self.inner.get_sub_issues(id).await
5705 }
5706 async fn create_sub_task(
5707 &self,
5708 p: &ExternalId,
5709 t: &str,
5710 b: &str,
5711 l: &[String],
5712 ) -> anyhow::Result<ExternalId> {
5713 self.inner.create_sub_task(p, t, b, l).await
5714 }
5715 async fn ensure_status_label(&self, l: &str) -> anyhow::Result<()> {
5716 self.inner.ensure_status_label(l).await
5717 }
5718 async fn has_open_issue_with_title(&self, t: &str, l: &str) -> anyhow::Result<bool> {
5719 self.inner.has_open_issue_with_title(t, l).await
5720 }
5721 async fn health_check(&self) -> anyhow::Result<()> {
5722 self.inner.health_check().await
5723 }
5724 async fn is_pr_merged(&self, b: &str) -> anyhow::Result<bool> {
5725 self.inner.is_pr_merged(b).await
5726 }
5727 async fn update_status(&self, id: &ExternalId, s: Status) -> anyhow::Result<()> {
5728 self.inner.update_status(id, s).await
5729 }
5730 }
5731
5732 #[serial(cooldown_state)]
5735 #[tokio::test]
5736 async fn create_mention_task_skips_on_ack_failure() {
5737 cooldown::reset_global_state().await;
5738 let backend: Arc<AckTrackingBackend> = AckTrackingBackend::new(true);
5739 let store: Arc<TaskStore> = Arc::new(TaskStore::open_memory().await.unwrap());
5740
5741 let initial_ts = "2026-01-01T00:00:00Z";
5743 store
5744 .kv_set("mentions_last_checked", initial_ts)
5745 .await
5746 .unwrap();
5747
5748 let mention = Mention {
5749 id: "mention-abc".into(),
5750 body: "@bot can you help?".into(),
5751 author: "alice".into(),
5752 created_at: "2026-01-02T00:00:00Z".into(),
5753 issue_url: Some("https://api.github.com/repos/owner/repo/issues/42".into()),
5754 };
5755
5756 let backend_trait: Arc<dyn ExternalBackend> = Arc::<AckTrackingBackend>::clone(&backend);
5757 let task_manager = Arc::new(TaskManager::new(Arc::clone(&backend_trait)));
5758
5759 scan_comments(
5760 &backend_trait,
5761 Some(&store),
5762 "owner/repo",
5763 &task_manager,
5764 Some(&[mention]),
5765 )
5766 .await
5767 .unwrap();
5768
5769 let calls = backend.ack_calls.lock().await;
5771 assert_eq!(
5772 &**calls,
5773 &["mention-abc"],
5774 "acknowledge_mention must be called"
5775 );
5776
5777 let all = store.list_all("owner/repo").await.unwrap();
5779 assert!(
5780 all.is_empty(),
5781 "no task should be created when acknowledge_mention fails"
5782 );
5783
5784 let cursor = store.kv_get("mentions_last_checked").await.unwrap();
5786 assert_eq!(
5787 cursor.as_deref(),
5788 Some(initial_ts),
5789 "cursor must not advance when acknowledge_mention fails"
5790 );
5791 }
5792
5793 #[serial(cooldown_state)]
5796 #[tokio::test]
5797 async fn execute_command_for_mention_skips_on_ack_failure() {
5798 cooldown::reset_global_state().await;
5799 let backend: Arc<AckTrackingBackend> = AckTrackingBackend::new(true);
5800 let store: Arc<TaskStore> = Arc::new(TaskStore::open_memory().await.unwrap());
5801
5802 let initial_ts = "2026-01-01T00:00:00Z";
5803 store
5804 .kv_set("mentions_last_checked", initial_ts)
5805 .await
5806 .unwrap();
5807
5808 let mention = Mention {
5810 id: "mention-def".into(),
5811 body: "@bot /close".into(),
5812 author: "alice".into(),
5813 created_at: "2026-01-02T00:00:00Z".into(),
5814 issue_url: Some("https://api.github.com/repos/owner/repo/issues/42".into()),
5815 };
5816
5817 let backend_trait: Arc<dyn ExternalBackend> = Arc::<AckTrackingBackend>::clone(&backend);
5818 let task_manager = Arc::new(TaskManager::new(Arc::clone(&backend_trait)));
5819
5820 scan_comments(
5821 &backend_trait,
5822 Some(&store),
5823 "owner/repo",
5824 &task_manager,
5825 Some(&[mention]),
5826 )
5827 .await
5828 .unwrap();
5829
5830 let calls = backend.ack_calls.lock().await;
5832 assert_eq!(&**calls, &["mention-def"]);
5833
5834 let all = store.list_all("owner/repo").await.unwrap();
5836 assert!(
5837 all.is_empty(),
5838 "no task should be created when acknowledge_mention fails for ExecuteCommandForMention"
5839 );
5840
5841 let cursor = store.kv_get("mentions_last_checked").await.unwrap();
5843 assert_eq!(
5844 cursor.as_deref(),
5845 Some(initial_ts),
5846 "cursor must not advance when acknowledge_mention fails for ExecuteCommandForMention"
5847 );
5848 }
5849
5850 #[derive(Default)]
5854 struct FlakyGh {
5855 get_issue_calls: AtomicUsize,
5856 }
5857
5858 #[async_trait]
5859 impl CommandGh for FlakyGh {
5860 async fn get_issue(&self, _repo: &str, number: &str) -> anyhow::Result<GitHubIssue> {
5861 if self.get_issue_calls.fetch_add(1, Ordering::SeqCst) == 0 {
5862 anyhow::bail!("network error: temporary failure fetching issue");
5863 }
5864 Ok(GitHubIssue {
5865 number: number.parse().unwrap_or(0),
5866 id: 1,
5867 title: "t".into(),
5868 body: None,
5869 state: "open".into(),
5870 labels: Vec::new(),
5871 user: GitHubUser {
5872 login: "alice".into(),
5873 },
5874 created_at: String::new(),
5875 updated_at: String::new(),
5876 html_url: String::new(),
5877 node_id: None,
5878 pull_request: None,
5879 author_association: Some("OWNER".into()),
5880 state_reason: None,
5881 })
5882 }
5883
5884 async fn is_collaborator(&self, _repo: &str, _username: &str) -> anyhow::Result<bool> {
5885 Ok(true)
5886 }
5887
5888 async fn close_issue(&self, _repo: &str, _number: &str) -> anyhow::Result<()> {
5889 Ok(())
5890 }
5891 }
5892
5893 #[serial(cooldown_state)]
5896 #[tokio::test]
5897 async fn slash_command_retries_after_transient_fetch_failure() {
5898 cooldown::reset_global_state().await;
5899 let backend: Arc<AckTrackingBackend> = AckTrackingBackend::new(false);
5900 let store: Arc<TaskStore> = Arc::new(TaskStore::open_memory().await.unwrap());
5901 let initial_ts = "2026-01-01T00:00:00Z";
5902
5903 let mention = Mention {
5904 id: "mention-ghi".into(),
5905 body: "@bot /unblock".into(),
5906 author: "alice".into(),
5907 created_at: "2026-01-02T00:00:00Z".into(),
5908 issue_url: Some("https://api.github.com/repos/owner/repo/issues/42".into()),
5909 };
5910
5911 let backend_trait: Arc<dyn ExternalBackend> = Arc::<AckTrackingBackend>::clone(&backend);
5912 let task_manager = Arc::new(TaskManager::new(Arc::clone(&backend_trait)));
5913 let gh = FlakyGh::default();
5914 let mut cursor = MentionCursor::default();
5915
5916 let advanced_safely = handle_slash_command(
5918 &backend_trait,
5919 Some(&store),
5920 "owner/repo",
5921 &task_manager,
5922 &gh,
5923 &mention,
5924 &OwnerCommand::Unblock,
5925 &mut cursor,
5926 )
5927 .await
5928 .unwrap();
5929
5930 assert!(
5931 !advanced_safely,
5932 "a transient fetch failure must stop cursor advancement"
5933 );
5934 assert!(
5935 cursor.last_safe_ts.is_none(),
5936 "cursor must stay before the comment so the next tick retries it"
5937 );
5938 assert!(
5939 store.list_all("owner/repo").await.unwrap().is_empty(),
5940 "no mention task on a transient failure"
5941 );
5942 assert!(
5943 backend.posted.lock().await.is_empty(),
5944 "nothing is posted to GitHub on a transient failure"
5945 );
5946 assert_eq!(
5947 filter_mentions_by_since(slice::from_ref(&mention), initial_ts).len(),
5948 1,
5949 "the comment is still in range for the next scan"
5950 );
5951
5952 let mut cursor = MentionCursor::default();
5954 let advanced_safely = handle_slash_command(
5955 &backend_trait,
5956 Some(&store),
5957 "owner/repo",
5958 &task_manager,
5959 &gh,
5960 &mention,
5961 &OwnerCommand::Unblock,
5962 &mut cursor,
5963 )
5964 .await
5965 .unwrap();
5966
5967 assert!(advanced_safely);
5968 assert_eq!(
5969 cursor.into_last_safe_ts().as_deref(),
5970 Some("2026-01-02T00:00:00Z"),
5971 "cursor advances once the command actually runs"
5972 );
5973 assert_eq!(
5974 backend.posted.lock().await.len(),
5975 1,
5976 "the command runs once, posting a single confirmation"
5977 );
5978 assert_eq!(
5979 store.list_all("owner/repo").await.unwrap().len(),
5980 1,
5981 "one sentinel mention task for the executed command"
5982 );
5983 }
5984
5985 #[serial(cooldown_state)]
5988 #[tokio::test]
5989 async fn execute_command_keeps_cursor_on_fetch_failure() {
5990 cooldown::reset_global_state().await;
5991 let backend: Arc<AckTrackingBackend> = AckTrackingBackend::new(false);
5992 let store: Arc<TaskStore> = Arc::new(TaskStore::open_memory().await.unwrap());
5993
5994 let initial_ts = "2026-01-01T00:00:00Z";
5995 store
5996 .kv_set("mentions_last_checked", initial_ts)
5997 .await
5998 .unwrap();
5999
6000 let comment = Mention {
6002 id: "comment-jkl".into(),
6003 body: "/unblock".into(),
6004 author: "alice".into(),
6005 created_at: "2026-01-02T00:00:00Z".into(),
6006 issue_url: Some("https://api.github.com/repos/owner/repo/issues/42".into()),
6007 };
6008
6009 let backend_trait: Arc<dyn ExternalBackend> = Arc::<AckTrackingBackend>::clone(&backend);
6010 let task_manager = Arc::new(TaskManager::new(Arc::clone(&backend_trait)));
6011
6012 scan_comments(
6013 &backend_trait,
6014 Some(&store),
6015 "owner/repo",
6016 &task_manager,
6017 Some(&[comment]),
6018 )
6019 .await
6020 .unwrap();
6021
6022 assert!(
6023 backend.posted.lock().await.is_empty(),
6024 "nothing is posted when the issue fetch fails"
6025 );
6026 assert_eq!(
6027 store
6028 .kv_get("mentions_last_checked")
6029 .await
6030 .unwrap()
6031 .as_deref(),
6032 Some(initial_ts),
6033 "cursor must not advance past a command that never ran"
6034 );
6035 }
6036
6037 fn make_task(block_reason: Option<String>) -> Task {
6040 Task {
6041 id: 1,
6042 external_id: None,
6043 repo: "owner/repo".to_string(),
6044 origin: String::new(),
6045 title: String::new(),
6046 body: String::new(),
6047 status: TaskStatus::Blocked,
6048 source: String::new(),
6049 source_id: String::new(),
6050 author: String::new(),
6051 url: String::new(),
6052 labels: vec![],
6053 agent: None,
6054 model: None,
6055 complexity: String::new(),
6056 estimate: 0,
6057 route_reason: String::new(),
6058 agent_profile: String::new(),
6059 selected_skills: String::new(),
6060 route_attempts: 0,
6061 attempts: 0,
6062 branch: String::new(),
6063 worktree: String::new(),
6064 worktree_cleaned: false,
6065 owner_node: None,
6066 summary: String::new(),
6067 last_error: String::new(),
6068 parent_id: None,
6069 block_reason,
6070 pr_number: Some(42),
6071 pr_review_context: String::new(),
6072 last_review_ts: String::new(),
6073 review_ts_map: String::new(),
6074 last_comment_review_ts: String::new(),
6075 merge_conflict_retries: 0,
6076 ci_merge_failures: 0,
6077 pr_create_failures: 0,
6078 push_failures: 0,
6079 network_retries: 0,
6080 timeout_retries: 0,
6081 review_agent_failures: 0,
6082 review_resets: 0,
6083 review_cycles: 0,
6084 review_invocations: 0,
6085 needs_review_refires: 0,
6086 input_tokens: 0,
6087 output_tokens: 0,
6088 total_cost_usd: 0.0,
6089 model_reroute_chain: String::new(),
6090 limit_reroute_chain: String::new(),
6091 memory: vec![],
6092 delegations: vec![],
6093 auto_unblock_count: 0,
6094 auto_unblock_last_at: String::new(),
6095 auto_unblock_last_reason: String::new(),
6096 ci_recovery_count: 0,
6097 no_code_reroutes: 0,
6098 no_code_last_agent: String::new(),
6099 created_at: String::new(),
6100 updated_at: String::new(),
6101 }
6102 }
6103
6104 #[serial(cooldown_state)]
6105 #[test]
6106 fn is_ci_failure_block_matches_ci_failure_limit() {
6107 let task = make_task(Some(
6108 "CI failure limit (3) reached during auto-merge".to_string(),
6109 ));
6110 assert!(is_ci_failure_block(&task));
6111 }
6112
6113 #[serial(cooldown_state)]
6114 #[test]
6115 fn is_ci_failure_block_matches_ci_checks_timeout() {
6116 let task = make_task(Some(
6117 "CI checks timed out after 3 auto-merge failures".to_string(),
6118 ));
6119 assert!(is_ci_failure_block(&task));
6120 }
6121
6122 #[serial(cooldown_state)]
6123 #[test]
6124 fn is_ci_failure_block_false_for_regular_block() {
6125 let task = make_task(Some("waiting on input".to_string()));
6126 assert!(!is_ci_failure_block(&task));
6127 }
6128
6129 #[serial(cooldown_state)]
6130 #[test]
6131 fn is_ci_failure_block_false_for_none_reason() {
6132 let task = make_task(None);
6133 assert!(!is_ci_failure_block(&task));
6134 }
6135
6136 #[serial(cooldown_state)]
6139 #[test]
6140 fn ci_failure_cooldown_zero_count_immediate() {
6141 let task = make_task(None);
6143 assert!(ci_failure_unblock_cooldown_elapsed(&task));
6144 }
6145
6146 #[serial(cooldown_state)]
6147 #[test]
6148 fn ci_failure_cooldown_empty_last_at_immediate() {
6149 let mut task = make_task(None);
6151 task.auto_unblock_count = 1;
6152 assert!(ci_failure_unblock_cooldown_elapsed(&task));
6153 }
6154
6155 #[serial(cooldown_state)]
6156 #[test]
6157 fn ci_failure_cooldown_invalid_timestamp_immediate() {
6158 let mut task = make_task(None);
6160 task.auto_unblock_count = 1;
6161 task.auto_unblock_last_at = "not-a-timestamp".to_string();
6162 assert!(ci_failure_unblock_cooldown_elapsed(&task));
6163 }
6164
6165 #[serial(cooldown_state)]
6166 #[test]
6167 fn ci_failure_cooldown_high_count_still_eventually_elapses() {
6168 let long_ago = (chrono::Utc::now() - chrono::Duration::days(30)).to_rfc3339();
6171 for count in [3i32, 5, 99] {
6172 let mut task = make_task(None);
6173 task.auto_unblock_count = count;
6174 task.auto_unblock_last_at = long_ago.clone();
6175 assert!(
6176 ci_failure_unblock_cooldown_elapsed(&task),
6177 "count={count} should eventually elapse"
6178 );
6179 }
6180 }
6181
6182 #[serial(cooldown_state)]
6183 #[test]
6184 fn ci_failure_cooldown_recent_timestamp_still_blocks() {
6185 let just_now = chrono::Utc::now().to_rfc3339();
6187 for count in [1i32, 2, 3, 99] {
6188 let mut task = make_task(None);
6189 task.auto_unblock_count = count;
6190 task.auto_unblock_last_at = just_now.clone();
6191 assert!(
6192 !ci_failure_unblock_cooldown_elapsed(&task),
6193 "count={count} should still be cooling down"
6194 );
6195 }
6196 }
6197
6198 #[serial(cooldown_state)]
6201 #[tokio::test]
6202 async fn global_ci_failure_sweep_returns_ok_with_no_blocked_tasks() {
6203 cooldown::reset_global_state().await;
6204 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6205 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6206 let task_manager = Arc::new(TaskManager::with_store(
6207 backend,
6208 Arc::clone(&store),
6209 "owner/repo".to_string(),
6210 ));
6211 auto_unblock_ci_failure_blocked_tasks_global(
6212 &task_manager,
6213 &store,
6214 &Arc::new(DashSet::new()),
6215 )
6216 .await
6217 .unwrap();
6218 }
6219
6220 #[serial(cooldown_state)]
6221 #[tokio::test]
6222 async fn global_ci_failure_sweep_skips_tasks_in_cooldown() {
6223 cooldown::reset_global_state().await;
6226 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6227 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6228 let task_manager = Arc::new(TaskManager::with_store(
6229 backend,
6230 Arc::clone(&store),
6231 "active/repo".to_string(),
6232 ));
6233
6234 let id = store
6236 .upsert_external(&crate::store::UpsertExternal {
6237 repo: "inactive/repo",
6238 ext_id: "10",
6239 title: "Old blocked task",
6240 body: "",
6241 author: "",
6242 url: "",
6243 labels: &[],
6244 origin: "github",
6245 })
6246 .await
6247 .unwrap();
6248 store.update_status(id, TaskStatus::Blocked).await.unwrap();
6249 let just_now = chrono::Utc::now().to_rfc3339();
6250 store
6251 .set_fields(
6252 id,
6253 &[
6254 (
6255 "block_reason",
6256 serde_json::json!("CI failure limit reached during auto-merge"),
6257 ),
6258 ("pr_number", serde_json::json!(99)),
6259 ("auto_unblock_count", serde_json::json!(1)),
6260 ("auto_unblock_last_at", serde_json::json!(just_now)),
6261 ],
6262 )
6263 .await
6264 .unwrap();
6265
6266 auto_unblock_ci_failure_blocked_tasks_global(
6267 &task_manager,
6268 &store,
6269 &Arc::new(DashSet::new()),
6270 )
6271 .await
6272 .unwrap();
6273
6274 let task = store.get(id).await.unwrap();
6276 assert_eq!(
6277 task.auto_unblock_count, 1,
6278 "task in cooldown must not be processed"
6279 );
6280 }
6281
6282 #[serial(cooldown_state)]
6283 #[tokio::test]
6284 async fn global_ci_failure_sweep_ignores_non_ci_failure_blocks() {
6285 cooldown::reset_global_state().await;
6286 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6287 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6288 let task_manager = Arc::new(TaskManager::with_store(
6289 backend,
6290 Arc::clone(&store),
6291 "active/repo".to_string(),
6292 ));
6293
6294 let id = store
6296 .upsert_external(&crate::store::UpsertExternal {
6297 repo: "inactive/repo",
6298 ext_id: "20",
6299 title: "Manual block",
6300 body: "",
6301 author: "",
6302 url: "",
6303 labels: &[],
6304 origin: "github",
6305 })
6306 .await
6307 .unwrap();
6308 store.update_status(id, TaskStatus::Blocked).await.unwrap();
6309 store
6310 .set_fields(
6311 id,
6312 &[("block_reason", serde_json::json!("waiting for human input"))],
6313 )
6314 .await
6315 .unwrap();
6316
6317 auto_unblock_ci_failure_blocked_tasks_global(
6318 &task_manager,
6319 &store,
6320 &Arc::new(DashSet::new()),
6321 )
6322 .await
6323 .unwrap();
6324
6325 let task = store.get(id).await.unwrap();
6327 assert_eq!(task.status, TaskStatus::Blocked);
6328 }
6329
6330 #[serial(cooldown_state)]
6333 #[tokio::test]
6334 async fn global_rebroadcast_sweep_returns_ok_with_no_blocked_tasks() {
6335 cooldown::reset_global_state().await;
6336 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6337 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6338 let task_manager = Arc::new(TaskManager::with_store(
6339 backend,
6340 Arc::clone(&store),
6341 "owner/repo".to_string(),
6342 ));
6343 let router = router_with_healthy_review_agent();
6344 auto_recover_rebroadcast_blocked_tasks_global(
6345 &router,
6346 &task_manager,
6347 &store,
6348 &sweep_repos(),
6349 )
6350 .await
6351 .unwrap();
6352 }
6353
6354 #[serial(cooldown_state)]
6355 #[tokio::test]
6356 async fn global_rebroadcast_sweep_ignores_non_rebroadcast_blocks() {
6357 cooldown::reset_global_state().await;
6358 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6359 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6360 let task_manager = Arc::new(TaskManager::with_store(
6361 backend,
6362 Arc::clone(&store),
6363 "active/repo".to_string(),
6364 ));
6365 let router = router_with_healthy_review_agent();
6366
6367 let id = store
6368 .upsert_external(&crate::store::UpsertExternal {
6369 repo: "inactive/repo",
6370 ext_id: "30",
6371 title: "Manual block",
6372 body: "",
6373 author: "",
6374 url: "",
6375 labels: &[],
6376 origin: "github",
6377 })
6378 .await
6379 .unwrap();
6380 store.update_status(id, TaskStatus::Blocked).await.unwrap();
6381 store
6382 .set_fields(
6383 id,
6384 &[("block_reason", serde_json::json!("waiting for human input"))],
6385 )
6386 .await
6387 .unwrap();
6388 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
6389 .bind(id)
6390 .execute(store.pool())
6391 .await
6392 .unwrap();
6393
6394 auto_recover_rebroadcast_blocked_tasks_global(
6395 &router,
6396 &task_manager,
6397 &store,
6398 &sweep_repos(),
6399 )
6400 .await
6401 .unwrap();
6402
6403 let task = store.get(id).await.unwrap();
6404 assert_eq!(task.status, TaskStatus::Blocked);
6405 assert_eq!(
6406 task.block_reason.as_deref(),
6407 Some("waiting for human input")
6408 );
6409 }
6410
6411 #[serial(cooldown_state)]
6412 #[tokio::test]
6413 async fn global_rebroadcast_sweep_skips_freshly_blocked_tasks() {
6414 cooldown::reset_global_state().await;
6415 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6416 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6417 let task_manager = Arc::new(TaskManager::with_store(
6418 backend,
6419 Arc::clone(&store),
6420 "active/repo".to_string(),
6421 ));
6422 let router = router_with_healthy_review_agent();
6423
6424 let id = store
6425 .upsert_external(&crate::store::UpsertExternal {
6426 repo: "inactive/repo",
6427 ext_id: "40",
6428 title: "Just escalated",
6429 body: "",
6430 author: "",
6431 url: "",
6432 labels: &[],
6433 origin: "github",
6434 })
6435 .await
6436 .unwrap();
6437 store.update_status(id, TaskStatus::Blocked).await.unwrap();
6438 store
6439 .set_fields(
6440 id,
6441 &[("block_reason", serde_json::json!(REBROADCAST_BLOCK_REASON))],
6442 )
6443 .await
6444 .unwrap();
6445 auto_recover_rebroadcast_blocked_tasks_global(
6449 &router,
6450 &task_manager,
6451 &store,
6452 &sweep_repos(),
6453 )
6454 .await
6455 .unwrap();
6456
6457 let task = store.get(id).await.unwrap();
6458 assert_eq!(task.status, TaskStatus::Blocked);
6459 }
6460
6461 #[serial(cooldown_state)]
6466 #[tokio::test]
6467 async fn global_rebroadcast_sweep_recovers_task_from_inactive_repo() {
6468 cooldown::reset_global_state().await;
6469 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6470 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6471 let task_manager = Arc::new(TaskManager::with_store(
6472 backend,
6473 Arc::clone(&store),
6474 "active/repo".to_string(),
6475 ));
6476 let router = router_with_healthy_review_agent();
6477
6478 let id = store
6479 .upsert_external(&crate::store::UpsertExternal {
6480 repo: "inactive/repo",
6481 ext_id: "458",
6482 title: "Rebroadcast blocked from inactive repo",
6483 body: "",
6484 author: "",
6485 url: "",
6486 labels: &[],
6487 origin: "github",
6488 })
6489 .await
6490 .unwrap();
6491 store.update_status(id, TaskStatus::Blocked).await.unwrap();
6492 store
6493 .set_fields(
6494 id,
6495 &[
6496 ("block_reason", serde_json::json!(REBROADCAST_BLOCK_REASON)),
6497 ("needs_review_refires", serde_json::json!(6)),
6498 ],
6499 )
6500 .await
6501 .unwrap();
6502 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
6503 .bind(id)
6504 .execute(store.pool())
6505 .await
6506 .unwrap();
6507
6508 auto_recover_rebroadcast_blocked_tasks_global(
6509 &router,
6510 &task_manager,
6511 &store,
6512 &sweep_repos(),
6513 )
6514 .await
6515 .unwrap();
6516
6517 let task = store.get(id).await.unwrap();
6518 assert_eq!(
6519 task.status,
6520 TaskStatus::NeedsReview,
6521 "cross-repo rebroadcast-blocked task must be recovered to needs_review"
6522 );
6523 assert!(
6524 task.block_reason.is_none(),
6525 "block_reason must be cleared on recovery"
6526 );
6527 assert_eq!(
6528 task.needs_review_refires, 0,
6529 "needs_review_refires must be reset to 0 on recovery"
6530 );
6531 }
6532
6533 #[serial(cooldown_state)]
6536 #[tokio::test]
6537 async fn global_rebroadcast_sweep_skips_reset_without_routable_review_agent() {
6538 cooldown::reset_global_state().await;
6539 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6540 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6541 let task_manager = Arc::new(TaskManager::with_store(
6542 backend,
6543 Arc::clone(&store),
6544 "active/repo".to_string(),
6545 ));
6546 let router = Router::new_for_test(RouterConfig::default(), vec!["claude".to_string()]);
6549
6550 let id = store
6551 .upsert_external(&crate::store::UpsertExternal {
6552 repo: "inactive/repo",
6553 ext_id: "459",
6554 title: "Rebroadcast blocked, no routable agents",
6555 body: "",
6556 author: "",
6557 url: "",
6558 labels: &[],
6559 origin: "github",
6560 })
6561 .await
6562 .unwrap();
6563 store.update_status(id, TaskStatus::Blocked).await.unwrap();
6564 store
6565 .set_fields(
6566 id,
6567 &[("block_reason", serde_json::json!(REBROADCAST_BLOCK_REASON))],
6568 )
6569 .await
6570 .unwrap();
6571 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
6572 .bind(id)
6573 .execute(store.pool())
6574 .await
6575 .unwrap();
6576
6577 auto_recover_rebroadcast_blocked_tasks_global(
6578 &router,
6579 &task_manager,
6580 &store,
6581 &sweep_repos(),
6582 )
6583 .await
6584 .unwrap();
6585
6586 let task = store.get(id).await.unwrap();
6587 assert_eq!(task.status, TaskStatus::Blocked);
6588 assert_eq!(task.block_reason.as_deref(), Some(REBROADCAST_BLOCK_REASON));
6589 }
6590}