1use super::EngineConfig;
13use super::ci_bootstrap;
14use super::cleanup::{check_merged_prs, cleanup_done_worktrees, prune_old_run_output};
15use super::commands::{CommandOutcome, CommandStoreOps, parse_command, validate_and_run_command};
16use super::review_poll::review_open_prs;
17use crate::backends::ExternalId;
18use crate::backends::ExternalTask;
19use crate::backends::{ExternalBackend, Mention, Status};
20use crate::cmd::CommandErrorContext;
21use crate::config;
22use crate::engine;
23use crate::engine::auto_merge::{is_pr_behind, required_checks_state};
24use crate::engine::cleanup::cleanup_task_worktree;
25use crate::engine::commands::CommandGh;
26use crate::engine::commands::OwnerCommand;
27use crate::engine::cooldown;
28use crate::engine::cooldown::{
29 cooldown_reason, github_circuit_remaining_secs, is_agent_in_cooldown, is_github_circuit_open,
30 is_model_in_cooldown, refresh_degraded_agents,
31};
32use crate::engine::dependencies;
33use crate::engine::router::Router;
34use crate::engine::router::RouterConfig;
35use crate::engine::router::config as router_config;
36use crate::engine::runner::agents::patterns;
37use crate::engine::runner::git_ops;
38use crate::engine::tasks;
39use crate::engine::tasks::{TaskManager, status_to_task_status};
40use crate::github::http::GhHttp;
41use crate::github::types::extract_issue_number_from_url;
42use crate::home;
43use crate::store;
44use crate::store::Task;
45use crate::store::TaskRun;
46use crate::store::TaskStatus;
47use crate::store::TaskStore;
48use crate::tmux::TmuxManager;
49use dashmap::{DashMap, DashSet};
50use futures::future;
51use std::collections::HashMap;
52use std::collections::HashSet;
53use std::path::PathBuf;
54use std::sync::Arc;
55use std::time::Duration;
56use std::time::Instant;
57use tokio::fs;
58use tokio::sync::OnceCell;
59use tokio::sync::RwLock;
60use tokio::time;
61
62async fn kv_get_prefer_store(store: Option<&Arc<TaskStore>>, key: &str) -> Option<String> {
68 match try_kv_get_prefer_store(store, key).await {
69 Ok(opt) => opt,
70 Err(e) => {
71 tracing::error!(key, err = %e, "kv_get failed");
72 None
73 }
74 }
75}
76
77async fn try_kv_get_prefer_store(
83 store: Option<&Arc<TaskStore>>,
84 key: &str,
85) -> anyhow::Result<Option<String>> {
86 if let Some(s) = store {
87 Ok(s.kv_get(key).await?)
88 } else {
89 Ok(None)
90 }
91}
92
93async fn kv_set_prefer_store(store: Option<&Arc<TaskStore>>, key: &str, value: &str) {
99 if let Err(e) = try_kv_set_prefer_store(store, key, value).await {
100 tracing::error!(key, err = %e, "kv_set failed");
101 }
102}
103
104async fn try_kv_set_prefer_store(
110 store: Option<&Arc<TaskStore>>,
111 key: &str,
112 value: &str,
113) -> anyhow::Result<()> {
114 if let Some(s) = store {
115 s.kv_set(key, value).await?;
116 }
117 Ok(())
118}
119
120#[derive(Debug, Clone)]
121pub(crate) struct ReviewTaskSnapshot {
122 pub external: ExternalTask,
123 pub stored: Task,
124}
125
126async fn prefetch_review_tasks(
127 store: &Arc<TaskStore>,
128 repo: &str,
129) -> anyhow::Result<(
130 Vec<ReviewTaskSnapshot>,
131 Vec<ReviewTaskSnapshot>,
132 Vec<ReviewTaskSnapshot>,
133)> {
134 let mut in_review = store.list_by_status(repo, TaskStatus::InReview).await?;
136 in_review.extend(store.list_by_status(repo, TaskStatus::Approved).await?);
137 let in_review = in_review
138 .into_iter()
139 .map(|stored| ReviewTaskSnapshot {
140 external: tasks::store_task_to_external(&stored),
141 stored,
142 })
143 .collect();
144 let needs_review = store
145 .list_by_status(repo, TaskStatus::NeedsReview)
146 .await?
147 .into_iter()
148 .map(|stored| ReviewTaskSnapshot {
149 external: tasks::store_task_to_external(&stored),
150 stored,
151 })
152 .collect();
153
154 let blocked_candidates = store
158 .list_by_status(repo, TaskStatus::Blocked)
159 .await?
160 .into_iter()
161 .filter(|stored| stored.pr_number.is_some() || !stored.branch.is_empty())
162 .map(|stored| ReviewTaskSnapshot {
163 external: tasks::store_task_to_external(&stored),
164 stored,
165 })
166 .collect();
167
168 Ok((in_review, needs_review, blocked_candidates))
169}
170
171async fn fetch_comments_since(
172 backend: &Arc<dyn ExternalBackend>,
173 since: &str,
174) -> anyhow::Result<Vec<Mention>> {
175 backend.get_mentions(since).await
176}
177
178fn filter_mentions_by_since(comments: &[Mention], since: &str) -> Vec<Mention> {
179 comments
180 .iter()
181 .filter(|comment| comment.created_at.as_str() > since)
182 .cloned()
183 .collect()
184}
185
186#[derive(Debug, Clone, Copy, PartialEq, Eq)]
187enum FailureCategory {
188 FalseFailure,
189 RateLimit,
190 CreditExhausted,
191 SilentExit0,
192 Timeout,
193 ConnectionError,
194 CliFlagError,
195 AllAgentsExhausted,
196 ParseError,
197 MaxAttempts,
198 PushFailed,
199 PrCreateFailed,
200 ModelUnavailable,
201 Unknown,
202}
203
204impl FailureCategory {
205 fn is_recoverable(self) -> bool {
212 matches!(
213 self,
214 FailureCategory::FalseFailure
215 | FailureCategory::RateLimit
216 | FailureCategory::CreditExhausted
217 | FailureCategory::SilentExit0
218 | FailureCategory::Timeout
219 | FailureCategory::ConnectionError
220 | FailureCategory::CliFlagError
221 | FailureCategory::AllAgentsExhausted
222 | FailureCategory::ParseError
223 | FailureCategory::ModelUnavailable
224 )
225 }
226
227 fn as_str(self) -> &'static str {
228 match self {
229 FailureCategory::FalseFailure => "FalseFailure",
230 FailureCategory::RateLimit => "RateLimit",
231 FailureCategory::CreditExhausted => "CreditExhausted",
232 FailureCategory::SilentExit0 => "SilentExit0",
233 FailureCategory::Timeout => "Timeout",
234 FailureCategory::ConnectionError => "ConnectionError",
235 FailureCategory::CliFlagError => "CliFlagError",
236 FailureCategory::AllAgentsExhausted => "AllAgentsExhausted",
237 FailureCategory::ParseError => "ParseError",
238 FailureCategory::MaxAttempts => "MaxAttempts",
239 FailureCategory::PushFailed => "PushFailed",
240 FailureCategory::PrCreateFailed => "PrCreateFailed",
241 FailureCategory::ModelUnavailable => "ModelUnavailable",
242 FailureCategory::Unknown => "Unknown",
243 }
244 }
245}
246
247fn classify_failure(error: &str, outcome: &str) -> FailureCategory {
248 let lower = error.to_lowercase();
249
250 if outcome == "timeout" || lower.contains("timeout") {
251 return FailureCategory::Timeout;
252 }
253
254 if outcome == "billing_cycle_exhausted"
265 || outcome == "credit_exhausted"
266 || outcome == "org_level_disabled"
267 || cooldown::detect_credit_exhaustion(&lower).is_some()
268 {
269 return FailureCategory::CreditExhausted;
270 }
271
272 if outcome == "rate_limit" || lower.contains("rate limit") || lower.contains("usage limit") {
273 return FailureCategory::RateLimit;
274 }
275
276 if error.trim().is_empty() {
277 return FailureCategory::FalseFailure;
278 }
279
280 if lower.contains("all agents exhausted") {
281 return FailureCategory::AllAgentsExhausted;
282 }
283
284 if lower.contains("max attempts") || lower.contains("exceeded max attempts") {
285 return FailureCategory::MaxAttempts;
286 }
287
288 if lower.contains("output-format=stream-json") && lower.contains("requires --verbose") {
289 return FailureCategory::CliFlagError;
290 }
291
292 if lower.contains("model unavailable")
293 || lower.contains("model not found")
294 || (lower.contains("does not exist") && lower.contains("model"))
295 {
296 return FailureCategory::ModelUnavailable;
297 }
298
299 if lower.contains("push failed") {
300 return FailureCategory::PushFailed;
301 }
302
303 if lower.contains("create pr failed")
304 || lower.contains("failed to create pull request")
305 || lower.contains("pull request creation failed")
306 {
307 return FailureCategory::PrCreateFailed;
308 }
309
310 if lower.contains("exit 0") {
311 return FailureCategory::SilentExit0;
312 }
313
314 if outcome == "parse_error"
315 || outcome == "truncated"
316 || lower.contains("invalid response")
317 || lower.contains("parse error")
318 || lower.contains("truncated")
319 {
320 return FailureCategory::ParseError;
321 }
322
323 if patterns::detect_network_error(&lower).is_some() {
324 return FailureCategory::ConnectionError;
325 }
326
327 FailureCategory::Unknown
328}
329
330fn classify_failure_from_run(run: &TaskRun) -> Option<FailureCategory> {
331 if run.outcome == "success" && run.error.trim().is_empty() {
333 return None;
334 }
335
336 if run.outcome.trim().is_empty()
343 && run.error.trim().is_empty()
344 && run.exit_code.is_none()
345 && run.stdout.trim().is_empty()
346 && run.stderr.trim().is_empty()
347 {
348 return None;
349 }
350
351 let has_output = !run.stdout.trim().is_empty() || !run.stderr.trim().is_empty();
352 if run.exit_code == Some(0) && !has_output && run.outcome != "success" {
353 return Some(FailureCategory::SilentExit0);
354 }
355
356 let mut category = classify_failure(&run.error, &run.outcome);
357
358 if category == FailureCategory::FalseFailure {
367 let has_nonzero_exit = run.exit_code.is_some() && run.exit_code != Some(0);
368 let has_output = !run.stdout.trim().is_empty() || !run.stderr.trim().is_empty();
369 if has_nonzero_exit || has_output {
370 category = FailureCategory::Unknown;
371 }
372 }
373
374 Some(category)
375}
376
377async fn resolve_mention_parent_id(
387 store: &TaskStore,
388 repo: &str,
389 issue_num: &str,
390 is_pr: bool,
391) -> anyhow::Result<Option<i64>> {
392 if is_pr {
393 if let Ok(pr_num) = issue_num.parse::<i32>() {
394 store.resolve_id_by_pr_number(repo, pr_num).await
395 } else {
396 Ok(None)
397 }
398 } else {
399 store.resolve_id_by_external(repo, issue_num).await
400 }
401}
402
403async fn record_mention_task(
405 store: &TaskStore,
406 repo: &str,
407 mention: &Mention,
408 title: &str,
409 body: &str,
410 parent_id: Option<i64>,
411 cursor: &mut MentionCursor,
412) -> bool {
413 match store
414 .create_internal(repo, title, body, "mention", &mention.id, parent_id)
415 .await
416 {
417 Ok(task_id) => {
418 tracing::info!(task_id, mention_id = %mention.id, "created mention task");
419 cursor.advance(&mention.created_at);
420 true
421 }
422 Err(e) => {
423 tracing::warn!(mention_id = %mention.id, err = %e, "failed to create mention task");
424 false
425 }
426 }
427}
428
429fn advance_cursor(last_safe_ts: &mut Option<String>, ts: &str) {
431 if last_safe_ts.as_deref() < Some(ts) {
432 *last_safe_ts = Some(ts.to_string());
433 }
434}
435
436#[derive(Debug, Default)]
437struct MentionCursor {
438 last_safe_ts: Option<String>,
439 blocked_by_gap: bool,
440}
441
442impl MentionCursor {
443 fn advance(&mut self, ts: &str) {
444 if self.blocked_by_gap {
445 return;
446 }
447 advance_cursor(&mut self.last_safe_ts, ts);
448 }
449
450 fn block_on_gap(&mut self) {
451 self.blocked_by_gap = true;
452 }
453
454 fn into_last_safe_ts(self) -> Option<String> {
455 self.last_safe_ts
456 }
457}
458
459#[derive(Debug, PartialEq)]
461pub(crate) enum CommentAction {
462 Skip,
464 ExecuteCommand {
466 command: OwnerCommand,
467 issue_num: String,
468 },
469 ExecuteCommandForMention {
471 command: OwnerCommand,
472 issue_num: String,
473 },
474 CreateMentionTask { issue_num: Option<String> },
476}
477
478pub(crate) fn classify_comment(
483 body: &str,
484 issue_url: Option<&str>,
485 current_user: &str,
486 already_processed: bool,
487) -> CommentAction {
488 if already_processed {
489 return CommentAction::Skip;
490 }
491
492 let is_mention =
493 body.contains(current_user) || body.contains("@orch") || body.contains("@orchestrator");
494 let command = parse_command(body);
495 let issue_num = issue_url.and_then(extract_issue_number_from_url);
496
497 match (command, is_mention, issue_num) {
498 (Some(cmd), true, Some(num)) => CommentAction::ExecuteCommandForMention {
499 command: cmd,
500 issue_num: num,
501 },
502 (Some(cmd), false, Some(num)) => CommentAction::ExecuteCommand {
503 command: cmd,
504 issue_num: num,
505 },
506 (Some(_), true, None) => CommentAction::CreateMentionTask { issue_num: None },
508 (None, true, issue_num) => CommentAction::CreateMentionTask { issue_num },
510 (Some(_), false, None) | (None, false, _) => CommentAction::Skip,
512 }
513}
514
515#[expect(
528 clippy::too_many_arguments,
529 reason = "the sync tick hands over its shared state piece by piece"
530)]
531async fn handle_slash_command(
532 backend: &Arc<dyn ExternalBackend>,
533 store: Option<&Arc<TaskStore>>,
534 repo: &str,
535 task_manager: &Arc<TaskManager>,
536 gh: &dyn CommandGh,
537 mention: &Mention,
538 command: &OwnerCommand,
539 cursor: &mut MentionCursor,
540) -> anyhow::Result<bool> {
541 let Some(issue_num) = mention
542 .issue_url
543 .as_deref()
544 .and_then(extract_issue_number_from_url)
545 else {
546 tracing::warn!(comment_id = %mention.id, "slash command without valid issue number");
547 return Ok(true);
548 };
549
550 let store_opt: Option<Arc<dyn CommandStoreOps>> =
551 store.map(|s| Arc::clone(s) as Arc<dyn CommandStoreOps>);
552 let outcome = validate_and_run_command(
553 backend,
554 gh,
555 repo,
556 &issue_num,
557 command,
558 &mention.author,
559 &store_opt,
560 task_manager,
561 )
562 .await;
563
564 let is_pr = match &outcome {
566 CommandOutcome::Executed { is_pr }
567 | CommandOutcome::NotOpen { is_pr }
568 | CommandOutcome::NotCollaborator { is_pr } => *is_pr,
569 CommandOutcome::FetchFailed | CommandOutcome::CollaboratorCheckFailed => {
570 tracing::warn!(
573 comment_id = %mention.id,
574 command = %command,
575 "deferring @orch slash command after a transient GitHub error"
576 );
577 return Ok(false);
578 }
579 };
580
581 if let Some(s) = store {
584 let parent_id = match resolve_mention_parent_id(s, repo, &issue_num, is_pr).await {
585 Ok(parent_id) => parent_id,
586 Err(e) => {
587 tracing::warn!(
588 comment_id = %mention.id,
589 err = %e,
590 "deferring mention task due to parent lookup failure"
591 );
592 return Err(e);
593 }
594 };
595
596 let (title, body) = match &outcome {
597 CommandOutcome::NotOpen { .. } => (
598 format!(
599 "Skipped @orch command on closed #{issue_num} from @{}",
600 mention.author
601 ),
602 format!(
603 "Mention by @{}:\n\n{}\n\n**Skipped:** target is closed",
604 mention.author, mention.body
605 ),
606 ),
607 CommandOutcome::NotCollaborator { .. } => (
608 format!(
609 "Skipped @orch command on #{issue_num} from non-collaborator @{}",
610 mention.author
611 ),
612 format!(
613 "Mention by @{}:\n\n{}\n\n**Skipped:** author is not a collaborator",
614 mention.author, mention.body
615 ),
616 ),
617 CommandOutcome::Executed { .. } => (
618 format!(
619 "Respond to mention by @{} on task #{issue_num} — command executed",
620 mention.author
621 ),
622 format!(
623 "Mention by @{} on #{issue_num}:\n\n{}\n\n**Status:** Command executed",
624 mention.author, mention.body
625 ),
626 ),
627 CommandOutcome::FetchFailed | CommandOutcome::CollaboratorCheckFailed => {
628 tracing::debug!("skipping mention task record due to earlier command failure");
629 return Ok(true);
630 }
631 };
632
633 let created = record_mention_task(s, repo, mention, &title, &body, parent_id, cursor).await;
634 if !created {
635 return Ok(false);
636 }
637 }
638 Ok(true)
639}
640
641fn auto_unblock_cooldown_elapsed(count: i32, last_at: &str) -> bool {
642 if count == 0 {
643 return true;
644 }
645
646 let required = match count {
647 1 => chrono::Duration::minutes(30),
648 2 => chrono::Duration::hours(2),
649 _ => return false,
650 };
651
652 if last_at.trim().is_empty() {
653 return true;
654 }
655
656 match chrono::DateTime::parse_from_rfc3339(last_at) {
657 Ok(ts) => chrono::Utc::now() - ts.with_timezone(&chrono::Utc) >= required,
658 Err(_) => true,
659 }
660}
661
662fn is_ci_failure_block(task: &Task) -> bool {
663 let reason = task.block_reason.as_deref().unwrap_or("");
664 reason.contains("CI failure limit") || reason.contains("CI checks timed out")
665}
666
667fn ci_failure_unblock_cooldown_elapsed(task: &Task) -> bool {
668 if task.auto_unblock_count == 0 {
669 return true;
670 }
671
672 let required = match task.auto_unblock_count {
676 1 => chrono::Duration::minutes(10),
677 2 => chrono::Duration::hours(1),
678 3 => chrono::Duration::hours(6),
679 _ => chrono::Duration::hours(24),
680 };
681
682 let last_at = &task.auto_unblock_last_at;
683 if last_at.trim().is_empty() {
684 return true;
685 }
686
687 match chrono::DateTime::parse_from_rfc3339(last_at) {
688 Ok(ts) => chrono::Utc::now() - ts.with_timezone(&chrono::Utc) >= required,
689 Err(_) => true,
690 }
691}
692
693async fn try_unblock_ci_failure_task(
694 gh: &GhHttp,
695 repo: &str,
696 task_manager: &Arc<TaskManager>,
697 store: &Arc<TaskStore>,
698 task: &Task,
699 auto_merge_in_flight: &Arc<DashSet<String>>,
700) -> anyhow::Result<bool> {
701 let Some(pr_number) = task.pr_number.and_then(|n| u64::try_from(n).ok()) else {
702 tracing::debug!(
703 task_id = task.id,
704 "CI failure blocked task has no PR number"
705 );
706 return Ok(false);
707 };
708
709 let pr = match gh.get_pr(repo, pr_number).await {
710 Ok(p) => p,
711 Err(e) => {
712 tracing::warn!(task_id = task.id, pr_number, err = %e, "failed to get PR status for CI failure block");
713 return Ok(false);
714 }
715 };
716
717 tracing::debug!(
718 task_id = task.id,
719 pr_number,
720 merged = pr.merged,
721 state = ?pr.state,
722 "verifying PR state for CI failure blocked task"
723 );
724
725 if pr.merged.unwrap_or(false) || pr.state.eq_ignore_ascii_case("closed") {
726 tracing::info!(
727 task_id = task.id,
728 pr_number,
729 merged = pr.merged,
730 state = %pr.state,
731 "PR merged/closed — unblocking CI failure blocked task"
732 );
733 let ext_id = task
734 .external_id
735 .clone()
736 .unwrap_or_else(|| format!("internal:{}", task.id));
737 task_manager
741 .mark_task_done_by_store_id(task.id, &task.repo, &ExternalId(ext_id))
742 .await?;
743 let fields = vec![
744 ("block_reason", serde_json::Value::Null),
745 ("auto_unblock_count", serde_json::json!(0)),
746 ("auto_unblock_last_at", serde_json::json!("")),
747 ("auto_unblock_last_reason", serde_json::json!("")),
748 ];
749 store.set_fields(task.id, &fields).await?;
750 return Ok(true);
751 }
752
753 let task_id_str = task
757 .external_id
758 .clone()
759 .unwrap_or_else(|| format!("internal:{}", task.id));
760 if auto_merge_in_flight.contains(&task_id_str) {
761 tracing::debug!(
762 task_id = task.id,
763 pr_number,
764 "CI-failure recovery merge already in flight — skipping"
765 );
766 return Ok(false);
767 }
768
769 let branch_update_result = gh.update_pr_branch(repo, pr_number).await;
775 match branch_update_result {
776 Ok(()) => {
777 tracing::info!(
778 task_id = task.id,
779 pr_number,
780 "updated CI-failure-blocked PR branch against base to retrigger CI"
781 );
782 }
783 Err(ref e) => {
784 tracing::warn!(
786 task_id = task.id,
787 pr_number,
788 err = %e,
789 outcome = "branch_update_failed",
790 "failed to update CI-failure-blocked PR branch (may already be up to date)"
791 );
792 }
793 }
794
795 if let Err(e) = gh.enable_auto_merge(repo, pr_number).await {
796 tracing::warn!(task_id = task.id, pr_number, err = %e, "failed to re-enable auto-merge after branch update");
797 }
798
799 let pr_state = pr.state.as_str();
802 let new_reason = if let Err(ref e) = branch_update_result {
803 format!(
804 "CI failure limit reached during auto-merge (PR #{pr_number} still open, state: {pr_state}, branch update failed: {e})"
805 )
806 } else {
807 format!(
808 "CI failure limit reached during auto-merge (PR #{pr_number} still open, state: {pr_state})"
809 )
810 };
811 let now = chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string();
812 let new_count = task.auto_unblock_count.saturating_add(1);
815 let fields = vec![
816 ("block_reason", serde_json::json!(&new_reason)),
817 ("auto_unblock_count", serde_json::json!(new_count)),
818 ("auto_unblock_last_at", serde_json::json!(&now)),
819 ];
820 store.set_fields_silent(task.id, &fields).await?;
824
825 auto_merge_in_flight.insert(task_id_str.clone());
829 let gh_clone = gh.clone();
830 let repo_string = repo.to_string();
831 let task_manager_clone = Arc::clone(task_manager);
832 let store_clone = Arc::clone(store);
833 let task_clone = task.clone();
834 let in_flight_clone = Arc::clone(auto_merge_in_flight);
835
836 tokio::spawn(async move {
837 let _guard = scopeguard::guard((), |()| {
838 in_flight_clone.remove(&task_id_str);
839 });
840
841 match poll_and_merge_recovered_pr(&gh_clone, &repo_string, pr_number, task_clone.id).await {
842 Ok(true) => {
843 tracing::info!(
844 task_id = task_clone.id,
845 pr_number,
846 "CI-failure-blocked PR merged during recovery sweep"
847 );
848 let ext_id = task_clone
849 .external_id
850 .clone()
851 .unwrap_or_else(|| format!("internal:{}", task_clone.id));
852 let fields = [
853 ("block_reason", serde_json::Value::Null),
854 ("auto_unblock_count", serde_json::json!(0)),
855 ("auto_unblock_last_at", serde_json::json!("")),
856 ("auto_unblock_last_reason", serde_json::json!("")),
857 ];
858 if let Err(e) = task_manager_clone
859 .update_task_status_and_result_by_store_id(
860 task_clone.id,
861 &task_clone.repo,
862 &ExternalId(ext_id),
863 Status::Done,
864 &fields,
865 )
866 .await
867 {
868 tracing::warn!(
869 task_id = task_clone.id,
870 pr_number,
871 err = %e,
872 "failed to mark CI-failure-blocked task done after recovery merge"
873 );
874 }
875 if let Err(e) =
876 cleanup_task_worktree(&format!("{}", task_clone.id), &repo_string, &store_clone)
877 .await
878 {
879 tracing::warn!(
880 task_id = task_clone.id,
881 pr_number,
882 err = %e,
883 "cleanup after CI-failure recovery merge failed"
884 );
885 }
886 }
887 Ok(false) => {
888 tracing::debug!(
889 task_id = task_clone.id,
890 pr_number,
891 "CI-failure-blocked PR not merged this recovery tick — poll completed without merge"
892 );
893 }
894 Err(e) => {
895 tracing::warn!(
896 task_id = task_clone.id,
897 pr_number,
898 err = %e,
899 "error while polling CI-failure-blocked PR for recovery"
900 );
901 }
902 }
903 });
904
905 Ok(false)
906}
907
908async fn poll_and_merge_recovered_pr(
919 gh: &GhHttp,
920 repo: &str,
921 pr_number: u64,
922 task_id: i64,
923) -> anyhow::Result<bool> {
924 const MAX_BEHIND_RETRIES: u32 = 3;
928
929 let max_wait_secs: u64 = config::get("workflow.ci_poll_max_wait_secs")
930 .ok()
931 .and_then(|s| s.parse().ok())
932 .unwrap_or(600);
933 let base_interval_secs: u64 = config::get("workflow.ci_poll_interval_secs")
934 .ok()
935 .and_then(|s| s.parse().ok())
936 .unwrap_or(15);
937 let max_wait = Duration::from_secs(max_wait_secs);
938 let start = Instant::now();
939 let mut poll_count: u32 = 0;
940 let mut behind_retries: u32 = 0;
941
942 let initial_pr = gh.get_pr(repo, pr_number).await?;
943 let base_branch = initial_pr.base.ref_.clone();
944 let required_contexts = gh
945 .get_required_status_check_contexts(repo, &base_branch)
946 .await?;
947 let repo_has_workflows = if required_contexts.is_empty() {
948 gh.has_workflows(repo).await?
949 } else {
950 true
951 };
952
953 loop {
954 let pr = gh.get_pr(repo, pr_number).await?;
955 if pr.merged.unwrap_or(false) || pr.state.eq_ignore_ascii_case("closed") {
956 tracing::info!(
957 task_id = task_id,
958 pr_number,
959 "PR merged/closed during recovery poll"
960 );
961 return Ok(true);
962 }
963
964 if pr.mergeable == Some(false) {
965 tracing::info!(
966 task_id = task_id,
967 pr_number,
968 outcome = "poll_exited_without_merge",
969 reason = "merge_conflicts",
970 "PR has merge conflicts after recovery update — leaving for next recovery cycle"
971 );
972 return Ok(false);
973 }
974
975 if is_pr_behind(&pr) {
976 if behind_retries >= MAX_BEHIND_RETRIES {
977 tracing::info!(
978 task_id = task_id,
979 pr_number,
980 behind_retries,
981 max_behind_retries = MAX_BEHIND_RETRIES,
982 outcome = "poll_exited_without_merge",
983 reason = "behind_retry_exhausted",
984 "PR still behind base after recovery retries — leaving for next recovery cycle"
985 );
986 return Ok(false);
987 }
988 behind_retries += 1;
989 match gh.update_pr_branch(repo, pr_number).await {
990 Ok(()) => {
991 tracing::info!(
992 task_id = task_id,
993 pr_number,
994 behind_retries,
995 "PR was behind base — updated branch and continuing recovery poll"
996 );
997 continue;
998 }
999 Err(e) => {
1000 tracing::warn!(
1001 task_id = task_id,
1002 pr_number,
1003 behind_retries,
1004 err = %e,
1005 outcome = "branch_update_failed",
1006 "failed to update behind PR branch during recovery poll"
1007 );
1008 return Ok(false);
1009 }
1010 }
1011 }
1012
1013 let head_sha = pr.head.sha.clone();
1014 let (state, total, passing, failing, pending) = if required_contexts.is_empty() {
1015 gh.get_combined_status(
1016 repo,
1017 &head_sha,
1018 repo_has_workflows,
1019 pr.mergeable_state.as_deref(),
1020 )
1021 .await?
1022 } else {
1023 let check_runs = gh.get_check_runs(repo, &head_sha).await?;
1024 let statuses = gh.get_commit_status_contexts(repo, &head_sha).await?;
1025 required_checks_state(&required_contexts, &check_runs, &statuses)
1026 };
1027
1028 tracing::info!(
1029 task_id = task_id,
1030 pr_number,
1031 state = %state,
1032 total,
1033 passing,
1034 failing,
1035 pending,
1036 "CI recovery status check"
1037 );
1038
1039 match state.as_str() {
1040 "success" => {
1041 tracing::info!(
1042 task_id = task_id,
1043 pr_number,
1044 "CI green after recovery — attempting direct merge"
1045 );
1046 match gh.merge_pr(repo, pr_number, true, &head_sha).await {
1047 Ok(()) => {
1048 tracing::info!(
1049 task_id = task_id,
1050 pr_number,
1051 "direct merge succeeded after recovery"
1052 );
1053 return Ok(true);
1054 }
1055 Err(e) => {
1056 let err = e.to_string().to_ascii_lowercase();
1057 tracing::warn!(
1058 task_id = task_id,
1059 pr_number,
1060 err = %e,
1061 "direct merge failed after recovery"
1062 );
1063 if err.contains("already merged")
1066 || err.contains("no merge commit")
1067 || err.contains("pull request already merged")
1068 {
1069 return Ok(true);
1070 }
1071 if err.contains("head branch was modified")
1073 || err.contains("sha does not match")
1074 {
1075 tracing::info!(
1076 task_id = task_id,
1077 pr_number,
1078 outcome = "poll_exited_without_merge",
1079 reason = "head_moved",
1080 "merge rejected because the head sha changed"
1081 );
1082 return Ok(false);
1083 }
1084 if (err.contains("head branch is out of date")
1088 || err.contains("behind")
1089 || err.contains("409"))
1090 && behind_retries < MAX_BEHIND_RETRIES
1091 {
1092 behind_retries += 1;
1093 match gh.update_pr_branch(repo, pr_number).await {
1094 Ok(()) => {
1095 tracing::info!(
1096 task_id = task_id,
1097 pr_number,
1098 behind_retries,
1099 "merge rejected due to out-of-date branch — updated and continuing recovery poll"
1100 );
1101 continue;
1102 }
1103 Err(update_err) => {
1104 tracing::warn!(
1105 task_id = task_id,
1106 pr_number,
1107 behind_retries,
1108 err = %update_err,
1109 outcome = "branch_update_failed",
1110 "failed to update out-of-date PR branch during recovery poll"
1111 );
1112 }
1113 }
1114 }
1115 tracing::info!(
1116 task_id = task_id,
1117 pr_number,
1118 outcome = "poll_exited_without_merge",
1119 reason = "direct_merge_failed",
1120 "direct merge did not succeed — leaving for next recovery cycle"
1121 );
1122 return Ok(false);
1123 }
1124 }
1125 }
1126 "failure" => {
1127 tracing::info!(
1128 task_id = task_id,
1129 pr_number,
1130 outcome = "poll_exited_without_merge",
1131 reason = "ci_failure",
1132 "CI still failing after recovery — will retry later"
1133 );
1134 return Ok(false);
1135 }
1136 _ => {}
1137 }
1138
1139 if start.elapsed() >= max_wait {
1140 tracing::info!(
1141 task_id = task_id,
1142 pr_number,
1143 outcome = "poll_exited_without_merge",
1144 reason = "poll_timeout",
1145 "CI still pending after recovery poll timeout — will retry later"
1146 );
1147 return Ok(false);
1148 }
1149
1150 poll_count += 1;
1151 let multiplier = (2_u64).pow(poll_count.saturating_sub(1)).min(4);
1152 let sleep_secs = base_interval_secs.saturating_mul(multiplier);
1153 time::sleep(Duration::from_secs(sleep_secs)).await;
1154 }
1155}
1156
1157const REBROADCAST_BLOCK_REASON: &str = "review agent rebroadcast escalated after repeated retries";
1162
1163pub(crate) async fn auto_recover_rebroadcast_blocked_tasks_global(
1178 router: &Router,
1179 task_manager: &Arc<TaskManager>,
1180 store: &Arc<TaskStore>,
1181 active_repos: &HashSet<String>,
1182) -> anyhow::Result<()> {
1183 const MIN_BLOCK_AGE_MINUTES: i64 = 5;
1184
1185 let all_blocked = match store.list_all_by_status_global(TaskStatus::Blocked).await {
1186 Ok(tasks) => tasks,
1187 Err(e) => {
1188 tracing::warn!(err = %e, "global rebroadcast-blocked sweep: failed to list blocked tasks");
1189 return Ok(());
1190 }
1191 };
1192
1193 let now = chrono::Utc::now();
1194 let candidates: Vec<_> = all_blocked
1195 .into_iter()
1196 .filter(|t| {
1197 if t.block_reason.as_deref() != Some(REBROADCAST_BLOCK_REASON) {
1198 return false;
1199 }
1200 if !active_repos.contains(&t.repo) {
1202 return false;
1203 }
1204 match chrono::DateTime::parse_from_rfc3339(&t.updated_at) {
1206 Ok(updated_at) => {
1207 let age = now - updated_at.with_timezone(&chrono::Utc);
1208 age.num_minutes() >= MIN_BLOCK_AGE_MINUTES
1209 }
1210 Err(_) => true,
1211 }
1212 })
1213 .collect();
1214
1215 if candidates.is_empty() {
1216 return Ok(());
1217 }
1218
1219 let gh = match GhHttp::new() {
1220 Ok(g) => Some(g),
1221 Err(e) => {
1222 tracing::warn!(err = %e, "global rebroadcast-blocked sweep: failed to create GhHttp — skipping PR-state checks this tick");
1223 None
1224 }
1225 };
1226
1227 let any_routable =
1230 !router.available_agents.is_empty() && router.healthy_agent_count("review") > 0;
1231
1232 for task in candidates {
1233 let ext_id = task
1234 .external_id
1235 .clone()
1236 .unwrap_or_else(|| format!("internal:{}", task.id));
1237
1238 if let (Some(gh), Some(pr_number)) =
1239 (&gh, task.pr_number.and_then(|n| u64::try_from(n).ok()))
1240 {
1241 match gh.get_pr(&task.repo, pr_number).await {
1242 Ok(pr) => {
1243 if pr.merged.unwrap_or(false) {
1244 tracing::info!(
1245 task_id = task.id,
1246 repo = %task.repo,
1247 pr_number,
1248 "global rebroadcast-blocked sweep: PR already merged — marking task done"
1249 );
1250 if let Err(e) = task_manager
1251 .mark_task_done_by_store_id(
1252 task.id,
1253 &task.repo,
1254 &ExternalId(ext_id.clone()),
1255 )
1256 .await
1257 {
1258 tracing::warn!(task_id = task.id, repo = %task.repo, err = %e, "global rebroadcast-blocked sweep: failed to mark merged task done");
1259 continue;
1260 }
1261 let fields: &[(&str, serde_json::Value)] = &[
1262 ("block_reason", serde_json::Value::Null),
1263 ("needs_review_refires", serde_json::json!(0)),
1264 ];
1265 if let Err(e) = store.set_fields(task.id, fields).await {
1266 tracing::warn!(task_id = task.id, err = %e, "global rebroadcast-blocked sweep: failed to clear fields after marking done");
1267 }
1268 continue;
1269 }
1270 if pr.state.eq_ignore_ascii_case("closed") {
1271 tracing::info!(
1272 task_id = task.id,
1273 repo = %task.repo,
1274 pr_number,
1275 "global rebroadcast-blocked sweep: PR closed unmerged — leaving task blocked, not re-dispatching"
1276 );
1277 let new_reason = format!(
1278 "{REBROADCAST_BLOCK_REASON} (PR #{pr_number} closed unmerged — no further auto-recovery)"
1279 );
1280 let fields: &[(&str, serde_json::Value)] =
1281 &[("block_reason", serde_json::json!(new_reason))];
1282 if let Err(e) = store.set_fields_silent(task.id, fields).await {
1286 tracing::warn!(task_id = task.id, err = %e, "global rebroadcast-blocked sweep: failed to annotate closed-unmerged block reason");
1287 }
1288 continue;
1289 }
1290 }
1292 Err(e) => {
1293 tracing::warn!(task_id = task.id, repo = %task.repo, pr_number, err = %e, "global rebroadcast-blocked sweep: failed to get PR status — skipping this tick");
1294 continue;
1295 }
1296 }
1297 }
1298
1299 if !any_routable {
1300 continue;
1301 }
1302
1303 if let Err(e) = task_manager
1304 .update_task_status_by_store_id(
1305 task.id,
1306 &task.repo,
1307 &ExternalId(ext_id.clone()),
1308 Status::NeedsReview,
1309 )
1310 .await
1311 {
1312 tracing::warn!(task_id = task.id, repo = %task.repo, err = %e, "global rebroadcast-blocked sweep: failed to reset task to needs_review — skipping");
1313 continue;
1314 }
1315
1316 let fields: &[(&str, serde_json::Value)] = &[
1317 ("block_reason", serde_json::Value::Null),
1318 ("needs_review_refires", serde_json::json!(0)),
1319 ];
1320 if let Err(e) = store.set_fields(task.id, fields).await {
1321 tracing::warn!(task_id = task.id, err = %e, "global rebroadcast-blocked sweep: failed to clear block_reason/needs_review_refires after status reset");
1322 }
1323
1324 tracing::info!(
1325 task_id = task.id,
1326 repo = %task.repo,
1327 ext_id = %ext_id,
1328 "global rebroadcast-blocked sweep: auto-recovered rebroadcast-blocked task — review agents now available"
1329 );
1330 }
1331
1332 Ok(())
1333}
1334
1335async fn auto_unblock_blocked_tasks(
1336 repo: &str,
1337 task_manager: &Arc<TaskManager>,
1338 store: &Arc<TaskStore>,
1339 dispatching: &Arc<DashMap<String, String>>,
1340 auto_merge_in_flight: &Arc<DashSet<String>>,
1341) -> anyhow::Result<()> {
1342 let blocked = match store.list_by_status(repo, TaskStatus::Blocked).await {
1343 Ok(tasks) => tasks,
1344 Err(e) => {
1345 tracing::warn!(err = %e, "failed to list blocked tasks — skipping auto-unblock this tick");
1346 return Ok(());
1347 }
1348 };
1349
1350 if blocked.is_empty() {
1351 return Ok(());
1352 }
1353
1354 let task_ids: Vec<i64> = blocked.iter().map(|t| t.id).collect();
1356 let mut runs_by_task = match store.get_runs_batch(&task_ids).await {
1357 Ok(map) => map,
1358 Err(e) => {
1359 tracing::warn!(err = %e, "get_runs_batch failed — skipping auto-unblock this tick");
1360 return Ok(());
1361 }
1362 };
1363
1364 let (ci_failure_tasks, other_blocked): (Vec<_>, Vec<_>) =
1366 blocked.into_iter().partition(is_ci_failure_block);
1367
1368 if !ci_failure_tasks.is_empty() {
1370 let gh = match GhHttp::new() {
1371 Ok(g) => Some(g),
1372 Err(e) => {
1373 tracing::warn!(err = %e, "failed to create GhHttp — skipping CI failure unblock this tick");
1374 None
1375 }
1376 };
1377 for task in ci_failure_tasks {
1378 if ci_failure_unblock_cooldown_elapsed(&task)
1379 && let Some(ref gh) = gh
1380 && let Err(e) = try_unblock_ci_failure_task(
1381 gh,
1382 repo,
1383 task_manager,
1384 store,
1385 &task,
1386 auto_merge_in_flight,
1387 )
1388 .await
1389 {
1390 tracing::warn!(task_id = task.id, err = %e, "CI failure unblock check failed");
1391 }
1392 }
1393 }
1394
1395 for task in other_blocked {
1397 if task.block_reason.is_some() {
1398 continue;
1399 }
1400
1401 let task_id_str = task
1402 .external_id
1403 .clone()
1404 .unwrap_or_else(|| format!("internal:{}", task.id));
1405 let dispatch_key = format!("{repo}/{task_id_str}");
1406 if dispatching.contains_key(&dispatch_key) {
1407 continue;
1408 }
1409
1410 let runs = runs_by_task.remove(&task.id).unwrap_or_default();
1411 let relevant_runs: Vec<_> = runs
1412 .into_iter()
1413 .filter(|r| r.run_type == "agent" || r.run_type == "review")
1414 .collect();
1415 let mut failures = Vec::new();
1416 let mut has_review_failure = false;
1417 for run in relevant_runs.iter().rev() {
1418 if let Some(category) = classify_failure_from_run(run) {
1419 if run.run_type == "review" {
1420 has_review_failure = true;
1421 }
1422 failures.push(category);
1423 }
1424 if failures.len() >= 3 {
1425 break;
1426 }
1427 }
1428
1429 if failures.is_empty() || !failures.iter().all(|f| f.is_recoverable()) {
1430 continue;
1431 }
1432
1433 let reason_key = failures.first().map_or("", |f| f.as_str());
1435
1436 let current_reason = task.auto_unblock_last_reason.clone();
1439 let reason_changed = reason_key != current_reason;
1440
1441 let cooldown_count = if reason_changed {
1444 0
1445 } else {
1446 task.auto_unblock_count
1447 };
1448
1449 if cooldown_count >= 3 {
1453 continue;
1454 }
1455
1456 if !auto_unblock_cooldown_elapsed(cooldown_count, &task.auto_unblock_last_at) {
1457 continue;
1458 }
1459
1460 let new_count = cooldown_count + 1;
1464 let now = chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string();
1465
1466 let ext_id = task
1467 .external_id
1468 .clone()
1469 .unwrap_or_else(|| format!("internal:{}", task.id));
1470 let new_status = if has_review_failure {
1471 Status::NeedsReview
1472 } else {
1473 Status::New
1474 };
1475 let new_status_str = if has_review_failure {
1476 "needs_review"
1477 } else {
1478 "new"
1479 };
1480
1481 if let Err(e) = task_manager
1484 .update_task_status(&ExternalId(ext_id), new_status)
1485 .await
1486 {
1487 tracing::warn!(
1488 task_id = task.id,
1489 err = %e,
1490 "auto-unblock failed to update status — counter not incremented"
1491 );
1492 continue;
1493 }
1494
1495 let mut fields: Vec<(&str, serde_json::Value)> = vec![
1497 ("auto_unblock_count", serde_json::json!(new_count)),
1498 ("auto_unblock_last_at", serde_json::json!(now)),
1499 ("auto_unblock_last_reason", serde_json::json!(reason_key)),
1500 ("agent", serde_json::Value::Null),
1501 ("model", serde_json::Value::Null),
1502 ];
1503 fields.push(("no_code_reroutes", serde_json::json!(0)));
1504 fields.push(("no_code_last_agent", serde_json::json!("")));
1505 if has_review_failure {
1506 fields.push(("review_agent_failures", serde_json::json!(0)));
1507 fields.push(("review_invocations", serde_json::json!(0)));
1508 fields.push(("review_resets", serde_json::json!(0)));
1509 }
1510 if let Err(e) = store.set_fields(task.id, &fields).await {
1511 tracing::warn!(task_id = task.id, err = %e, "failed to set auto_unblock fields after successful status update");
1512 }
1515
1516 if let Err(e) = store
1517 .append_activity(
1518 task.id,
1519 "auto_unblock",
1520 Some("blocked"),
1521 Some(new_status_str),
1522 None,
1523 None,
1524 Some(&serde_json::json!({
1525 "reason": reason_key,
1526 "count": if reason_changed { 1 } else { task.auto_unblock_count + 1 },
1527 "failures": failures.iter().map(|f| format!("{f:?}")).collect::<Vec<_>>(),
1528 })),
1529 )
1530 .await
1531 {
1532 tracing::warn!(task_id = task.id, err = %e, "failed to log auto_unblock activity");
1533 }
1534
1535 tracing::info!(
1536 task_id = task.id,
1537 failures = ?failures,
1538 review_failure = has_review_failure,
1539 "auto-unblocked task with recoverable failures"
1540 );
1541 }
1542
1543 Ok(())
1544}
1545
1546pub(crate) async fn auto_unblock_ci_failure_blocked_tasks_global(
1557 task_manager: &Arc<TaskManager>,
1558 store: &Arc<TaskStore>,
1559 auto_merge_in_flight: &Arc<DashSet<String>>,
1560) -> anyhow::Result<()> {
1561 let all_blocked = match store.list_all_by_status_global(TaskStatus::Blocked).await {
1562 Ok(tasks) => tasks,
1563 Err(e) => {
1564 tracing::warn!(err = %e, "global CI-failure blocked sweep: failed to list blocked tasks");
1565 return Ok(());
1566 }
1567 };
1568
1569 let ci_failure_tasks: Vec<_> = all_blocked
1570 .into_iter()
1571 .filter(is_ci_failure_block)
1572 .collect();
1573 if ci_failure_tasks.is_empty() {
1574 return Ok(());
1575 }
1576
1577 let gh = match GhHttp::new() {
1578 Ok(g) => g,
1579 Err(e) => {
1580 tracing::warn!(err = %e, "global CI-failure blocked sweep: failed to create GhHttp");
1581 return Ok(());
1582 }
1583 };
1584
1585 for task in ci_failure_tasks {
1586 if !ci_failure_unblock_cooldown_elapsed(&task) {
1587 continue;
1588 }
1589 if let Err(e) = try_unblock_ci_failure_task(
1590 &gh,
1591 &task.repo,
1592 task_manager,
1593 store,
1594 &task,
1595 auto_merge_in_flight,
1596 )
1597 .await
1598 {
1599 tracing::warn!(
1600 task_id = task.id,
1601 repo = %task.repo,
1602 err = %e,
1603 "global CI-failure blocked sweep: unblock check failed"
1604 );
1605 }
1606 }
1607
1608 Ok(())
1609}
1610
1611pub(crate) async fn refire_and_escalate_stale_needs_review_global(
1625 task_manager: &Arc<TaskManager>,
1626 store: &Arc<TaskStore>,
1627 router: &Router,
1628 dispatching: &Arc<DashMap<String, String>>,
1629 active_repos: &HashSet<String>,
1630) -> anyhow::Result<()> {
1631 const MIN_STALE_NEEDS_REVIEW_MINUTES: i64 = 1;
1632 const MAX_NEEDS_REVIEW_REFIRE_ATTEMPTS: u64 = 5;
1633
1634 let needs_review_tasks = match store
1635 .list_all_by_status_global(TaskStatus::NeedsReview)
1636 .await
1637 {
1638 Ok(tasks) => tasks,
1639 Err(e) => {
1640 tracing::warn!(err = %e, "global NeedsReview refire sweep: failed to list needs_review tasks");
1641 return Ok(());
1642 }
1643 };
1644
1645 if needs_review_tasks.is_empty() {
1646 return Ok(());
1647 }
1648
1649 let all_review_agents_cooled = router.healthy_agent_count("review") == 0;
1652 if all_review_agents_cooled {
1653 tracing::info!(
1654 count = needs_review_tasks.len(),
1655 "global NeedsReview refire sweep: no review agent available — skipping this pass"
1656 );
1657 return Ok(());
1658 }
1659
1660 for stored in needs_review_tasks {
1661 if !active_repos.contains(&stored.repo) {
1663 continue;
1664 }
1665
1666 let external = tasks::store_task_to_external(&stored);
1667 let ext_id = external.id.clone();
1668 let repo = stored.repo.clone();
1669
1670 let age_minutes = match chrono::DateTime::parse_from_rfc3339(&external.updated_at) {
1671 Ok(updated_at) => {
1672 let age = chrono::Utc::now() - updated_at.with_timezone(&chrono::Utc);
1673 if age.num_minutes() < MIN_STALE_NEEDS_REVIEW_MINUTES {
1674 continue;
1675 }
1676 Some(age.num_minutes())
1677 }
1678 Err(_) => None,
1679 };
1680
1681 let dispatch_key = format!("{}/{}", repo, ext_id.0);
1683 if dispatching.contains_key(&dispatch_key) {
1684 continue;
1685 }
1686
1687 let current_refires = u64::try_from(stored.needs_review_refires).unwrap_or(0);
1688 let new_refires = current_refires + 1;
1689
1690 if new_refires > MAX_NEEDS_REVIEW_REFIRE_ATTEMPTS {
1691 tracing::warn!(
1692 task_id = stored.id,
1693 repo = %repo,
1694 new_refires,
1695 "global NeedsReview refire sweep: escalating to Blocked after repeated refires"
1696 );
1697 let fields = [
1698 ("block_reason", serde_json::json!(REBROADCAST_BLOCK_REASON)),
1699 (
1700 "last_error",
1701 serde_json::json!(format!("escalated after {} retries", new_refires)),
1702 ),
1703 ];
1704 if let Err(e) = task_manager
1705 .update_task_status_and_result_by_store_id(
1706 stored.id,
1707 &repo,
1708 &ext_id,
1709 Status::Blocked,
1710 &fields,
1711 )
1712 .await
1713 {
1714 tracing::error!(task_id = stored.id, repo = %repo, err = %e, "global NeedsReview refire sweep: escalation to Blocked failed — skipping to avoid silent auto-unblock loop");
1715 continue;
1716 }
1717 if let Err(e) = store::store_increment_by_id(
1718 &Some(Arc::clone(store)),
1719 stored.id,
1720 "needs_review_refires",
1721 )
1722 .await
1723 {
1724 tracing::warn!(
1725 task_id = stored.id,
1726 repo = %repo,
1727 err = %e,
1728 "global NeedsReview refire sweep: failed to increment needs_review_refires after escalation"
1729 );
1730 }
1731 continue;
1732 }
1733
1734 let required_minutes =
1736 MIN_STALE_NEEDS_REVIEW_MINUTES * (1i64 << u32::try_from(current_refires).unwrap_or(0));
1737
1738 let should_fire = match age_minutes {
1739 Some(age) => age >= required_minutes,
1740 None => true,
1741 };
1742
1743 if !should_fire {
1744 continue;
1745 }
1746
1747 if let Err(e) = task_manager
1748 .update_task_status_by_store_id(stored.id, &repo, &ext_id, Status::NeedsReview)
1749 .await
1750 {
1751 tracing::warn!(
1752 task_id = stored.id,
1753 repo = %repo,
1754 err = %e,
1755 "global NeedsReview refire sweep: failed to re-fire NeedsReview event — not incrementing counter"
1756 );
1757 continue;
1758 }
1759
1760 let fired_refires = match store::store_increment_by_id(
1761 &Some(Arc::clone(store)),
1762 stored.id,
1763 "needs_review_refires",
1764 )
1765 .await
1766 {
1767 Ok(v) => v,
1768 Err(e) => {
1769 tracing::warn!(
1770 task_id = stored.id,
1771 repo = %repo,
1772 err = %e,
1773 "global NeedsReview refire sweep: failed to increment needs_review_refires after re-fire"
1774 );
1775 current_refires + 1
1776 }
1777 };
1778
1779 tracing::info!(
1780 task_id = stored.id,
1781 repo = %repo,
1782 age_minutes,
1783 refires = fired_refires,
1784 required_minutes,
1785 "global NeedsReview refire sweep: re-firing NeedsReview event for stale task"
1786 );
1787 }
1788
1789 Ok(())
1790}
1791
1792#[expect(
1799 clippy::too_many_arguments,
1800 reason = "the engine passes its shared handles one by one"
1801)]
1802pub(crate) async fn sync_tick(
1803 backend: &Arc<dyn ExternalBackend>,
1804 tmux: &Arc<TmuxManager>,
1805 repo: &str,
1806 config: &EngineConfig,
1807 router: &Arc<RwLock<Router>>,
1808 task_manager: &Arc<TaskManager>,
1809 store: &Arc<TaskStore>,
1810 dispatching: &Arc<DashMap<String, String>>,
1811 auto_merge_in_flight: &Arc<DashSet<String>>,
1812) -> anyhow::Result<()> {
1813 if !store.is_leader() {
1815 return Ok(());
1816 }
1817
1818 tracing::debug!("sync tick");
1819
1820 let mut phases: Vec<(&str, u128)> = Vec::new();
1821 let mut at = Instant::now();
1822
1823 let gh_circuit_open = is_github_circuit_open();
1826
1827 if gh_circuit_open {
1828 let remaining = github_circuit_remaining_secs();
1829 tracing::debug!(
1830 remaining_secs = remaining,
1831 "GitHub 5xx circuit breaker open — skipping non-critical sync work"
1832 );
1833 if let Err(e) =
1836 try_kv_set_prefer_store(Some(store), "metrics:orch.github_5xx_circuit.open", "1").await
1837 {
1838 tracing::error!(err = %e, "failed to persist circuit breaker state — may cause thundering herd after restart");
1839 }
1840 } else if let Err(e) =
1841 try_kv_set_prefer_store(Some(store), "metrics:orch.github_5xx_circuit.open", "0").await
1842 {
1843 tracing::error!(err = %e, "failed to clear circuit breaker state");
1844 }
1845
1846 if !gh_circuit_open {
1849 if let Err(e) = ingest_external_tasks(backend, repo, store).await {
1850 tracing::debug!(err = %e, "external task ingest failed");
1851 }
1852 phases.push(("ingest", at.elapsed().as_millis()));
1853 at = Instant::now();
1854
1855 if let Err(e) = backend.reconcile_project_statuses().await {
1859 tracing::warn!(err = %e, "project status reconciliation failed");
1860 }
1861 phases.push(("board_reconcile", at.elapsed().as_millis()));
1862 }
1863 at = Instant::now();
1864
1865 if !gh_circuit_open {
1867 let backend = Arc::clone(backend);
1868 let repo = repo.to_string();
1869 let task_manager = Arc::clone(task_manager);
1870 let store = Arc::clone(store);
1871 tokio::spawn(async move {
1872 if let Err(e) = cleanup_done_worktrees(&backend, &repo, &task_manager, &store).await {
1873 tracing::warn!(err = %e, "worktree cleanup failed");
1874 }
1875 if let Err(e) = prune_old_run_output(&store).await {
1876 tracing::warn!(err = %e, "task run pruning failed");
1877 }
1878 });
1879 }
1880
1881 let (in_review_tasks, needs_review_tasks, blocked_review_candidates) =
1882 match prefetch_review_tasks(store, repo).await {
1883 Ok(tasks) => tasks,
1884 Err(e) => {
1885 tracing::warn!(err = %e, "failed to prefetch review tasks");
1886 (vec![], vec![], vec![])
1887 }
1888 };
1889 phases.push(("prefetch_review", at.elapsed().as_millis()));
1890 at = Instant::now();
1891
1892 if !gh_circuit_open {
1893 let mention_fallback = chrono::Utc::now() - chrono::Duration::hours(24);
1894 let mention_since = kv_get_prefer_store(Some(store), "mentions_last_checked")
1895 .await
1896 .unwrap_or_else(|| mention_fallback.format("%Y-%m-%dT%H:%M:%SZ").to_string());
1897 let shared_since = mention_since.clone();
1898
1899 let cached_router_config = router.read().await.config.clone();
1900 let mut combined_needs = needs_review_tasks.clone();
1902 combined_needs.extend(blocked_review_candidates.clone());
1903
1904 let (merge_result, comments_result, review_result) = tokio::join!(
1905 check_merged_prs(
1906 backend,
1907 repo,
1908 store,
1909 task_manager,
1910 &in_review_tasks,
1911 &combined_needs,
1912 ),
1913 fetch_comments_since(backend, &shared_since),
1914 async {
1915 let gh = match GhHttp::new() {
1916 Ok(g) => g,
1917 Err(e) => {
1918 tracing::warn!(err = %e, "failed to create GitHub HTTP client for review poll");
1919 return Ok(());
1920 }
1921 };
1922 Box::pin(review_open_prs(
1923 backend,
1924 repo,
1925 config,
1926 task_manager,
1927 store,
1928 dispatching,
1929 auto_merge_in_flight,
1930 &in_review_tasks,
1931 &gh,
1932 &cached_router_config,
1933 ))
1934 .await
1935 }
1936 );
1937
1938 if let Err(e) = merge_result {
1939 tracing::warn!(err = %e, "PR merge check failed");
1940 }
1941
1942 match comments_result {
1943 Ok(comments) => {
1944 let filtered = filter_mentions_by_since(&comments, &mention_since);
1945
1946 if let Err(e) =
1947 scan_comments(backend, Some(store), repo, task_manager, Some(&filtered)).await
1948 {
1949 tracing::warn!(err = %e, "comment scan failed");
1950 }
1951 }
1952 Err(e) => {
1953 tracing::warn!(err = %e, "comment fetch failed");
1954 }
1955 }
1956
1957 if let Err(e) = review_result {
1958 tracing::warn!(err = %e, "PR review failed");
1959 }
1960
1961 if let Err(e) = ci_bootstrap::apply_pending(store, task_manager, repo).await {
1962 tracing::warn!(err = %e, "CI bootstrap ruleset apply failed");
1963 }
1964
1965 phases.push(("merge_comments_review", at.elapsed().as_millis()));
1966 }
1967 at = Instant::now();
1968
1969 let in_progress_tasks: Vec<_> = match store.list_by_status(repo, TaskStatus::InProgress).await {
1979 Ok(tasks) => tasks
1980 .into_iter()
1981 .map(|stored| {
1982 let external = tasks::store_task_to_external(&stored);
1983 (stored, external)
1984 })
1985 .collect(),
1986 Err(e) => {
1987 tracing::warn!(err = %e, "failed to list in_progress tasks — skipping stale InProgress check");
1988 vec![]
1989 }
1990 };
1991
1992 if !in_progress_tasks.is_empty() {
1993 let live_sessions: HashSet<String> = tmux
1995 .list_sessions()
1996 .await
1997 .unwrap_or_default()
1998 .into_iter()
1999 .map(|s| s.name)
2000 .collect();
2001
2002 for (_stored, external) in &in_progress_tasks {
2003 const MIN_STALE_MINUTES: i64 = 5;
2005
2006 let task_id = &external.id.0;
2007
2008 let dispatch_key = format!("{repo}/{task_id}");
2010 {
2011 if dispatching.contains_key(&dispatch_key) {
2012 tracing::debug!(
2013 task_id = %task_id,
2014 "task locked by dispatch flow, skipping stale check"
2015 );
2016 continue;
2017 }
2018 }
2019
2020 match chrono::DateTime::parse_from_rfc3339(&external.updated_at) {
2023 Ok(updated_at) => {
2024 let age = chrono::Utc::now() - updated_at.with_timezone(&chrono::Utc);
2025 if age.num_minutes() < MIN_STALE_MINUTES {
2026 tracing::debug!(
2027 task_id = %task_id,
2028 age_seconds = age.num_seconds(),
2029 "InProgress task is too young to be considered stale, skipping"
2030 );
2031 continue;
2032 }
2033 }
2034 Err(e) => {
2035 tracing::warn!(
2036 task_id = %task_id,
2037 ts = %external.updated_at,
2038 err = %e,
2039 "invalid updated_at timestamp — treating task as potentially stale"
2040 );
2041 }
2042 }
2043
2044 let agent_session = tmux.session_name(repo, task_id);
2045 if live_sessions.contains(&agent_session) {
2046 continue;
2048 }
2049
2050 {
2054 tracing::warn!(
2055 task_id = %task_id,
2056 session = %agent_session,
2057 "InProgress task has no active agent session — resetting to Routed"
2058 );
2059 match task_manager
2060 .update_task_status_if(&external.id, Status::Routed, Status::InProgress)
2061 .await
2062 {
2063 Err(e) => {
2064 tracing::error!(task_id = %task_id, err = %e, "failed to reset stale InProgress task — task may be stuck in InProgress indefinitely");
2065 }
2066 Ok(false) => {
2067 tracing::debug!(
2068 task_id = %task_id,
2069 "stale InProgress reset skipped — task already transitioned (concurrent Done/Blocked/NeedsReview)"
2070 );
2071 }
2072 Ok(true) => {
2073 }
2075 }
2076 }
2077 }
2078 }
2079
2080 phases.push(("stale_in_progress", at.elapsed().as_millis()));
2081 at = Instant::now();
2082 if let Err(e) =
2084 auto_unblock_blocked_tasks(repo, task_manager, store, dispatching, auto_merge_in_flight)
2085 .await
2086 {
2087 tracing::warn!(err = %e, "auto-unblock failed");
2088 }
2089
2090 phases.push(("auto_unblock", at.elapsed().as_millis()));
2091
2092 engine::log_phase_timings("sync", repo, &phases);
2093 Ok(())
2094}
2095
2096pub(crate) async fn sync_shared_tail(router: &Arc<RwLock<Router>>, store: Option<&Arc<TaskStore>>) {
2101 let Some(store) = store else {
2102 return;
2103 };
2104 if !store.is_leader() {
2105 return;
2106 }
2107 match skills_sync().await {
2108 Ok(()) => {
2109 let llm = router.read().await.llm_router_handle();
2112 llm.invalidate_skills_catalog().await;
2113 }
2114 Err(e) => {
2115 tracing::warn!(err = %e, "skills sync failed");
2116 }
2117 }
2118
2119 let (available_agents, config) = {
2123 let r = router.read().await;
2124 (r.available_agents.clone(), r.config.clone())
2125 };
2126 let model_checker = |agent: &str| -> bool {
2127 for comp in &["simple", "medium", "complex", "review"] {
2128 if config.has_available_model_for_complexity(agent, comp) {
2129 return true;
2130 }
2131 }
2132 false
2133 };
2134 refresh_degraded_agents(
2135 store,
2136 &available_agents,
2137 &model_checker,
2138 router_config::health_check_window_hours(),
2139 router_config::degraded_rate_limit_threshold(),
2140 )
2141 .await;
2142
2143 emit_degraded_agents_if_needed(&available_agents, &config, Some(store)).await;
2144}
2145
2146async fn scan_comments(
2156 backend: &Arc<dyn ExternalBackend>,
2157 store: Option<&Arc<TaskStore>>,
2158 repo: &str,
2159 task_manager: &Arc<TaskManager>,
2160 prefetched_comments: Option<&[Mention]>,
2161) -> anyhow::Result<()> {
2162 let current_user = match backend.get_authenticated_user().await {
2163 Ok(Some(u)) => format!("@{u}"),
2164 Ok(None) => {
2165 tracing::debug!("backend does not support user identity, skipping comment scan");
2166 return Ok(());
2167 }
2168 Err(e) => {
2169 tracing::warn!(err = %e, "failed to get current user, skipping comment scan");
2170 return Ok(());
2171 }
2172 };
2173
2174 let comments = if let Some(c) = prefetched_comments {
2175 c.to_vec()
2176 } else {
2177 let fallback = chrono::Utc::now() - chrono::Duration::hours(24);
2178 let since_str = match kv_get_prefer_store(store, "mentions_last_checked").await {
2179 Some(ts) => ts,
2180 None => fallback.format("%Y-%m-%dT%H:%M:%SZ").to_string(),
2181 };
2182 match backend.get_mentions(&since_str).await {
2183 Ok(m) => m,
2184 Err(e) => {
2185 tracing::warn!(err = %e, "failed to get comments");
2186 return Ok(());
2187 }
2188 }
2189 };
2190
2191 let existing_mentions: HashSet<String> = if let Some(s) = store {
2193 match s.list_source_ids_by_source(repo, "mention", 30).await {
2194 Ok(ids) => ids.into_iter().collect(),
2195 Err(e) => {
2196 tracing::warn!(err = %e, "failed to load mention source IDs — skipping to prevent duplicates");
2197 return Ok(());
2198 }
2199 }
2200 } else {
2201 HashSet::new()
2202 };
2203
2204 let gh_cell: OnceCell<GhHttp> = OnceCell::new();
2208 let get_gh = || async { gh_cell.get_or_try_init(|| async { GhHttp::new() }).await };
2209
2210 let actions: Vec<CommentAction> = comments
2213 .iter()
2214 .map(|c| {
2215 classify_comment(
2216 &c.body,
2217 c.issue_url.as_deref(),
2218 ¤t_user,
2219 existing_mentions.contains(&c.id),
2220 )
2221 })
2222 .collect();
2223
2224 let pr_check_inputs: Vec<(usize, String)> = actions
2228 .iter()
2229 .enumerate()
2230 .filter_map(|(i, action)| {
2231 if let CommentAction::CreateMentionTask {
2232 issue_num: Some(num),
2233 } = action
2234 {
2235 Some((i, num.clone()))
2236 } else {
2237 None
2238 }
2239 })
2240 .collect();
2241
2242 let is_pr_values: Vec<bool> = if pr_check_inputs.is_empty() {
2244 Vec::new()
2245 } else {
2246 let gh = get_gh().await?;
2247 future::join_all(
2248 pr_check_inputs
2249 .iter()
2250 .map(|(_, num)| async move { gh.is_pull_request(repo, num).await }),
2251 )
2252 .await
2253 };
2254
2255 let is_pr_map: HashMap<usize, bool> = pr_check_inputs
2257 .iter()
2258 .map(|(i, _)| *i)
2259 .zip(is_pr_values)
2260 .collect();
2261
2262 let mut cursor = MentionCursor::default();
2263
2264 for (comment_idx, (comment, action)) in comments.iter().zip(actions).enumerate() {
2265 match action {
2266 CommentAction::Skip => {
2267 cursor.advance(&comment.created_at);
2268 }
2269
2270 CommentAction::ExecuteCommand { command, issue_num } => {
2271 let store_opt: Option<Arc<dyn CommandStoreOps>> =
2272 store.map(|s| Arc::clone(s) as Arc<dyn CommandStoreOps>);
2273 let gh = get_gh().await?;
2274 let outcome = validate_and_run_command(
2275 backend,
2276 gh,
2277 repo,
2278 &issue_num,
2279 &command,
2280 &comment.author,
2281 &store_opt,
2282 task_manager,
2283 )
2284 .await;
2285 match outcome {
2286 CommandOutcome::FetchFailed | CommandOutcome::CollaboratorCheckFailed => {
2289 tracing::warn!(
2290 comment_id = %comment.id,
2291 command = %command,
2292 "deferring slash command after a transient GitHub error"
2293 );
2294 cursor.block_on_gap();
2295 break;
2296 }
2297 CommandOutcome::Executed { .. }
2300 | CommandOutcome::NotOpen { .. }
2301 | CommandOutcome::NotCollaborator { .. } => cursor.advance(&comment.created_at),
2302 }
2303 }
2304
2305 CommentAction::ExecuteCommandForMention { command, .. } => {
2306 if let Err(e) = backend.acknowledge_mention(&comment.id).await {
2307 tracing::debug!(err = %e, mention_id = %comment.id, "failed to acknowledge mention");
2308 } else {
2312 let gh = get_gh().await?;
2315 let advanced_safely = handle_slash_command(
2316 backend,
2317 store,
2318 repo,
2319 task_manager,
2320 gh,
2321 comment,
2322 &command,
2323 &mut cursor,
2324 )
2325 .await?;
2326 if !advanced_safely {
2327 cursor.block_on_gap();
2328 break;
2329 }
2330 }
2331 }
2332
2333 CommentAction::CreateMentionTask { issue_num } => {
2334 if let Err(e) = backend.acknowledge_mention(&comment.id).await {
2335 tracing::debug!(err = %e, mention_id = %comment.id, "failed to acknowledge mention");
2336 } else {
2340 let is_pr = is_pr_map.get(&comment_idx).copied().unwrap_or(false);
2345
2346 let (title, task_body) = match (&issue_num, is_pr) {
2347 (Some(num), false) => (
2348 format!("Respond to mention by @{} on task #{num}", comment.author),
2349 format!(
2350 "Mention by @{} on GitHub issue #{num}:\n\n{}\n\n---\nThis mention was posted on issue #{num}. Review the full issue to understand the context before responding.",
2351 comment.author, comment.body
2352 ),
2353 ),
2354 (Some(num), true) => (
2355 format!("Respond to mention by @{} on PR #{num}", comment.author),
2356 format!(
2357 "Mention by @{} on GitHub PR #{num}:\n\n{}\n\n---\nThis mention was posted on PR #{num}. Review the full PR (diff, description, and comments) to understand the context before responding.",
2358 comment.author, comment.body
2359 ),
2360 ),
2361 (None, _) => (
2362 format!("Respond to mention by @{}", comment.author),
2363 format!("Mention by @{}:\n\n{}", comment.author, comment.body),
2364 ),
2365 };
2366
2367 if let Some(s) = store {
2368 let parent_id = match issue_num {
2371 Some(ref num) => {
2372 match resolve_mention_parent_id(s, repo, num, is_pr).await {
2373 Ok(id) => id,
2374 Err(e) => {
2375 tracing::warn!(
2376 mention_id = %comment.id,
2377 err = %e,
2378 "deferring mention task due to parent lookup failure"
2379 );
2380 return Err(anyhow::anyhow!(
2381 "deferring mention {} due to parent lookup failure",
2382 comment.id
2383 ));
2384 }
2385 }
2386 }
2387 None => None,
2388 };
2389 if !record_mention_task(
2390 s,
2391 repo,
2392 comment,
2393 &title,
2394 &task_body,
2395 parent_id,
2396 &mut cursor,
2397 )
2398 .await
2399 {
2400 cursor.block_on_gap();
2401 break;
2402 }
2403 }
2404 }
2405 }
2406 }
2407 }
2408
2409 if let Some(ts) = cursor.into_last_safe_ts() {
2410 kv_set_prefer_store(store, "mentions_last_checked", &ts).await;
2411 }
2412
2413 Ok(())
2414}
2415
2416async fn skills_sync() -> anyhow::Result<()> {
2422 let skills = match config::get_skills() {
2423 Ok(s) => s,
2424 Err(e) => {
2425 tracing::debug!(err = %e, "no skills configured");
2426 return Ok(());
2427 }
2428 };
2429
2430 if skills.is_empty() {
2431 tracing::debug!("no skills configured, skipping sync");
2432 return Ok(());
2433 }
2434
2435 let skills_base = home::skills_dir()?;
2436 let git_timeout = Duration::from_secs(60);
2437
2438 let futs: Vec<_> = skills
2441 .into_iter()
2442 .map(|skill| sync_one_skill(skill, skills_base.clone(), git_timeout))
2443 .collect();
2444 future::join_all(futs).await;
2445
2446 Ok(())
2447}
2448
2449async fn sync_one_skill(skill: config::SkillConfig, skills_base: PathBuf, git_timeout: Duration) {
2454 if skill.repo.contains("..") || skill.repo.matches('/').count() != 1 {
2456 tracing::warn!(repo = %skill.repo, "invalid skill repo format, expected 'owner/repo'");
2457 return;
2458 }
2459
2460 let repo_dir = skills_base.join(&skill.repo);
2461 let repo_url = format!("https://github.com/{}.git", skill.repo);
2462
2463 let repo_exists = fs::metadata(&repo_dir).await.is_ok_and(|m| m.is_dir());
2466
2467 if repo_exists {
2468 tracing::debug!(repo = %skill.repo, "pulling skill repo");
2470 let pull_result = time::timeout(
2471 git_timeout,
2472 git_ops::git()
2473 .await
2474 .args(["pull", "--ff-only", "--prune"])
2475 .current_dir(&repo_dir)
2476 .output_with_context(),
2477 )
2478 .await;
2479
2480 match pull_result {
2481 Ok(Ok(output)) if !output.status.success() => {
2482 let stderr = String::from_utf8_lossy(&output.stderr);
2483 tracing::warn!(repo = %skill.repo, err = %stderr, "git pull failed");
2484 }
2485 Ok(Ok(_)) => {
2486 tracing::debug!(repo = %skill.repo, "skill repo updated");
2487 }
2488 Ok(Err(e)) => {
2489 tracing::warn!(repo = %skill.repo, err = %e, "git pull error");
2490 }
2491 Err(_) => {
2492 tracing::warn!(repo = %skill.repo, "git pull timed out after 60s");
2493 }
2494 }
2495 } else {
2496 tracing::debug!(repo = %skill.repo, "cloning skill repo");
2498 let Some(parent) = repo_dir.parent() else {
2499 tracing::warn!(repo = %skill.repo, "skill repo path has no parent directory");
2500 return;
2501 };
2502 if let Err(e) = fs::create_dir_all(parent).await {
2503 tracing::warn!(repo = %skill.repo, err = %e, "failed to create parent dir for skill");
2504 return;
2505 }
2506 let Some(repo_dir_str) = repo_dir.to_str() else {
2507 tracing::warn!(repo = %skill.repo, "skill repo path is not valid UTF-8");
2508 return;
2509 };
2510
2511 let clone_result = time::timeout(
2512 git_timeout,
2513 git_ops::git()
2514 .await
2515 .args(["clone", "--depth", "1", &repo_url, repo_dir_str])
2516 .output_with_context(),
2517 )
2518 .await;
2519
2520 match clone_result {
2521 Ok(Ok(output)) if !output.status.success() => {
2522 let stderr = String::from_utf8_lossy(&output.stderr);
2523 tracing::warn!(repo = %skill.repo, err = %stderr, "git clone failed");
2524 let _ = fs::remove_dir_all(&repo_dir).await;
2526 }
2527 Ok(Ok(_)) => {
2528 tracing::info!(repo = %skill.repo, "skill repo cloned");
2529 }
2530 Ok(Err(e)) => {
2531 tracing::warn!(repo = %skill.repo, err = %e, "git clone error");
2532 let _ = fs::remove_dir_all(&repo_dir).await;
2533 }
2534 Err(_) => {
2535 tracing::warn!(repo = %skill.repo, "git clone timed out after 60s");
2536 let _ = fs::remove_dir_all(&repo_dir).await;
2537 }
2538 }
2539 }
2540}
2541
2542struct DegradedAgentDetail {
2544 agent: String,
2546 reason: String,
2550 cooled_models: Vec<String>,
2553}
2554
2555struct AgentPoolHealth {
2557 agent: String,
2558 cooled_models: Vec<String>,
2559 persistent_failed_models: Vec<String>,
2560 total_configured_models: usize,
2561}
2562
2563pub(crate) async fn emit_degraded_agents_if_needed(
2572 available_agents: &[String],
2573 config: &RouterConfig,
2574 store: Option<&Arc<TaskStore>>,
2575) {
2576 const STALE_AGENTS_KV: &str = "metrics:orch.model_pool_stale_persistent.agents";
2579
2580 const COMPLEXITIES: &[&str] = &["simple", "medium", "complex", "review"];
2582 const HEAVILY_COOLED_RATIO_THRESHOLD: f64 = 0.5;
2583
2584 let mut details: Vec<DegradedAgentDetail> = Vec::new();
2585 let mut pool_health: Vec<AgentPoolHealth> = Vec::new();
2586
2587 for agent in available_agents {
2588 let agent_in_cd = is_agent_in_cooldown(agent);
2589
2590 let mut cooled_models: Vec<String> = Vec::new();
2592 let mut persistent_failed_models: Vec<String> = Vec::new();
2593 let mut seen_models = HashSet::new();
2594 for comp in COMPLEXITIES {
2595 if let Some(pool) = config.model_pool_for_complexity(agent, comp) {
2596 for model in pool {
2597 if seen_models.insert(model.clone()) && is_model_in_cooldown(agent, &model) {
2598 let key = format!("{agent}:{model}");
2599 if matches!(
2600 cooldown_reason(&key).as_deref(),
2601 Some("persistent_model_error" | "model_not_found")
2602 ) {
2603 persistent_failed_models.push(model.clone());
2604 }
2605 cooled_models.push(model);
2606 }
2607 }
2608 }
2609 }
2610 let total_configured_models = seen_models.len();
2611 pool_health.push(AgentPoolHealth {
2612 agent: agent.clone(),
2613 cooled_models: cooled_models.clone(),
2614 persistent_failed_models: persistent_failed_models.clone(),
2615 total_configured_models,
2616 });
2617
2618 let has_model = COMPLEXITIES
2621 .iter()
2622 .any(|comp| config.has_available_model_for_complexity(agent, comp));
2623
2624 if agent_in_cd || !has_model {
2625 let reason = if agent_in_cd {
2626 cooldown_reason(agent).unwrap_or_else(|| "agent_cooldown".to_string())
2627 } else {
2628 "no_available_model".to_string()
2629 };
2630 details.push(DegradedAgentDetail {
2631 agent: agent.clone(),
2632 reason,
2633 cooled_models,
2634 });
2635 }
2636 }
2637
2638 let count = details.len();
2639
2640 if let Err(e) = try_kv_set_prefer_store(
2643 store,
2644 "metrics:orch.agents_degraded.count",
2645 &count.to_string(),
2646 )
2647 .await
2648 {
2649 tracing::warn!(err = %e, "failed to persist degraded agents count metric");
2650 }
2651
2652 if count >= 3 {
2653 let degraded_agents: Vec<&str> = details.iter().map(|d| d.agent.as_str()).collect();
2655 let cooled_models_dim: Vec<String> = details
2656 .iter()
2657 .filter(|d| !d.cooled_models.is_empty())
2658 .map(|d| format!("{}:[{}]", d.agent, d.cooled_models.join(",")))
2659 .collect();
2660 let reasons_dim: Vec<String> = details
2661 .iter()
2662 .map(|d| format!("{}={}", d.agent, d.reason))
2663 .collect();
2664
2665 tracing::warn!(
2666 degraded_count = count,
2667 degraded_agents = ?degraded_agents,
2668 cooled_models = %cooled_models_dim.join("; "),
2669 cooldown_reasons = %reasons_dim.join("; "),
2670 "multi-agent degradation detected"
2671 );
2672
2673 if let Err(e) =
2675 try_kv_set_prefer_store(store, "metrics:orch.agents_degraded.alert", "1").await
2676 {
2677 tracing::warn!(err = %e, "failed to persist degraded agents alert metric");
2678 }
2679 } else if let Err(e) =
2680 try_kv_set_prefer_store(store, "metrics:orch.agents_degraded.alert", "0").await
2681 {
2682 tracing::warn!(err = %e, "failed to clear degraded agents alert metric");
2684 }
2685
2686 let heavily_cooled_agents: Vec<String> = pool_health
2690 .iter()
2691 .filter_map(|d| {
2692 if d.total_configured_models == 0 || d.persistent_failed_models.is_empty() {
2693 return None;
2694 }
2695 #[expect(
2696 clippy::cast_precision_loss,
2697 reason = "model counts are far below 2^52"
2698 )]
2699 let ratio = d.cooled_models.len() as f64 / d.total_configured_models as f64;
2700 (ratio >= HEAVILY_COOLED_RATIO_THRESHOLD).then(|| {
2701 format!(
2702 "{}:{}/{}:{}",
2703 d.agent,
2704 d.cooled_models.len(),
2705 d.total_configured_models,
2706 d.persistent_failed_models.join(",")
2707 )
2708 })
2709 })
2710 .collect();
2711
2712 let current_dim = heavily_cooled_agents.join(",");
2713 let prev_dim = kv_get_prefer_store(store, STALE_AGENTS_KV)
2714 .await
2715 .unwrap_or_default();
2716
2717 if heavily_cooled_agents.is_empty() {
2718 if !prev_dim.is_empty() {
2719 tracing::info!("agent model pool stale condition cleared");
2720 }
2721 if let Err(e) = try_kv_set_prefer_store(store, STALE_AGENTS_KV, "").await {
2722 tracing::warn!(err = %e, "failed to clear stale model pool agents state");
2723 }
2724 if let Err(e) =
2725 try_kv_set_prefer_store(store, "metrics:orch.model_pool_stale_persistent.alert", "0")
2726 .await
2727 {
2728 tracing::warn!(err = %e, "failed to clear stale model pool alert metric");
2729 }
2730 } else {
2731 if current_dim != prev_dim {
2732 tracing::warn!(
2733 affected_agents = ?heavily_cooled_agents,
2734 "agent model pool appears stale: persistent model failures in heavily cooled pool"
2735 );
2736 }
2737 if let Err(e) = try_kv_set_prefer_store(store, STALE_AGENTS_KV, ¤t_dim).await {
2738 tracing::warn!(err = %e, "failed to persist stale model pool agents state");
2739 }
2740 if let Err(e) =
2741 try_kv_set_prefer_store(store, "metrics:orch.model_pool_stale_persistent.alert", "1")
2742 .await
2743 {
2744 tracing::warn!(err = %e, "failed to persist stale model pool alert metric");
2745 }
2746 }
2747}
2748
2749pub(crate) async fn ingest_external_tasks(
2757 backend: &Arc<dyn ExternalBackend>,
2758 repo: &str,
2759 store: &Arc<TaskStore>,
2760) -> anyhow::Result<()> {
2761 #[derive(Clone)]
2762 struct DuplicateTarget {
2763 id: String,
2764 status_label: String,
2765 }
2766
2767 let mut existing_by_title: HashMap<String, DuplicateTarget> = HashMap::new();
2768 let mut accepted_by_title: HashMap<String, DuplicateTarget> = HashMap::new();
2769
2770 let cutoff = (chrono::Utc::now() - chrono::Duration::hours(24))
2771 .format("%Y-%m-%dT%H:%M:%SZ")
2772 .to_string();
2773 match store.list_for_doctor(repo, &cutoff).await {
2774 Ok(recent) => {
2775 let mut insert_target = |task: &Task| {
2776 if task.origin == "internal" {
2777 return;
2778 }
2779 let Some(ext_id) = task.external_id.as_ref() else {
2780 return;
2781 };
2782 let status_label = task
2783 .labels
2784 .iter()
2785 .find(|l| l.starts_with("status:"))
2786 .cloned()
2787 .unwrap_or_default();
2788 existing_by_title
2789 .entry(task.title.clone())
2790 .or_insert(DuplicateTarget {
2791 id: ext_id.clone(),
2792 status_label,
2793 });
2794 };
2795
2796 for task in recent.iter().filter(|t| t.status != TaskStatus::Done) {
2798 insert_target(task);
2799 }
2800 for task in recent.iter().filter(|t| t.status == TaskStatus::Done) {
2801 insert_target(task);
2802 }
2803 }
2804 Err(e) => {
2805 tracing::warn!(repo, err = %e, "ingest: failed to load recent tasks for dedup");
2806 }
2807 }
2808
2809 let all_tasks: Vec<(ExternalTask, Option<Status>)> =
2819 match backend.list_active_open_issues().await {
2820 Ok(tasks) => tasks,
2821 Err(e) => {
2822 tracing::warn!(
2823 repo,
2824 err = %e,
2825 "ingest: failed to list open issues — no new issue is ingested this cycle"
2826 );
2827 return Ok(());
2828 }
2829 };
2830
2831 let mut id_status_pairs: Vec<(i64, Status)> = Vec::new();
2834 let mut new_task_node_ids: Vec<(i64, String)> = Vec::new();
2837 let existing_ext_ids: HashSet<String> = match store.existing_external_ids(repo).await {
2842 Ok(s) => s,
2843 Err(e) => {
2844 tracing::warn!(repo = repo, err = %e, "ingest: failed to load existing external ids — proceeding");
2845 HashSet::new()
2846 }
2847 };
2848 for (task, status) in &all_tasks {
2849 let is_new = !existing_ext_ids.contains(&task.id.0);
2852
2853 if is_new {
2854 let duplicate_target = existing_by_title
2855 .get(&task.title)
2856 .or_else(|| accepted_by_title.get(&task.title));
2857 if let Some(target) = duplicate_target
2858 && target.id != task.id.0
2859 {
2860 let label = target.status_label.as_str();
2861 let should_dedupe =
2862 match backend.has_open_issue_with_title(&task.title, label).await {
2863 Ok(true) => true,
2864 Ok(false) => false,
2865 Err(e) => {
2866 tracing::warn!(
2867 task_id = task.id.0,
2868 err = %e,
2869 "ingest: dedup check failed — defaulting to keep issue"
2870 );
2871 false
2872 }
2873 };
2874
2875 if should_dedupe {
2876 let comment = format!(
2877 "Duplicate of #{id} — closing to avoid duplicate work.{footer}",
2878 id = target.id,
2879 footer = engine::orch_footer()
2880 );
2881 if let Err(e) = backend.post_comment(&task.id, &comment).await {
2882 tracing::warn!(
2883 task_id = task.id.0,
2884 err = %e,
2885 "ingest: failed to comment on duplicate issue"
2886 );
2887 }
2888 match backend.update_status(&task.id, Status::Done).await {
2889 Ok(()) => {
2890 tracing::info!(
2891 task_id = task.id.0,
2892 duplicate_of = target.id,
2893 "ingest: closed duplicate issue"
2894 );
2895 continue;
2896 }
2897 Err(e) => {
2898 tracing::warn!(
2899 task_id = task.id.0,
2900 err = %e,
2901 "ingest: failed to close duplicate issue — falling through to ingest normally"
2902 );
2903 }
2904 }
2905 }
2906 }
2907 }
2908
2909 match store.ensure_external_task(repo, task).await {
2910 Ok(store_id) => {
2911 if let Ok(number) = task.id.0.parse::<i64>()
2912 && let Err(err) = dependencies::sync_issue_dependencies(
2913 backend.as_ref(),
2914 store,
2915 store_id,
2916 repo,
2917 Some(number),
2918 &task.body,
2919 )
2920 .await
2921 {
2922 tracing::warn!(
2923 task_id = task.id.0,
2924 err = %err,
2925 "ingest: dependency sync failed"
2926 );
2927 }
2928 if let Some(s) = status {
2929 id_status_pairs.push((store_id, *s));
2930 }
2931 if is_new {
2933 let status_label = task
2934 .labels
2935 .iter()
2936 .find(|l| l.starts_with("status:"))
2937 .cloned()
2938 .unwrap_or_default();
2939 accepted_by_title
2940 .entry(task.title.clone())
2941 .or_insert(DuplicateTarget {
2942 id: task.id.0.clone(),
2943 status_label,
2944 });
2945 if let Err(e) = backend.acknowledge_issue(&task.id).await {
2946 tracing::debug!(
2947 task_id = task.id.0,
2948 err = %e,
2949 "ingest: acknowledgment failed"
2950 );
2951 }
2952 let task_status = status.unwrap_or(Status::New);
2955 match backend.sync_to_project(&task.id, task_status).await {
2956 Ok(Some(node_id)) => {
2957 new_task_node_ids.push((store_id, node_id));
2958 }
2959 Ok(None) => {
2960 }
2962 Err(e) => {
2963 tracing::debug!(
2964 task_id = task.id.0,
2965 err = %e,
2966 "ingest: project board sync failed"
2967 );
2968 }
2969 }
2970 }
2971 }
2972 Err(e) => {
2973 tracing::debug!(task_id = task.id.0, ?e, "ingest: upsert failed");
2974 }
2975 }
2976 }
2977
2978 if !id_status_pairs.is_empty() {
2984 let all_ids: Vec<i64> = id_status_pairs.iter().map(|(id, _)| *id).collect();
2985 let existing_map = match store.get_batch(&all_ids).await {
2986 Ok(map) => map,
2987 Err(e) => {
2988 tracing::warn!(err = %e, task_count = all_ids.len(), "ingest: get_batch failed — skipping initial status sync for newly ingested tasks this tick");
2989 return Ok(());
2990 }
2991 };
2992 for (store_id, status) in id_status_pairs {
2993 if let Some(existing) = existing_map.get(&store_id)
2994 && existing.status == TaskStatus::New
2995 {
2996 let db_status = status_to_task_status(status);
2997 if db_status != TaskStatus::New
2998 && let Err(e) = store.update_status(store_id, db_status).await
2999 {
3000 tracing::debug!(?e, "ingest: status sync failed");
3001 }
3002 }
3003 }
3004 }
3005
3006 if !new_task_node_ids.is_empty() {
3009 let node_ids: Vec<String> = new_task_node_ids.iter().map(|(_, n)| n.clone()).collect();
3010 match backend.get_project_item_estimates(&node_ids).await {
3011 Ok(estimates) => {
3012 for (store_id, node_id) in new_task_node_ids {
3013 if let Some(&estimate) = estimates.get(&node_id)
3014 && estimate > 0
3015 && let Err(e) = store.set_estimate_if_zero(store_id, estimate).await
3016 {
3017 tracing::debug!(
3018 store_id,
3019 estimate,
3020 err = %e,
3021 "ingest: failed to set estimate from project"
3022 );
3023 }
3024 }
3025 }
3026 Err(e) => {
3027 tracing::debug!(
3028 err = %e,
3029 "ingest: project estimate fetch failed — skipping estimate sync this tick"
3030 );
3031 }
3032 }
3033 }
3034
3035 Ok(())
3036}
3037
3038#[cfg(test)]
3039mod tests {
3040 use super::*;
3041 use super::{CommentAction, MentionCursor, classify_comment};
3042 use crate::backends::test_helpers::NoopBackend;
3043 use crate::backends::{ExternalId, ExternalTask, Mention, Status};
3044 use crate::engine::commands::OwnerCommand;
3045 use crate::engine::cooldown::record_persistent_model_failure;
3046 use crate::engine::cooldown::set_agent_cooldown;
3047 use crate::engine::dispatch_guard::DispatchGuard;
3048 use crate::engine::router::Router;
3049 use crate::engine::router::RouterConfig;
3050 use crate::github::types::GitHubIssue;
3051 use crate::github::types::GitHubUser;
3052 use crate::store::NewTask;
3053 use crate::store::{RunTokenUsage, TaskStore};
3054 use async_trait::async_trait;
3055 use serial_test::serial;
3056 use std::slice;
3057 use std::sync::Arc;
3058 use std::sync::atomic::AtomicUsize;
3059 use std::sync::atomic::Ordering;
3060 use tokio::sync::Mutex;
3061 use tokio::sync::oneshot;
3062
3063 struct IngestMockBackend {
3067 tasks: Vec<(String, Vec<ExternalTask>)>,
3069 dedup_result: bool,
3070 comments: Mutex<Vec<(String, String)>>,
3071 status_updates: Mutex<Vec<(String, Status)>>,
3072 project_syncs: Mutex<Vec<(String, Status)>>,
3073 project_reconciliations: AtomicUsize,
3074 list_calls: AtomicUsize,
3076 }
3077
3078 impl IngestMockBackend {
3079 fn with_tasks(tasks: Vec<(Status, ExternalTask)>) -> Arc<Self> {
3080 Self::with_tasks_and_dedup(tasks, false)
3081 }
3082
3083 fn with_tasks_and_dedup(
3084 tasks: Vec<(Status, ExternalTask)>,
3085 dedup_result: bool,
3086 ) -> Arc<Self> {
3087 let mut grouped: Vec<(String, Vec<ExternalTask>)> = Vec::new();
3088 for (status, task) in tasks {
3089 let label = status.as_label().to_string();
3090 if let Some(entry) = grouped.iter_mut().find(|(l, _)| l == &label) {
3091 entry.1.push(task);
3092 } else {
3093 grouped.push((label, vec![task]));
3094 }
3095 }
3096 Arc::new(Self {
3097 tasks: grouped,
3098 dedup_result,
3099 comments: Mutex::new(Vec::new()),
3100 status_updates: Mutex::new(Vec::new()),
3101 project_syncs: Mutex::new(Vec::new()),
3102 project_reconciliations: AtomicUsize::new(0),
3103 list_calls: AtomicUsize::new(0),
3104 })
3105 }
3106 }
3107
3108 #[async_trait]
3109 impl ExternalBackend for IngestMockBackend {
3110 fn name(&self) -> &'static str {
3111 "ingest-mock"
3112 }
3113 async fn create_task(&self, _: &str, _: &str, _: &[String]) -> anyhow::Result<ExternalId> {
3114 Ok(ExternalId("new".into()))
3115 }
3116 async fn get_task(&self, _: &ExternalId) -> anyhow::Result<ExternalTask> {
3117 anyhow::bail!("not implemented")
3118 }
3119 async fn list_by_status(&self, status: Status) -> anyhow::Result<Vec<ExternalTask>> {
3120 let label = status.as_label().to_string();
3121 Ok(self
3122 .tasks
3123 .iter()
3124 .find(|(l, _)| l == &label)
3125 .map(|(_, t)| t.clone())
3126 .unwrap_or_default())
3127 }
3128 async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
3129 Ok(vec![])
3130 }
3131 async fn list_active_open_issues(
3132 &self,
3133 ) -> anyhow::Result<Vec<(ExternalTask, Option<Status>)>> {
3134 self.list_calls.fetch_add(1, Ordering::Relaxed);
3135 let mut result = Vec::new();
3136 for (label, tasks) in &self.tasks {
3137 let status = match label.as_str() {
3138 "status:routed" => Some(Status::Routed),
3139 "status:in_progress" => Some(Status::InProgress),
3140 "status:needs_review" => Some(Status::NeedsReview),
3141 "status:in_review" => Some(Status::InReview),
3142 "status:approved" => Some(Status::Approved),
3143 "status:blocked" => Some(Status::Blocked),
3144 _ => None,
3145 };
3146 for task in tasks {
3147 result.push((task.clone(), status));
3148 }
3149 }
3150 Ok(result)
3151 }
3152 async fn post_comment(&self, id: &ExternalId, body: &str) -> anyhow::Result<()> {
3153 self.comments
3154 .lock()
3155 .await
3156 .push((id.0.clone(), body.to_string()));
3157 Ok(())
3158 }
3159 async fn set_labels(&self, _: &ExternalId, _: &[String]) -> anyhow::Result<()> {
3160 Ok(())
3161 }
3162 async fn remove_label(&self, _: &ExternalId, _: &str) -> anyhow::Result<()> {
3163 Ok(())
3164 }
3165 async fn get_sub_issues(&self, _: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
3166 Ok(vec![])
3167 }
3168 async fn health_check(&self) -> anyhow::Result<()> {
3169 Ok(())
3170 }
3171 async fn get_mentions(&self, _: &str) -> anyhow::Result<Vec<Mention>> {
3172 Ok(vec![])
3173 }
3174 async fn update_status(&self, id: &ExternalId, status: Status) -> anyhow::Result<()> {
3175 self.status_updates
3176 .lock()
3177 .await
3178 .push((id.0.clone(), status));
3179 Ok(())
3180 }
3181
3182 async fn has_open_issue_with_title(&self, _: &str, _: &str) -> anyhow::Result<bool> {
3183 Ok(self.dedup_result)
3184 }
3185
3186 async fn sync_to_project(
3187 &self,
3188 id: &ExternalId,
3189 status: Status,
3190 ) -> anyhow::Result<Option<String>> {
3191 self.project_syncs.lock().await.push((id.0.clone(), status));
3192 Ok(None) }
3194
3195 async fn reconcile_project_statuses(&self) -> anyhow::Result<()> {
3196 self.project_reconciliations.fetch_add(1, Ordering::Relaxed);
3197 Ok(())
3198 }
3199 }
3200
3201 fn make_ext_task(id: &str, title: &str) -> ExternalTask {
3202 ExternalTask {
3203 id: ExternalId(id.to_string()),
3204 title: title.to_string(),
3205 body: String::new(),
3206 state: "open".to_string(),
3207 labels: vec![],
3208 author: "user".to_string(),
3209 created_at: "2026-01-01T00:00:00Z".to_string(),
3210 updated_at: "2026-01-01T00:00:00Z".to_string(),
3211 url: format!("https://github.com/owner/repo/issues/{id}"),
3212 }
3213 }
3214
3215 #[serial(cooldown_state)]
3216 #[tokio::test]
3217 async fn ingest_upserts_tasks_into_store() {
3218 cooldown::reset_global_state().await;
3219 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![
3220 (Status::New, make_ext_task("1", "First issue")),
3221 (Status::InProgress, make_ext_task("2", "Second issue")),
3222 ]);
3223 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3224
3225 ingest_external_tasks(&backend, "owner/repo", &store)
3226 .await
3227 .unwrap();
3228
3229 let all = store.list_all("owner/repo").await.unwrap();
3231 assert_eq!(all.len(), 2);
3232
3233 let task1 = store
3234 .get_by_external_id("owner/repo", "1")
3235 .await
3236 .unwrap()
3237 .unwrap();
3238 assert_eq!(task1.title, "First issue");
3239 assert_eq!(task1.status, TaskStatus::New);
3240
3241 let task2 = store
3242 .get_by_external_id("owner/repo", "2")
3243 .await
3244 .unwrap()
3245 .unwrap();
3246 assert_eq!(task2.title, "Second issue");
3247 assert_eq!(task2.status, TaskStatus::InProgress);
3248 }
3249
3250 #[serial(cooldown_state)]
3251 #[tokio::test]
3252 async fn ingest_takes_every_open_issue_whatever_its_age_or_the_last_ingest() {
3253 cooldown::reset_global_state().await;
3257 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3258 store
3259 .kv_set("issues_last_ingested:owner/repo", "2026-10-26T00:00:00Z")
3260 .await
3261 .unwrap();
3262
3263 let mut old = make_ext_task("73", "Filed weeks ago, never ingested");
3264 old.updated_at = "2026-01-01T00:00:00Z".to_string();
3265 let backend = IngestMockBackend::with_tasks(vec![(Status::New, old)]);
3266 let backend_dyn: Arc<dyn ExternalBackend> = Arc::<IngestMockBackend>::clone(&backend);
3267 ingest_external_tasks(&backend_dyn, "owner/repo", &store)
3268 .await
3269 .unwrap();
3270 assert!(
3271 store
3272 .get_by_external_id("owner/repo", "73")
3273 .await
3274 .unwrap()
3275 .is_some()
3276 );
3277
3278 let backend = IngestMockBackend::with_tasks(vec![(
3280 Status::New,
3281 make_ext_task("78", "Opened after the restart"),
3282 )]);
3283 let backend_dyn: Arc<dyn ExternalBackend> = Arc::<IngestMockBackend>::clone(&backend);
3284 ingest_external_tasks(&backend_dyn, "owner/repo", &store)
3285 .await
3286 .unwrap();
3287 assert_eq!(
3288 backend.list_calls.load(Ordering::Relaxed),
3289 1,
3290 "one sync is one listing"
3291 );
3292 assert!(
3293 store
3294 .get_by_external_id("owner/repo", "78")
3295 .await
3296 .unwrap()
3297 .is_some()
3298 );
3299 assert_eq!(
3300 store
3301 .kv_get("issues_last_ingested:owner/repo")
3302 .await
3303 .unwrap()
3304 .as_deref(),
3305 Some("2026-10-26T00:00:00Z"),
3306 "the ingest must keep no cursor, so a stale one is never read or advanced"
3307 );
3308 }
3309
3310 #[serial(cooldown_state)]
3311 #[tokio::test]
3312 async fn ingest_updates_existing_tasks() {
3313 cooldown::reset_global_state().await;
3314 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![(
3315 Status::Routed,
3316 make_ext_task("42", "Updated title"),
3317 )]);
3318 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3319
3320 store
3322 .create(&NewTask {
3323 external_id: Some("42".to_string()),
3324 repo: "owner/repo".to_string(),
3325 origin: "github".to_string(),
3326 title: "Original title".to_string(),
3327 ..Default::default()
3328 })
3329 .await
3330 .unwrap();
3331
3332 ingest_external_tasks(&backend, "owner/repo", &store)
3333 .await
3334 .unwrap();
3335
3336 let task = store
3338 .get_by_external_id("owner/repo", "42")
3339 .await
3340 .unwrap()
3341 .unwrap();
3342 assert_eq!(task.title, "Updated title");
3343 assert_eq!(task.status, TaskStatus::Routed);
3344 }
3345
3346 #[serial(cooldown_state)]
3347 #[tokio::test]
3348 async fn ingest_closes_duplicate_titles() {
3349 cooldown::reset_global_state().await;
3350
3351 let backend = IngestMockBackend::with_tasks_and_dedup(
3352 vec![(Status::New, make_ext_task("2", "Duplicate issue"))],
3353 true,
3354 );
3355 let backend_dyn: Arc<dyn ExternalBackend> = Arc::<IngestMockBackend>::clone(&backend);
3356 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3357
3358 store
3359 .create(&NewTask {
3360 external_id: Some("1".to_string()),
3361 repo: "owner/repo".to_string(),
3362 origin: "github".to_string(),
3363 title: "Duplicate issue".to_string(),
3364 labels: vec!["status:new".to_string()],
3365 ..Default::default()
3366 })
3367 .await
3368 .unwrap();
3369
3370 ingest_external_tasks(&backend_dyn, "owner/repo", &store)
3371 .await
3372 .unwrap();
3373
3374 let all = store.list_all("owner/repo").await.unwrap();
3375 assert_eq!(all.len(), 1);
3376
3377 let comments = backend.comments.lock().await;
3378 assert_eq!(comments.len(), 1);
3379 assert!(comments[0].1.contains("Duplicate of #1"));
3380
3381 let status_updates = backend.status_updates.lock().await;
3382 assert_eq!(status_updates.len(), 1);
3383 assert_eq!(status_updates[0].0, "2");
3384 assert_eq!(status_updates[0].1, Status::Done);
3385 }
3386
3387 #[serial(cooldown_state)]
3388 #[tokio::test]
3389 async fn ingest_handles_empty_backend() {
3390 cooldown::reset_global_state().await;
3391 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
3392 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3393
3394 let result = ingest_external_tasks(&backend, "owner/repo", &store).await;
3396 assert!(result.is_ok());
3397
3398 let all = store.list_all("owner/repo").await.unwrap();
3399 assert!(all.is_empty());
3400 }
3401
3402 #[serial(cooldown_state)]
3403 #[tokio::test]
3404 async fn ingest_syncs_status_correctly_across_statuses() {
3405 cooldown::reset_global_state().await;
3406 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![
3407 (Status::New, make_ext_task("1", "New")),
3408 (Status::Routed, make_ext_task("2", "Routed")),
3409 (Status::InProgress, make_ext_task("3", "InProgress")),
3410 (Status::NeedsReview, make_ext_task("4", "NeedsReview")),
3411 (Status::InReview, make_ext_task("5", "InReview")),
3412 (Status::Blocked, make_ext_task("6", "Blocked")),
3413 ]);
3414 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3415
3416 ingest_external_tasks(&backend, "owner/repo", &store)
3417 .await
3418 .unwrap();
3419
3420 let counts = store.status_counts("owner/repo").await.unwrap();
3421 assert_eq!(counts.get("new"), Some(&1));
3422 assert_eq!(counts.get("routed"), Some(&1));
3423 assert_eq!(counts.get("in_progress"), Some(&1));
3424 assert_eq!(counts.get("needs_review"), Some(&1));
3425 assert_eq!(counts.get("in_review"), Some(&1));
3426 assert_eq!(counts.get("blocked"), Some(&1));
3427 }
3428
3429 #[serial(cooldown_state)]
3430 #[tokio::test]
3431 async fn ingest_does_not_overwrite_store_authoritative_status() {
3432 cooldown::reset_global_state().await;
3433 let backend: Arc<dyn ExternalBackend> =
3435 IngestMockBackend::with_tasks(vec![(Status::New, make_ext_task("99", "My task"))]);
3436 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3437
3438 let id = store
3440 .create(&NewTask {
3441 external_id: Some("99".to_string()),
3442 repo: "owner/repo".to_string(),
3443 origin: "github".to_string(),
3444 title: "My task".to_string(),
3445 ..Default::default()
3446 })
3447 .await
3448 .unwrap();
3449 store.update_status(id, TaskStatus::Routed).await.unwrap();
3450
3451 ingest_external_tasks(&backend, "owner/repo", &store)
3453 .await
3454 .unwrap();
3455
3456 let task = store.get(id).await.unwrap();
3457 assert_eq!(
3458 task.status,
3459 TaskStatus::Routed,
3460 "ingest must not overwrite store-authoritative status"
3461 );
3462 }
3463
3464 #[serial(cooldown_state)]
3465 #[tokio::test]
3466 async fn ingest_sync_to_project_called_only_for_new_tasks() {
3467 cooldown::reset_global_state().await;
3468
3469 let backend = IngestMockBackend::with_tasks(vec![
3471 (Status::New, make_ext_task("10", "Existing task")),
3472 (Status::New, make_ext_task("11", "New task")),
3473 ]);
3474 let backend_dyn: Arc<dyn ExternalBackend> = Arc::<IngestMockBackend>::clone(&backend);
3475 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3476
3477 store
3479 .create(&NewTask {
3480 external_id: Some("10".to_string()),
3481 repo: "owner/repo".to_string(),
3482 origin: "github".to_string(),
3483 title: "Existing task".to_string(),
3484 ..Default::default()
3485 })
3486 .await
3487 .unwrap();
3488
3489 ingest_external_tasks(&backend_dyn, "owner/repo", &store)
3490 .await
3491 .unwrap();
3492
3493 let syncs = backend.project_syncs.lock().await;
3494 assert_eq!(
3496 syncs.len(),
3497 1,
3498 "sync_to_project must be called exactly once (for new task only), got: {syncs:?}"
3499 );
3500 assert_eq!(syncs[0].0, "11", "sync_to_project must target the new task");
3501 }
3502
3503 #[serial(cooldown_state)]
3504 #[tokio::test]
3505 async fn ingest_handles_mixed_labeled_and_unlabeled_open_issues() {
3506 struct MixedBackend;
3510
3511 #[async_trait]
3512 impl ExternalBackend for MixedBackend {
3513 fn name(&self) -> &'static str {
3514 "mixed"
3515 }
3516 async fn create_task(
3517 &self,
3518 _: &str,
3519 _: &str,
3520 _: &[String],
3521 ) -> anyhow::Result<ExternalId> {
3522 Ok(ExternalId("x".into()))
3523 }
3524 async fn get_task(&self, _: &ExternalId) -> anyhow::Result<ExternalTask> {
3525 anyhow::bail!("not implemented")
3526 }
3527 async fn list_by_status(&self, _: Status) -> anyhow::Result<Vec<ExternalTask>> {
3528 Ok(vec![])
3529 }
3530 async fn list_active_open_issues(
3532 &self,
3533 ) -> anyhow::Result<Vec<(ExternalTask, Option<Status>)>> {
3534 Ok(vec![
3535 (make_ext_task("100", "Unlabeled routable"), None),
3536 (
3537 make_ext_task("101", "In progress issue"),
3538 Some(Status::InProgress),
3539 ),
3540 ])
3541 }
3542 async fn post_comment(&self, _: &ExternalId, _: &str) -> anyhow::Result<()> {
3543 Ok(())
3544 }
3545 async fn set_labels(&self, _: &ExternalId, _: &[String]) -> anyhow::Result<()> {
3546 Ok(())
3547 }
3548 async fn remove_label(&self, _: &ExternalId, _: &str) -> anyhow::Result<()> {
3549 Ok(())
3550 }
3551 async fn get_sub_issues(&self, _: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
3552 Ok(vec![])
3553 }
3554 async fn health_check(&self) -> anyhow::Result<()> {
3555 Ok(())
3556 }
3557 async fn get_mentions(&self, _: &str) -> anyhow::Result<Vec<Mention>> {
3558 Ok(vec![])
3559 }
3560 async fn update_status(&self, _: &ExternalId, _: Status) -> anyhow::Result<()> {
3561 Ok(())
3562 }
3563 async fn has_open_issue_with_title(&self, _: &str, _: &str) -> anyhow::Result<bool> {
3564 Ok(false)
3565 }
3566 async fn sync_to_project(
3567 &self,
3568 _: &ExternalId,
3569 _: Status,
3570 ) -> anyhow::Result<Option<String>> {
3571 Ok(None)
3572 }
3573 }
3574
3575 cooldown::reset_global_state().await;
3576
3577 let backend: Arc<dyn ExternalBackend> = Arc::new(MixedBackend);
3578 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3579
3580 ingest_external_tasks(&backend, "owner/repo", &store)
3581 .await
3582 .unwrap();
3583
3584 let all = store.list_all("owner/repo").await.unwrap();
3585 assert_eq!(all.len(), 2, "both issues must be ingested");
3586
3587 let unlabeled = store
3588 .get_by_external_id("owner/repo", "100")
3589 .await
3590 .unwrap()
3591 .unwrap();
3592 assert_eq!(
3593 unlabeled.status,
3594 TaskStatus::New,
3595 "unlabeled issue should start as New"
3596 );
3597
3598 let in_progress = store
3599 .get_by_external_id("owner/repo", "101")
3600 .await
3601 .unwrap()
3602 .unwrap();
3603 assert_eq!(
3604 in_progress.status,
3605 TaskStatus::InProgress,
3606 "labeled in_progress issue must be synced to InProgress"
3607 );
3608 }
3609
3610 #[serial(cooldown_state)]
3613 #[tokio::test]
3614 async fn kv_get_prefer_store_reads_from_store() {
3615 cooldown::reset_global_state().await;
3616 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3617
3618 store.kv_set("k1", "store_val").await.unwrap();
3619
3620 let opt = Some(&store);
3621 let val = kv_get_prefer_store(opt, "k1").await;
3622 assert_eq!(val.as_deref(), Some("store_val"));
3623 }
3624
3625 #[serial(cooldown_state)]
3626 #[tokio::test]
3627 async fn kv_get_prefer_store_returns_none_without_store() {
3628 cooldown::reset_global_state().await;
3629 let opt: Option<&Arc<TaskStore>> = None;
3630 let val = kv_get_prefer_store(opt, "k2").await;
3631 assert_eq!(val, None);
3632 }
3633
3634 #[serial(cooldown_state)]
3635 #[tokio::test]
3636 async fn kv_set_prefer_store_writes_to_store() {
3637 cooldown::reset_global_state().await;
3638 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3639
3640 let opt = Some(&store);
3641 kv_set_prefer_store(opt, "k3", "val3").await;
3642
3643 assert_eq!(store.kv_get("k3").await.unwrap().as_deref(), Some("val3"));
3644 }
3645
3646 #[serial(cooldown_state)]
3647 #[tokio::test]
3648 async fn kv_set_prefer_store_noop_without_store() {
3649 cooldown::reset_global_state().await;
3650 let opt: Option<&Arc<TaskStore>> = None;
3651 kv_set_prefer_store(opt, "k4", "val4").await;
3653 }
3654
3655 #[serial(cooldown_state)]
3658 #[tokio::test]
3659 async fn try_kv_get_prefer_store_returns_value_from_store() {
3660 cooldown::reset_global_state().await;
3661 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3662 store.kv_set("try_k1", "try_val1").await.unwrap();
3663
3664 let opt = Some(&store);
3665 let result = try_kv_get_prefer_store(opt, "try_k1").await;
3666 assert_eq!(result.unwrap().as_deref(), Some("try_val1"));
3667 }
3668
3669 #[serial(cooldown_state)]
3670 #[tokio::test]
3671 async fn try_kv_get_prefer_store_returns_none_for_missing_key() {
3672 cooldown::reset_global_state().await;
3673 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3674 let opt = Some(&store);
3675
3676 let result = try_kv_get_prefer_store(opt, "nonexistent_key").await;
3677 assert_eq!(result.unwrap(), None); }
3679
3680 #[serial(cooldown_state)]
3681 #[tokio::test]
3682 async fn try_kv_get_prefer_store_returns_none_without_store() {
3683 cooldown::reset_global_state().await;
3684 let opt: Option<&Arc<TaskStore>> = None;
3685 let result = try_kv_get_prefer_store(opt, "try_k2").await;
3686 assert_eq!(result.unwrap(), None);
3687 }
3688
3689 #[serial(cooldown_state)]
3690 #[tokio::test]
3691 async fn try_kv_set_prefer_store_writes_to_store() {
3692 cooldown::reset_global_state().await;
3693 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3694 let opt = Some(&store);
3695
3696 let result = try_kv_set_prefer_store(opt, "try_k3", "try_val3").await;
3697 assert!(result.is_ok());
3698 assert_eq!(
3699 store.kv_get("try_k3").await.unwrap().as_deref(),
3700 Some("try_val3")
3701 );
3702 }
3703
3704 #[serial(cooldown_state)]
3705 #[tokio::test]
3706 async fn try_kv_set_prefer_store_ok_without_store() {
3707 cooldown::reset_global_state().await;
3708 let opt: Option<&Arc<TaskStore>> = None;
3709 let result = try_kv_set_prefer_store(opt, "try_k4", "try_val4").await;
3711 assert!(result.is_ok());
3712 }
3713
3714 #[serial(cooldown_state)]
3717 #[test]
3718 fn dispatching_set_blocks_duplicate_processing() {
3719 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3720
3721 let repo = "owner/repo";
3722 let task_id = "42";
3723 let dispatch_key = format!("{repo}/{task_id}");
3724
3725 assert!(!dispatching.contains_key(&dispatch_key));
3727
3728 dispatching.insert(dispatch_key.clone(), task_id.to_string());
3730
3731 assert!(
3733 dispatching.contains_key(&dispatch_key),
3734 "task should be locked while dispatching"
3735 );
3736
3737 dispatching.remove(&dispatch_key);
3739
3740 assert!(
3742 !dispatching.contains_key(&dispatch_key),
3743 "task should be unlocked after review completes"
3744 );
3745 }
3746
3747 #[serial(cooldown_state)]
3748 #[test]
3749 fn dispatching_set_does_not_block_other_tasks() {
3750 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3751
3752 let repo = "owner/repo";
3753
3754 let key_42 = format!("{repo}/42");
3756 dispatching.insert(key_42.clone(), "42".to_string());
3757
3758 let key_43 = format!("{repo}/43");
3760 assert!(
3761 dispatching.contains_key(&key_42),
3762 "task 42 should be locked"
3763 );
3764 assert!(
3765 !dispatching.contains_key(&key_43),
3766 "task 43 should not be locked"
3767 );
3768 }
3769
3770 #[serial(cooldown_state)]
3783 #[tokio::test]
3784 async fn dispatch_key_held_during_review_agent_execution() {
3785 cooldown::reset_global_state().await;
3786
3787 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3788
3789 let dispatch_key = "owner/repo/42".to_string();
3790
3791 dispatching.insert(dispatch_key.clone(), "42".to_string());
3793
3794 let (agent_started_tx, agent_started_rx) = oneshot::channel::<()>();
3795 let (check_done_tx, check_done_rx) = oneshot::channel::<()>();
3796
3797 let dispatching_agent = Arc::clone(&dispatching);
3798 let key_agent = dispatch_key.clone();
3799
3800 let review_task = tokio::spawn(async move {
3802 agent_started_tx.send(()).unwrap();
3803 check_done_rx.await.unwrap();
3805 dispatching_agent.remove(&key_agent);
3807 });
3808
3809 agent_started_rx.await.unwrap();
3812 assert!(
3813 dispatching.contains_key(&dispatch_key),
3814 "dispatch_key must be visible in the dispatching set while the review \
3815 agent is running — without the a6d8b9a fix, review_open_prs would see \
3816 the key absent and re-dispatch the task, silently dropping \
3817 CHANGES_REQUESTED feedback"
3818 );
3819
3820 check_done_tx.send(()).unwrap();
3822 review_task.await.unwrap();
3823
3824 assert!(
3826 !dispatching.contains_key(&dispatch_key),
3827 "dispatch_key must be removed from the dispatching set after the \
3828 review agent completes so subsequent sync ticks can process the task"
3829 );
3830 }
3831
3832 #[serial(cooldown_state)]
3845 #[tokio::test]
3846 async fn dispatch_guard_releases_key_on_panic() {
3847 cooldown::reset_global_state().await;
3848
3849 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3850 let key = "owner/repo/42".to_string();
3851
3852 dispatching.insert(key.clone(), "42".to_string());
3856
3857 let guard = DispatchGuard::new(Arc::clone(&dispatching), key.clone());
3859
3860 let handle = tokio::spawn(async move {
3862 let _guard = guard; panic!("simulated panic inside review_and_merge");
3864 });
3865
3866 let _ = handle.await;
3868
3869 assert!(
3871 !dispatching.contains_key(&key),
3872 "dispatch key must be removed from the dispatching set even when the \
3873 spawned review task panics — without DispatchGuard the key leaks and \
3874 the task gets stuck in a permanent review loop"
3875 );
3876 }
3877
3878 #[serial(cooldown_state)]
3879 #[tokio::test]
3880 async fn sync_tick_reconciles_board_without_ingesting_or_transitioning_tasks() {
3881 cooldown::reset_global_state().await;
3882 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3883 let mock = IngestMockBackend::with_tasks(vec![]);
3884 let backend: Arc<dyn ExternalBackend> = Arc::<IngestMockBackend>::clone(&mock);
3885 let task_manager = Arc::new(TaskManager::with_store(
3886 Arc::clone(&backend),
3887 Arc::clone(&store),
3888 "owner/repo".to_string(),
3889 ));
3890 let dispatching = Arc::new(DashMap::new());
3891 Box::pin(sync_tick(
3892 &backend,
3893 &Arc::new(TmuxManager::new()),
3894 "owner/repo",
3895 &EngineConfig::default(),
3896 &Arc::new(RwLock::new(Router::new_for_test(
3897 RouterConfig::default(),
3898 vec!["claude".to_string()],
3899 ))),
3900 &task_manager,
3901 &store,
3902 &dispatching,
3903 &Arc::new(DashSet::new()),
3904 ))
3905 .await
3906 .unwrap();
3907 assert_eq!(mock.project_reconciliations.load(Ordering::Relaxed), 1);
3908 assert!(mock.project_syncs.lock().await.is_empty());
3909 assert!(mock.status_updates.lock().await.is_empty());
3910 assert!(dispatching.is_empty());
3911 assert!(
3912 store
3913 .existing_external_ids("owner/repo")
3914 .await
3915 .unwrap()
3916 .is_empty()
3917 );
3918 }
3919
3920 #[serial(cooldown_state)]
3921 #[tokio::test]
3922 async fn needs_review_refire_increments_and_fires_when_old_enough() {
3923 cooldown::reset_global_state().await;
3924 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3925 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
3926 let task_manager = Arc::new(TaskManager::with_store(
3927 backend,
3928 Arc::clone(&store),
3929 "owner/repo".to_string(),
3930 ));
3931 let router = reviewer_router();
3932 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3933
3934 let id = store
3935 .upsert_external(&crate::store::UpsertExternal {
3936 repo: "owner/repo",
3937 ext_id: "201",
3938 title: "Needs review old",
3939 body: "",
3940 author: "",
3941 url: "",
3942 labels: &[],
3943 origin: "github",
3944 })
3945 .await
3946 .unwrap();
3947 store
3948 .update_status(id, TaskStatus::NeedsReview)
3949 .await
3950 .unwrap();
3951 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
3952 .bind(id)
3953 .execute(store.pool())
3954 .await
3955 .unwrap();
3956
3957 let t = store.get(id).await.unwrap();
3958 assert_eq!(t.needs_review_refires, 0);
3959
3960 refire_and_escalate_stale_needs_review_global(
3961 &task_manager,
3962 &store,
3963 &router,
3964 &dispatching,
3965 &sweep_repos(),
3966 )
3967 .await
3968 .unwrap();
3969
3970 let t2 = store.get(id).await.unwrap();
3971 assert_eq!(t2.needs_review_refires, 1);
3972 assert_eq!(t2.status, TaskStatus::NeedsReview);
3973 }
3974
3975 #[serial(cooldown_state)]
3976 #[tokio::test]
3977 async fn needs_review_backoff_delays_fire_but_increments_counter() {
3978 cooldown::reset_global_state().await;
3979 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3980 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
3981 let task_manager = Arc::new(TaskManager::with_store(
3982 backend,
3983 Arc::clone(&store),
3984 "owner/repo".to_string(),
3985 ));
3986 let router = reviewer_router();
3987 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3988
3989 let id = store
3990 .upsert_external(&crate::store::UpsertExternal {
3991 repo: "owner/repo",
3992 ext_id: "202",
3993 title: "Needs review backoff",
3994 body: "",
3995 author: "",
3996 url: "",
3997 labels: &[],
3998 origin: "github",
3999 })
4000 .await
4001 .unwrap();
4002 store
4003 .update_status(id, TaskStatus::NeedsReview)
4004 .await
4005 .unwrap();
4006
4007 store
4008 .set_fields(id, &[("needs_review_refires", serde_json::json!(1))])
4009 .await
4010 .unwrap();
4011
4012 let now_minus_1 = (chrono::Utc::now() - chrono::Duration::minutes(1))
4014 .format("%Y-%m-%dT%H:%M:%SZ")
4015 .to_string();
4016 sqlx::query("UPDATE tasks SET updated_at = ? WHERE id = ?")
4017 .bind(now_minus_1.clone())
4018 .bind(id)
4019 .execute(store.pool())
4020 .await
4021 .unwrap();
4022
4023 let before = store.get(id).await.unwrap();
4024 assert_eq!(before.needs_review_refires, 1);
4025
4026 refire_and_escalate_stale_needs_review_global(
4027 &task_manager,
4028 &store,
4029 &router,
4030 &dispatching,
4031 &sweep_repos(),
4032 )
4033 .await
4034 .unwrap();
4035
4036 let after = store.get(id).await.unwrap();
4037 assert_eq!(after.needs_review_refires, 1);
4039 assert_eq!(after.updated_at, now_minus_1);
4040 assert_eq!(after.status, TaskStatus::NeedsReview);
4041 }
4042
4043 fn reviewer_router() -> Router {
4045 let mut config = RouterConfig::default();
4046 config.model_map.insert(
4047 "review".to_string(),
4048 HashMap::from([("claude".to_string(), vec!["sonnet".to_string()])]),
4049 );
4050 Router::new_for_test(config, vec!["claude".to_string()])
4051 }
4052
4053 #[serial(cooldown_state)]
4054 #[tokio::test]
4055 async fn needs_review_escalates_to_blocked_after_max_refires() {
4056 cooldown::reset_global_state().await;
4057 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4058 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4059 let task_manager = Arc::new(TaskManager::with_store(
4060 backend,
4061 Arc::clone(&store),
4062 "owner/repo".to_string(),
4063 ));
4064 let router = reviewer_router();
4065 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4066
4067 let id = store
4068 .upsert_external(&crate::store::UpsertExternal {
4069 repo: "owner/repo",
4070 ext_id: "203",
4071 title: "Needs review escalate",
4072 body: "",
4073 author: "",
4074 url: "",
4075 labels: &[],
4076 origin: "github",
4077 })
4078 .await
4079 .unwrap();
4080 store
4081 .update_status(id, TaskStatus::NeedsReview)
4082 .await
4083 .unwrap();
4084
4085 store
4087 .set_fields(id, &[("needs_review_refires", serde_json::json!(5))])
4088 .await
4089 .unwrap();
4090 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4091 .bind(id)
4092 .execute(store.pool())
4093 .await
4094 .unwrap();
4095
4096 refire_and_escalate_stale_needs_review_global(
4097 &task_manager,
4098 &store,
4099 &router,
4100 &dispatching,
4101 &sweep_repos(),
4102 )
4103 .await
4104 .unwrap();
4105
4106 let after = store.get(id).await.unwrap();
4107 assert_eq!(after.status, TaskStatus::Blocked);
4108 assert!(after.block_reason.is_some());
4109 }
4110
4111 #[serial(cooldown_state)]
4114 #[tokio::test]
4115 async fn needs_review_allows_max_refires_before_escalation() {
4116 cooldown::reset_global_state().await;
4117 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4118 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4119 let task_manager = Arc::new(TaskManager::with_store(
4120 backend,
4121 Arc::clone(&store),
4122 "owner/repo".to_string(),
4123 ));
4124 let router = reviewer_router();
4125 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4126
4127 let id = store
4128 .upsert_external(&crate::store::UpsertExternal {
4129 repo: "owner/repo",
4130 ext_id: "204",
4131 title: "Max refires test",
4132 body: "",
4133 author: "",
4134 url: "",
4135 labels: &[],
4136 origin: "github",
4137 })
4138 .await
4139 .unwrap();
4140 store
4141 .update_status(id, TaskStatus::NeedsReview)
4142 .await
4143 .unwrap();
4144
4145 store
4147 .set_fields(id, &[("needs_review_refires", serde_json::json!(4))])
4148 .await
4149 .unwrap();
4150 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4151 .bind(id)
4152 .execute(store.pool())
4153 .await
4154 .unwrap();
4155
4156 refire_and_escalate_stale_needs_review_global(
4157 &task_manager,
4158 &store,
4159 &router,
4160 &dispatching,
4161 &sweep_repos(),
4162 )
4163 .await
4164 .unwrap();
4165
4166 let after = store.get(id).await.unwrap();
4167 assert_eq!(after.status, TaskStatus::NeedsReview);
4168 assert_eq!(after.needs_review_refires, 5);
4169 assert!(after.block_reason.is_none());
4170 }
4171
4172 fn sweep_repos() -> HashSet<String> {
4174 ["owner/repo", "inactive/repo", "some/repo", "active/repo"]
4175 .into_iter()
4176 .map(String::from)
4177 .collect()
4178 }
4179
4180 #[serial(cooldown_state)]
4182 #[tokio::test]
4183 async fn global_sweeps_leave_tasks_of_a_removed_repo_alone() {
4184 cooldown::reset_global_state().await;
4185 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4186 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4187 let task_manager = Arc::new(TaskManager::with_store(
4188 backend,
4189 Arc::clone(&store),
4190 "active/repo".to_string(),
4191 ));
4192 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4193 let router = router_with_healthy_review_agent();
4194 let active: HashSet<String> = ["active/repo".to_string()].into_iter().collect();
4195
4196 let review_id = store
4197 .create_internal("removed/repo", "Needs review", "", "cron", "1", None)
4198 .await
4199 .unwrap();
4200 store
4201 .update_status(review_id, TaskStatus::NeedsReview)
4202 .await
4203 .unwrap();
4204 let blocked_id = store
4205 .create_internal("removed/repo", "Rebroadcast blocked", "", "cron", "2", None)
4206 .await
4207 .unwrap();
4208 store
4209 .update_status(blocked_id, TaskStatus::Blocked)
4210 .await
4211 .unwrap();
4212 store
4213 .set_block_reason(blocked_id, Some(REBROADCAST_BLOCK_REASON))
4214 .await
4215 .unwrap();
4216 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z'")
4217 .execute(store.pool())
4218 .await
4219 .unwrap();
4220
4221 refire_and_escalate_stale_needs_review_global(
4222 &task_manager,
4223 &store,
4224 &router,
4225 &dispatching,
4226 &active,
4227 )
4228 .await
4229 .unwrap();
4230 auto_recover_rebroadcast_blocked_tasks_global(&router, &task_manager, &store, &active)
4231 .await
4232 .unwrap();
4233
4234 let review = store.get(review_id).await.unwrap();
4235 assert_eq!(review.status, TaskStatus::NeedsReview);
4236 assert_eq!(review.needs_review_refires, 0);
4237 let blocked = store.get(blocked_id).await.unwrap();
4238 assert_eq!(blocked.status, TaskStatus::Blocked);
4239 assert_eq!(
4240 blocked.block_reason.as_deref(),
4241 Some(REBROADCAST_BLOCK_REASON)
4242 );
4243 }
4244
4245 fn router_with_healthy_review_agent() -> Router {
4248 let mut config = RouterConfig::default();
4249 config
4250 .model_map
4251 .entry("review".to_string())
4252 .or_default()
4253 .insert("claude".to_string(), vec!["sonnet".to_string()]);
4254 let mut router = Router::new(config);
4255 router.available_agents = vec!["claude".to_string()];
4256 router
4257 }
4258
4259 #[serial(cooldown_state)]
4264 #[tokio::test]
4265 async fn global_refire_sweep_refires_stale_task_from_inactive_repo() {
4266 cooldown::reset_global_state().await;
4267 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4268 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4269 let task_manager = Arc::new(TaskManager::with_store(
4271 backend,
4272 Arc::clone(&store),
4273 "active/repo".to_string(),
4274 ));
4275 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4276 let router = router_with_healthy_review_agent();
4277
4278 let id = store
4279 .upsert_external(&crate::store::UpsertExternal {
4280 repo: "inactive/repo",
4281 ext_id: "490",
4282 title: "Stale needs_review from inactive repo",
4283 body: "",
4284 author: "",
4285 url: "",
4286 labels: &[],
4287 origin: "github",
4288 })
4289 .await
4290 .unwrap();
4291 store
4292 .update_status(id, TaskStatus::NeedsReview)
4293 .await
4294 .unwrap();
4295 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4296 .bind(id)
4297 .execute(store.pool())
4298 .await
4299 .unwrap();
4300
4301 refire_and_escalate_stale_needs_review_global(
4302 &task_manager,
4303 &store,
4304 &router,
4305 &dispatching,
4306 &sweep_repos(),
4307 )
4308 .await
4309 .unwrap();
4310
4311 let task = store.get(id).await.unwrap();
4312 assert_eq!(
4313 task.needs_review_refires, 1,
4314 "cross-repo task must be refired"
4315 );
4316 assert_eq!(task.status, TaskStatus::NeedsReview);
4317 }
4318
4319 #[serial(cooldown_state)]
4322 #[tokio::test]
4323 async fn global_refire_sweep_escalates_cross_repo_task_to_blocked() {
4324 cooldown::reset_global_state().await;
4325 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4326 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4327 let task_manager = Arc::new(TaskManager::with_store(
4328 backend,
4329 Arc::clone(&store),
4330 "active/repo".to_string(),
4331 ));
4332 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4333 let router = router_with_healthy_review_agent();
4334
4335 let id = store
4336 .upsert_external(&crate::store::UpsertExternal {
4337 repo: "inactive/repo",
4338 ext_id: "493",
4339 title: "Exhausted refires from inactive repo",
4340 body: "",
4341 author: "",
4342 url: "",
4343 labels: &[],
4344 origin: "github",
4345 })
4346 .await
4347 .unwrap();
4348 store
4349 .update_status(id, TaskStatus::NeedsReview)
4350 .await
4351 .unwrap();
4352 store
4353 .set_fields(id, &[("needs_review_refires", serde_json::json!(5))])
4354 .await
4355 .unwrap();
4356 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4357 .bind(id)
4358 .execute(store.pool())
4359 .await
4360 .unwrap();
4361
4362 refire_and_escalate_stale_needs_review_global(
4363 &task_manager,
4364 &store,
4365 &router,
4366 &dispatching,
4367 &sweep_repos(),
4368 )
4369 .await
4370 .unwrap();
4371
4372 let task = store.get(id).await.unwrap();
4373 assert_eq!(task.status, TaskStatus::Blocked);
4374 assert_eq!(
4375 task.block_reason.as_deref(),
4376 Some("review agent rebroadcast escalated after repeated retries")
4377 );
4378 }
4379
4380 #[serial(cooldown_state)]
4385 #[tokio::test]
4386 async fn global_refire_sweep_skips_when_all_review_agents_cooled() {
4387 cooldown::reset_global_state().await;
4388 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4389 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4390 let task_manager = Arc::new(TaskManager::with_store(
4391 backend,
4392 Arc::clone(&store),
4393 "active/repo".to_string(),
4394 ));
4395 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4396 let router = Router::new_for_test(RouterConfig::default(), vec!["claude".to_string()]);
4399
4400 let id = store
4401 .upsert_external(&crate::store::UpsertExternal {
4402 repo: "inactive/repo",
4403 ext_id: "494",
4404 title: "Stale needs_review while agents cooled",
4405 body: "",
4406 author: "",
4407 url: "",
4408 labels: &[],
4409 origin: "github",
4410 })
4411 .await
4412 .unwrap();
4413 store
4414 .update_status(id, TaskStatus::NeedsReview)
4415 .await
4416 .unwrap();
4417 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4418 .bind(id)
4419 .execute(store.pool())
4420 .await
4421 .unwrap();
4422
4423 refire_and_escalate_stale_needs_review_global(
4424 &task_manager,
4425 &store,
4426 &router,
4427 &dispatching,
4428 &sweep_repos(),
4429 )
4430 .await
4431 .unwrap();
4432
4433 let task = store.get(id).await.unwrap();
4434 assert_eq!(
4435 task.needs_review_refires, 0,
4436 "counter must not be touched while cooled"
4437 );
4438 assert_eq!(task.status, TaskStatus::NeedsReview);
4439 }
4440
4441 #[serial(cooldown_state)]
4444 #[tokio::test]
4445 async fn global_refire_sweep_skips_when_no_reviewer_available() {
4446 cooldown::reset_global_state().await;
4447 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4448 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4449 let task_manager = Arc::new(TaskManager::with_store(
4450 backend,
4451 Arc::clone(&store),
4452 "active/repo".to_string(),
4453 ));
4454 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4455 let router = Router::new_for_test(RouterConfig::default(), vec![]);
4456
4457 let id = store
4458 .upsert_external(&crate::store::UpsertExternal {
4459 repo: "some/repo",
4460 ext_id: "495",
4461 title: "needs_review with no reviewer",
4462 body: "",
4463 author: "",
4464 url: "",
4465 labels: &[],
4466 origin: "github",
4467 })
4468 .await
4469 .unwrap();
4470 store
4471 .update_status(id, TaskStatus::NeedsReview)
4472 .await
4473 .unwrap();
4474 store
4475 .set_fields(id, &[("needs_review_refires", serde_json::json!(5))])
4476 .await
4477 .unwrap();
4478 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4479 .bind(id)
4480 .execute(store.pool())
4481 .await
4482 .unwrap();
4483
4484 refire_and_escalate_stale_needs_review_global(
4485 &task_manager,
4486 &store,
4487 &router,
4488 &dispatching,
4489 &sweep_repos(),
4490 )
4491 .await
4492 .unwrap();
4493
4494 let task = store.get(id).await.unwrap();
4495 assert_eq!(task.status, TaskStatus::NeedsReview);
4496 assert_eq!(task.needs_review_refires, 5);
4497 }
4498
4499 #[serial(cooldown_state)]
4505 #[tokio::test]
4506 async fn auto_recover_rebroadcast_blocked_task_after_review_cycle() {
4507 cooldown::reset_global_state().await;
4508 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4509 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4510 let task_manager = Arc::new(TaskManager::with_store(
4511 backend,
4512 Arc::clone(&store),
4513 "owner/repo".to_string(),
4514 ));
4515 let router = router_with_healthy_review_agent();
4516
4517 let id = store
4518 .upsert_external(&crate::store::UpsertExternal {
4519 repo: "owner/repo",
4520 ext_id: "511",
4521 title: "Rebroadcast blocked after review",
4522 body: "",
4523 author: "",
4524 url: "",
4525 labels: &[],
4526 origin: "github",
4527 })
4528 .await
4529 .unwrap();
4530 store.update_status(id, TaskStatus::Blocked).await.unwrap();
4531 store
4532 .set_block_reason(
4533 id,
4534 Some("review agent rebroadcast escalated after repeated retries"),
4535 )
4536 .await
4537 .unwrap();
4538 store
4539 .set_fields(id, &[("review_cycles", serde_json::json!(1))])
4540 .await
4541 .unwrap();
4542 store
4543 .set_fields(id, &[("needs_review_refires", serde_json::json!(6))])
4544 .await
4545 .unwrap();
4546 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4547 .bind(id)
4548 .execute(store.pool())
4549 .await
4550 .unwrap();
4551
4552 auto_recover_rebroadcast_blocked_tasks_global(
4553 &router,
4554 &task_manager,
4555 &store,
4556 &sweep_repos(),
4557 )
4558 .await
4559 .unwrap();
4560
4561 let task = store.get(id).await.unwrap();
4562 assert_eq!(
4563 task.status,
4564 TaskStatus::NeedsReview,
4565 "blocked rebroadcast task must be recovered to needs_review"
4566 );
4567 assert!(
4568 task.block_reason.is_none(),
4569 "block_reason must be cleared on recovery"
4570 );
4571 assert_eq!(
4572 task.needs_review_refires, 0,
4573 "needs_review_refires must be reset to 0 on recovery"
4574 );
4575 }
4576
4577 #[serial(cooldown_state)]
4580 #[tokio::test]
4581 async fn stale_in_review_recovery_skips_fresh_tasks() {
4582 cooldown::reset_global_state().await;
4583 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4584 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4585 let task_manager = Arc::new(TaskManager::with_store(
4586 Arc::clone(&backend),
4587 Arc::clone(&store),
4588 "owner/repo".to_string(),
4589 ));
4590 let tmux = Arc::new(TmuxManager::new());
4591 let router = Arc::new(RwLock::new(Router::new_for_test(
4592 RouterConfig::default(),
4593 vec!["claude".to_string()],
4594 )));
4595 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4596 let auto_merge_in_flight: Arc<DashSet<String>> = Arc::new(DashSet::new());
4597
4598 let id = store
4599 .upsert_external(&crate::store::UpsertExternal {
4600 repo: "owner/repo",
4601 ext_id: "66",
4602 title: "Fresh review",
4603 body: "",
4604 author: "",
4605 url: "",
4606 labels: &[],
4607 origin: "github",
4608 })
4609 .await
4610 .unwrap();
4611 store.update_status(id, TaskStatus::InReview).await.unwrap();
4612 store
4613 .set_fields(id, &[("branch", serde_json::json!("feature/66"))])
4614 .await
4615 .unwrap();
4616 Box::pin(sync_tick(
4619 &backend,
4620 &tmux,
4621 "owner/repo",
4622 &EngineConfig::default(),
4623 &router,
4624 &task_manager,
4625 &store,
4626 &dispatching,
4627 &auto_merge_in_flight,
4628 ))
4629 .await
4630 .unwrap();
4631
4632 let task = store.get(id).await.unwrap();
4634 assert_eq!(task.status, TaskStatus::InReview);
4635 }
4636
4637 #[serial(cooldown_state)]
4638 #[tokio::test]
4639 async fn auto_unblock_routes_recoverable_failure() {
4640 cooldown::reset_global_state().await;
4641 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4642 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4643 let task_manager = Arc::new(TaskManager::with_store(
4644 backend,
4645 Arc::clone(&store),
4646 "owner/repo".to_string(),
4647 ));
4648 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4649
4650 let id = store
4651 .upsert_external(&crate::store::UpsertExternal {
4652 repo: "owner/repo",
4653 ext_id: "77",
4654 title: "Blocked task",
4655 body: "",
4656 author: "",
4657 url: "",
4658 labels: &[],
4659 origin: "github",
4660 })
4661 .await
4662 .unwrap();
4663 store.update_status(id, TaskStatus::Blocked).await.unwrap();
4664 store
4665 .set_fields(
4666 id,
4667 &[
4668 ("agent", serde_json::json!("codex")),
4669 ("model", serde_json::json!("gpt-5")),
4670 ],
4671 )
4672 .await
4673 .unwrap();
4674
4675 let run_id = store
4676 .start_run(&crate::store::StartRun {
4677 task_id: id,
4678 attempt: 1,
4679 run_type: "agent",
4680 agent: "codex",
4681 model: "gpt-5",
4682 command: "codex --model gpt-5",
4683 prompt: "do thing",
4684 })
4685 .await
4686 .unwrap();
4687 store
4688 .complete_run(&crate::store::CompleteRun {
4689 run_id,
4690 exit_code: Some(1),
4691 stdout: "",
4692 stderr: "",
4693 parsed: "",
4694 outcome: "rate_limit",
4695 error: "rate limit exceeded",
4696 tokens: RunTokenUsage::default(),
4697 })
4698 .await
4699 .unwrap();
4700
4701 auto_unblock_blocked_tasks(
4702 "owner/repo",
4703 &task_manager,
4704 &store,
4705 &dispatching,
4706 &Arc::new(DashSet::new()),
4707 )
4708 .await
4709 .unwrap();
4710
4711 let task = store.get(id).await.unwrap();
4712 assert_eq!(task.status, TaskStatus::New);
4713 assert!(task.agent.is_none());
4714 assert!(task.model.is_none());
4715 assert_eq!(task.auto_unblock_count, 1);
4716 assert_ne!(task.auto_unblock_last_at, "");
4717 assert_eq!(task.auto_unblock_last_reason, "RateLimit");
4718 }
4719
4720 #[serial(cooldown_state)]
4721 #[tokio::test]
4722 async fn auto_unblock_skips_manual_block() {
4723 cooldown::reset_global_state().await;
4724 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4725 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4726 let task_manager = Arc::new(TaskManager::with_store(
4727 backend,
4728 Arc::clone(&store),
4729 "owner/repo".to_string(),
4730 ));
4731 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4732
4733 let id = store
4734 .upsert_external(&crate::store::UpsertExternal {
4735 repo: "owner/repo",
4736 ext_id: "88",
4737 title: "Manually blocked",
4738 body: "",
4739 author: "",
4740 url: "",
4741 labels: &[],
4742 origin: "github",
4743 })
4744 .await
4745 .unwrap();
4746 store.update_status(id, TaskStatus::Blocked).await.unwrap();
4747 store
4748 .set_block_reason(id, Some("waiting on input"))
4749 .await
4750 .unwrap();
4751
4752 let run_id = store
4753 .start_run(&crate::store::StartRun {
4754 task_id: id,
4755 attempt: 1,
4756 run_type: "agent",
4757 agent: "codex",
4758 model: "gpt-5",
4759 command: "codex --model gpt-5",
4760 prompt: "do thing",
4761 })
4762 .await
4763 .unwrap();
4764 store
4765 .complete_run(&crate::store::CompleteRun {
4766 run_id,
4767 exit_code: Some(1),
4768 stdout: "",
4769 stderr: "",
4770 parsed: "",
4771 outcome: "rate_limit",
4772 error: "rate limit exceeded",
4773 tokens: RunTokenUsage::default(),
4774 })
4775 .await
4776 .unwrap();
4777
4778 auto_unblock_blocked_tasks(
4779 "owner/repo",
4780 &task_manager,
4781 &store,
4782 &dispatching,
4783 &Arc::new(DashSet::new()),
4784 )
4785 .await
4786 .unwrap();
4787
4788 let task = store.get(id).await.unwrap();
4789 assert_eq!(task.status, TaskStatus::Blocked);
4790 assert_eq!(task.auto_unblock_count, 0);
4791 }
4792
4793 #[serial(cooldown_state)]
4796 #[test]
4797 fn failure_category_as_str() {
4798 assert_eq!(FailureCategory::RateLimit.as_str(), "RateLimit");
4801 assert_eq!(FailureCategory::Timeout.as_str(), "Timeout");
4802 assert_eq!(
4803 FailureCategory::ModelUnavailable.as_str(),
4804 "ModelUnavailable"
4805 );
4806 assert_eq!(FailureCategory::MaxAttempts.as_str(), "MaxAttempts");
4807 assert_eq!(FailureCategory::Unknown.as_str(), "Unknown");
4808 assert_eq!(FailureCategory::SilentExit0.as_str(), "SilentExit0");
4809 assert_eq!(FailureCategory::ConnectionError.as_str(), "ConnectionError");
4810 assert_eq!(FailureCategory::CliFlagError.as_str(), "CliFlagError");
4811 assert_eq!(
4812 FailureCategory::AllAgentsExhausted.as_str(),
4813 "AllAgentsExhausted"
4814 );
4815 assert_eq!(FailureCategory::ParseError.as_str(), "ParseError");
4816 assert_eq!(FailureCategory::FalseFailure.as_str(), "FalseFailure");
4817 assert_eq!(FailureCategory::PushFailed.as_str(), "PushFailed");
4818 assert_eq!(FailureCategory::PrCreateFailed.as_str(), "PrCreateFailed");
4819 }
4820
4821 #[serial(cooldown_state)]
4824 #[test]
4825 fn classify_failure_model_unavailable_phrase_is_recoverable() {
4826 let category = classify_failure(
4827 "model unavailable (anthropic/claude-sonnet-4-6): Model not found: anthropic/claude-sonnet-4-6",
4828 "error",
4829 );
4830 assert_eq!(
4831 category,
4832 FailureCategory::ModelUnavailable,
4833 "\"model unavailable\" error must classify as ModelUnavailable"
4834 );
4835 assert!(
4836 category.is_recoverable(),
4837 "ModelUnavailable must be recoverable so auto-unblock can re-route"
4838 );
4839 }
4840
4841 #[serial(cooldown_state)]
4843 #[test]
4844 fn classify_failure_kimi_billing_cycle_is_credit_exhausted_not_rate_limit() {
4845 let error = "kimi rate limit: Failed to authenticate. API Error: 403 You've reached \
4852 your usage limit for this billing cycle. Your quota will be refreshed soon.";
4853 let category = classify_failure(error, "rate_limit");
4854 assert_eq!(
4855 category,
4856 FailureCategory::CreditExhausted,
4857 "billing-cycle exhaustion must classify as CreditExhausted, not RateLimit"
4858 );
4859 assert!(category.is_recoverable());
4860 }
4861
4862 #[serial(cooldown_state)]
4863 #[test]
4864 fn classify_failure_billing_cycle_exhausted_outcome_is_credit_exhausted() {
4865 let category = classify_failure("some billing message", "billing_cycle_exhausted");
4868 assert_eq!(category, FailureCategory::CreditExhausted);
4869 }
4870
4871 #[serial(cooldown_state)]
4872 #[test]
4873 fn classify_failure_generic_rate_limit_is_still_rate_limit() {
4874 let category = classify_failure("429 Too Many Requests", "rate_limit");
4877 assert_eq!(category, FailureCategory::RateLimit);
4878 }
4879
4880 #[serial(cooldown_state)]
4882 #[test]
4883 fn classify_failure_pr_create_failed_specific_patterns() {
4884 assert_eq!(
4886 classify_failure("create pr failed", ""),
4887 FailureCategory::PrCreateFailed
4888 );
4889 assert_eq!(
4890 classify_failure("failed to create pull request", ""),
4891 FailureCategory::PrCreateFailed
4892 );
4893 assert_eq!(
4894 classify_failure("pull request creation failed", ""),
4895 FailureCategory::PrCreateFailed
4896 );
4897
4898 assert_ne!(
4900 classify_failure("network failed while fetching pull request status", ""),
4901 FailureCategory::PrCreateFailed
4902 );
4903 assert_ne!(
4904 classify_failure("connection failed, retrying pull request check", ""),
4905 FailureCategory::PrCreateFailed
4906 );
4907 assert_ne!(
4908 classify_failure("pull request created but failed to merge", ""),
4909 FailureCategory::PrCreateFailed
4910 );
4911 }
4912
4913 #[serial(cooldown_state)]
4914 #[test]
4915 fn classify_failure_model_not_found_is_recoverable() {
4916 let category = classify_failure("Model not found: gpt-5-ultra", "error");
4917 assert_eq!(
4918 category,
4919 FailureCategory::ModelUnavailable,
4920 "\"model not found\" error must classify as ModelUnavailable"
4921 );
4922 assert!(category.is_recoverable());
4923 }
4924
4925 #[serial(cooldown_state)]
4926 #[test]
4927 fn classify_failure_model_does_not_exist_is_recoverable() {
4928 let category = classify_failure("The model `claude-opus-99` does not exist", "error");
4929 assert_eq!(category, FailureCategory::ModelUnavailable);
4930 assert!(category.is_recoverable());
4931 }
4932
4933 #[serial(cooldown_state)]
4934 #[tokio::test]
4935 async fn auto_unblock_routes_task_blocked_by_model_unavailable() {
4936 cooldown::reset_global_state().await;
4937 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4938 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4939 let task_manager = Arc::new(TaskManager::with_store(
4940 backend,
4941 Arc::clone(&store),
4942 "owner/repo".to_string(),
4943 ));
4944 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4945
4946 let id = store
4947 .upsert_external(&crate::store::UpsertExternal {
4948 repo: "owner/repo",
4949 ext_id: "1202",
4950 title: "Task blocked by model unavailable",
4951 body: "",
4952 author: "",
4953 url: "",
4954 labels: &[],
4955 origin: "github",
4956 })
4957 .await
4958 .unwrap();
4959 store.update_status(id, TaskStatus::Blocked).await.unwrap();
4960
4961 let run_id = store
4962 .start_run(&crate::store::StartRun {
4963 task_id: id,
4964 attempt: 1,
4965 run_type: "agent",
4966 agent: "opencode",
4967 model: "anthropic/claude-sonnet-4-6",
4968 command: "opencode",
4969 prompt: "fix the bug",
4970 })
4971 .await
4972 .unwrap();
4973 store
4974 .complete_run(&crate::store::CompleteRun {
4975 run_id,
4976 exit_code: Some(1),
4977 stdout: "",
4978 stderr: "",
4979 parsed: "",
4980 outcome: "error",
4981 error: "model unavailable (anthropic/claude-sonnet-4-6): Model not found: anthropic/claude-sonnet-4-6",
4982 tokens: RunTokenUsage::default(),
4983 })
4984 .await
4985 .unwrap();
4986
4987 auto_unblock_blocked_tasks(
4988 "owner/repo",
4989 &task_manager,
4990 &store,
4991 &dispatching,
4992 &Arc::new(DashSet::new()),
4993 )
4994 .await
4995 .unwrap();
4996
4997 let task = store.get(id).await.unwrap();
4998 assert_eq!(
4999 task.status,
5000 TaskStatus::New,
5001 "task blocked by model unavailable must be auto-unblocked and re-routed"
5002 );
5003 assert_eq!(task.auto_unblock_count, 1);
5004 assert_eq!(task.auto_unblock_last_reason, "ModelUnavailable");
5005 }
5006
5007 #[serial(cooldown_state)]
5010 #[tokio::test]
5011 async fn auto_unblock_resets_count_when_reason_changes() {
5012 cooldown::reset_global_state().await;
5013 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5017 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
5018 let task_manager = Arc::new(TaskManager::with_store(
5019 backend,
5020 Arc::clone(&store),
5021 "owner/repo".to_string(),
5022 ));
5023 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
5024
5025 let id = store
5026 .upsert_external(&crate::store::UpsertExternal {
5027 repo: "owner/repo",
5028 ext_id: "1227",
5029 title: "Task with different failure reason",
5030 body: "",
5031 author: "",
5032 url: "",
5033 labels: &[],
5034 origin: "github",
5035 })
5036 .await
5037 .unwrap();
5038 store.update_status(id, TaskStatus::Blocked).await.unwrap();
5039 store
5041 .set_fields(
5042 id,
5043 &[
5044 ("auto_unblock_count", serde_json::json!(1)),
5045 (
5046 "auto_unblock_last_reason",
5047 serde_json::json!("ModelUnavailable"),
5048 ),
5049 ],
5050 )
5051 .await
5052 .unwrap();
5053
5054 let run_id = store
5056 .start_run(&crate::store::StartRun {
5057 task_id: id,
5058 attempt: 2,
5059 run_type: "agent",
5060 agent: "claude",
5061 model: "opus",
5062 command: "claude",
5063 prompt: "fix the bug",
5064 })
5065 .await
5066 .unwrap();
5067 store
5068 .complete_run(&crate::store::CompleteRun {
5069 run_id,
5070 exit_code: Some(1),
5071 stdout: "",
5072 stderr: "",
5073 parsed: "",
5074 outcome: "error",
5075 error: "rate limit reached for model opus",
5076 tokens: RunTokenUsage::default(),
5077 })
5078 .await
5079 .unwrap();
5080
5081 auto_unblock_blocked_tasks(
5082 "owner/repo",
5083 &task_manager,
5084 &store,
5085 &dispatching,
5086 &Arc::new(DashSet::new()),
5087 )
5088 .await
5089 .unwrap();
5090
5091 let task = store.get(id).await.unwrap();
5092 assert_eq!(
5093 task.status,
5094 TaskStatus::New,
5095 "task must be auto-unblocked even with count=1 from a different reason"
5096 );
5097 assert_eq!(task.auto_unblock_count, 1);
5099 assert_eq!(
5100 task.auto_unblock_last_reason, "RateLimit",
5101 "reason must be updated to the new failure category"
5102 );
5103 }
5104
5105 #[serial(cooldown_state)]
5111 #[tokio::test]
5112 async fn auto_unblock_leaves_max_attempts_blocked_task_alone() {
5113 cooldown::reset_global_state().await;
5114 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5115 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
5116 let task_manager = Arc::new(TaskManager::with_store(
5117 backend,
5118 Arc::clone(&store),
5119 "owner/repo".to_string(),
5120 ));
5121 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
5122
5123 let id = store
5124 .upsert_external(&crate::store::UpsertExternal {
5125 repo: "owner/repo",
5126 ext_id: "3783",
5127 title: "Approved work sent back to the agent",
5128 body: "",
5129 author: "",
5130 url: "",
5131 labels: &[],
5132 origin: "github",
5133 })
5134 .await
5135 .unwrap();
5136 store.update_status(id, TaskStatus::Blocked).await.unwrap();
5137
5138 let run_id = store
5139 .start_run(&crate::store::StartRun {
5140 task_id: id,
5141 attempt: 11,
5142 run_type: "agent",
5143 agent: "claude",
5144 model: "opus",
5145 command: "claude",
5146 prompt: "fix the bug",
5147 })
5148 .await
5149 .unwrap();
5150 store
5151 .complete_run(&crate::store::CompleteRun {
5152 run_id,
5153 exit_code: Some(1),
5154 stdout: "",
5155 stderr: "",
5156 parsed: "",
5157 outcome: "error",
5158 error: "exceeded max attempts (11/10). Use `/retry` to reset.",
5159 tokens: RunTokenUsage::default(),
5160 })
5161 .await
5162 .unwrap();
5163
5164 auto_unblock_blocked_tasks(
5165 "owner/repo",
5166 &task_manager,
5167 &store,
5168 &dispatching,
5169 &Arc::new(DashSet::new()),
5170 )
5171 .await
5172 .unwrap();
5173
5174 let task = store.get(id).await.unwrap();
5175 assert_eq!(
5176 task.status,
5177 TaskStatus::Blocked,
5178 "max-attempts block must survive the next tick until a person acts"
5179 );
5180 assert_eq!(
5181 task.auto_unblock_count, 0,
5182 "no auto-unblock attempt should be recorded"
5183 );
5184 }
5185
5186 #[serial(cooldown_state)]
5187 #[tokio::test]
5188 async fn auto_unblock_stores_reason_on_first_unblock() {
5189 cooldown::reset_global_state().await;
5190 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5193 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
5194 let task_manager = Arc::new(TaskManager::with_store(
5195 backend,
5196 Arc::clone(&store),
5197 "owner/repo".to_string(),
5198 ));
5199 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
5200
5201 let id = store
5202 .upsert_external(&crate::store::UpsertExternal {
5203 repo: "owner/repo",
5204 ext_id: "1227b",
5205 title: "Task for first-time reason storage",
5206 body: "",
5207 author: "",
5208 url: "",
5209 labels: &[],
5210 origin: "github",
5211 })
5212 .await
5213 .unwrap();
5214 store.update_status(id, TaskStatus::Blocked).await.unwrap();
5215
5216 let run_id = store
5217 .start_run(&crate::store::StartRun {
5218 task_id: id,
5219 attempt: 1,
5220 run_type: "agent",
5221 agent: "claude",
5222 model: "opus",
5223 command: "claude",
5224 prompt: "fix the bug",
5225 })
5226 .await
5227 .unwrap();
5228 store
5229 .complete_run(&crate::store::CompleteRun {
5230 run_id,
5231 exit_code: Some(1),
5232 stdout: "",
5233 stderr: "",
5234 parsed: "",
5235 outcome: "timeout",
5236 error: "task timed out",
5237 tokens: RunTokenUsage::default(),
5238 })
5239 .await
5240 .unwrap();
5241
5242 auto_unblock_blocked_tasks(
5243 "owner/repo",
5244 &task_manager,
5245 &store,
5246 &dispatching,
5247 &Arc::new(DashSet::new()),
5248 )
5249 .await
5250 .unwrap();
5251
5252 let task = store.get(id).await.unwrap();
5253 assert_eq!(task.status, TaskStatus::New);
5254 assert_eq!(task.auto_unblock_count, 1);
5255 assert_eq!(
5256 task.auto_unblock_last_reason, "Timeout",
5257 "reason must be stored on first auto-unblock"
5258 );
5259 }
5260
5261 #[serial(cooldown_state)]
5264 #[test]
5265 fn classify_failure_sparse_checkout_not_parse_error() {
5266 let category = classify_failure("error: cannot reset paths in a sparse checkout", "error");
5269 assert_ne!(
5270 category,
5271 FailureCategory::ParseError,
5272 "sparse checkout errors must NOT be classified as ParseError"
5273 );
5274 assert_eq!(
5275 category,
5276 FailureCategory::Unknown,
5277 "sparse checkout errors should be Unknown (not recoverable auto-unblock)"
5278 );
5279 }
5280
5281 #[serial(cooldown_state)]
5282 #[test]
5283 fn classify_failure_sparse_index_not_parse_error() {
5284 let category = classify_failure("failed due to sparse index configuration", "error");
5286 assert_ne!(category, FailureCategory::ParseError);
5287 assert_eq!(category, FailureCategory::Unknown);
5288 }
5289
5290 #[serial(cooldown_state)]
5297 #[test]
5298 fn classify_failure_truncated_is_recoverable() {
5299 let category = classify_failure(
5300 "truncated: review output was cut off before completion \
5301 (output/reasoning token budget exceeded)",
5302 "truncated",
5303 );
5304 assert_eq!(
5305 category,
5306 FailureCategory::ParseError,
5307 "truncated outcome must classify as a recoverable parse-style failure"
5308 );
5309 assert!(
5310 category.is_recoverable(),
5311 "truncated failures must be recoverable so auto-unblock can retry"
5312 );
5313 }
5314
5315 #[serial(cooldown_state)]
5316 #[test]
5317 fn dispatching_key_includes_repo_for_cross_project_isolation() {
5318 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
5319
5320 let key_a = "owner/repo-a/42".to_string();
5322 dispatching.insert(key_a.clone(), "42".to_string());
5323
5324 let key_b = "owner/repo-b/42".to_string();
5326 assert!(dispatching.contains_key(&key_a));
5327 assert!(
5328 !dispatching.contains_key(&key_b),
5329 "same task ID in different repo should not be blocked"
5330 );
5331 }
5332
5333 #[serial(cooldown_state)]
5334 #[tokio::test]
5335 async fn emit_degraded_agents_writes_metric_when_three_or_more_agents_degraded() {
5336 cooldown::reset_global_state().await;
5337
5338 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5339
5340 let mut config = RouterConfig::default();
5342 config
5343 .model_map
5344 .entry("medium".to_string())
5345 .or_default()
5346 .insert(
5347 "test-agent-3a".to_string(),
5348 vec!["test-model-a".to_string()],
5349 );
5350 config
5351 .model_map
5352 .entry("medium".to_string())
5353 .or_default()
5354 .insert(
5355 "test-agent-3b".to_string(),
5356 vec!["test-model-b".to_string()],
5357 );
5358 config
5359 .model_map
5360 .entry("medium".to_string())
5361 .or_default()
5362 .insert(
5363 "test-agent-3c".to_string(),
5364 vec!["test-model-c".to_string()],
5365 );
5366 config
5367 .model_map
5368 .entry("medium".to_string())
5369 .or_default()
5370 .insert(
5371 "test-agent-3d".to_string(),
5372 vec!["test-model-d".to_string()],
5373 );
5374 let mut router = Router::new(config);
5375 router.available_agents = vec![
5376 "test-agent-3a".to_string(),
5377 "test-agent-3b".to_string(),
5378 "test-agent-3c".to_string(),
5379 "test-agent-3d".to_string(),
5380 ];
5381
5382 set_agent_cooldown("test-agent-3a", 3600).await;
5384 set_agent_cooldown("test-agent-3b", 3600).await;
5385 set_agent_cooldown("test-agent-3c", 3600).await;
5386
5387 emit_degraded_agents_if_needed(&router.available_agents, &router.config, Some(&store))
5389 .await;
5390
5391 let count_val = store
5393 .kv_get("metrics:orch.agents_degraded.count")
5394 .await
5395 .unwrap();
5396 assert_eq!(count_val.as_deref(), Some("3"));
5397
5398 let alert_val = store
5400 .kv_get("metrics:orch.agents_degraded.alert")
5401 .await
5402 .unwrap();
5403 assert_eq!(alert_val.as_deref(), Some("1"));
5404 }
5405
5406 #[serial(cooldown_state)]
5407 #[tokio::test]
5408 async fn emit_degraded_agents_clears_alert_when_below_threshold() {
5409 cooldown::reset_global_state().await;
5410
5411 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5412
5413 let mut config = RouterConfig::default();
5414 config
5415 .model_map
5416 .entry("medium".to_string())
5417 .or_default()
5418 .insert(
5419 "test-agent-2a".to_string(),
5420 vec!["test-model-a".to_string()],
5421 );
5422 config
5423 .model_map
5424 .entry("medium".to_string())
5425 .or_default()
5426 .insert(
5427 "test-agent-2b".to_string(),
5428 vec!["test-model-b".to_string()],
5429 );
5430 let mut router = Router::new(config);
5431 router.available_agents = vec!["test-agent-2a".to_string(), "test-agent-2b".to_string()];
5432
5433 set_agent_cooldown("test-agent-2a", 3600).await;
5435 set_agent_cooldown("test-agent-2b", 3600).await;
5436
5437 emit_degraded_agents_if_needed(&router.available_agents, &router.config, Some(&store))
5438 .await;
5439
5440 let count_val = store
5442 .kv_get("metrics:orch.agents_degraded.count")
5443 .await
5444 .unwrap();
5445 assert_eq!(count_val.as_deref(), Some("2"));
5446
5447 let alert_val = store
5449 .kv_get("metrics:orch.agents_degraded.alert")
5450 .await
5451 .unwrap();
5452 assert_eq!(alert_val.as_deref(), Some("0"));
5453
5454 let stale_pool_alert = store
5455 .kv_get("metrics:orch.model_pool_stale_persistent.alert")
5456 .await
5457 .unwrap();
5458 assert_eq!(stale_pool_alert.as_deref(), Some("0"));
5459 }
5460
5461 #[serial(cooldown_state)]
5462 #[tokio::test]
5463 async fn emit_degraded_agents_sets_stale_pool_alert_for_persistent_model_failures() {
5464 cooldown::reset_global_state().await;
5465
5466 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5467
5468 let mut config = RouterConfig::default();
5469 config
5470 .model_map
5471 .entry("medium".to_string())
5472 .or_default()
5473 .insert(
5474 "test-agent-stale".to_string(),
5475 vec!["model-a".to_string(), "model-b".to_string()],
5476 );
5477 let mut router = Router::new(config);
5478 router.available_agents = vec!["test-agent-stale".to_string()];
5479
5480 record_persistent_model_failure("test-agent-stale", "model-a").await;
5482
5483 emit_degraded_agents_if_needed(&router.available_agents, &router.config, Some(&store))
5484 .await;
5485
5486 let stale_pool_alert = store
5487 .kv_get("metrics:orch.model_pool_stale_persistent.alert")
5488 .await
5489 .unwrap();
5490 assert_eq!(stale_pool_alert.as_deref(), Some("1"));
5491 }
5492
5493 #[serial(cooldown_state)]
5496 #[test]
5497 fn classify_skip_already_processed() {
5498 let action = classify_comment(
5499 "@orch /retry",
5500 Some("https://api.github.com/repos/o/r/issues/1"),
5501 "@bot",
5502 true,
5503 );
5504 assert_eq!(action, CommentAction::Skip);
5505 }
5506
5507 #[serial(cooldown_state)]
5508 #[test]
5509 fn classify_skip_irrelevant_comment() {
5510 let action = classify_comment("just a regular comment", None, "@bot", false);
5511 assert_eq!(action, CommentAction::Skip);
5512 }
5513
5514 #[serial(cooldown_state)]
5515 #[test]
5516 fn classify_command_without_mention() {
5517 let action = classify_comment(
5518 "/retry",
5519 Some("https://api.github.com/repos/o/r/issues/42"),
5520 "@bot",
5521 false,
5522 );
5523 assert_eq!(
5524 action,
5525 CommentAction::ExecuteCommand {
5526 command: OwnerCommand::Retry,
5527 issue_num: "42".to_string(),
5528 }
5529 );
5530 }
5531
5532 #[serial(cooldown_state)]
5533 #[test]
5534 fn classify_command_with_mention() {
5535 let action = classify_comment(
5536 "@orch /close",
5537 Some("https://api.github.com/repos/o/r/issues/7"),
5538 "@bot",
5539 false,
5540 );
5541 assert_eq!(
5542 action,
5543 CommentAction::ExecuteCommandForMention {
5544 command: OwnerCommand::Close,
5545 issue_num: "7".to_string(),
5546 }
5547 );
5548 }
5549
5550 #[serial(cooldown_state)]
5551 #[test]
5552 fn classify_mention_without_command() {
5553 let action = classify_comment(
5554 "@orch can you look at this?",
5555 Some("https://api.github.com/repos/o/r/issues/99"),
5556 "@bot",
5557 false,
5558 );
5559 assert_eq!(
5560 action,
5561 CommentAction::CreateMentionTask {
5562 issue_num: Some("99".to_string()),
5563 }
5564 );
5565 }
5566
5567 #[serial(cooldown_state)]
5568 #[test]
5569 fn classify_mention_without_issue_url() {
5570 let action = classify_comment("@orch help me", None, "@bot", false);
5571 assert_eq!(action, CommentAction::CreateMentionTask { issue_num: None });
5572 }
5573
5574 #[serial(cooldown_state)]
5575 #[test]
5576 fn classify_command_no_mention_no_url_skips() {
5577 let action = classify_comment("/retry", None, "@bot", false);
5578 assert_eq!(action, CommentAction::Skip);
5579 }
5580
5581 #[serial(cooldown_state)]
5582 #[test]
5583 fn classify_mention_via_current_user() {
5584 let action = classify_comment(
5585 "@bot please help",
5586 Some("https://api.github.com/repos/o/r/issues/5"),
5587 "@bot",
5588 false,
5589 );
5590 assert_eq!(
5591 action,
5592 CommentAction::CreateMentionTask {
5593 issue_num: Some("5".to_string()),
5594 }
5595 );
5596 }
5597
5598 #[serial(cooldown_state)]
5599 #[test]
5600 fn classify_command_with_mention_no_url_creates_task() {
5601 let action = classify_comment("@orch /retry", None, "@bot", false);
5603 assert_eq!(action, CommentAction::CreateMentionTask { issue_num: None });
5604 }
5605
5606 #[serial(cooldown_state)]
5607 #[test]
5608 fn mention_cursor_stops_advancing_after_gap() {
5609 let mut cursor = MentionCursor::default();
5610 cursor.advance("2026-01-01T00:00:00Z");
5611 cursor.block_on_gap();
5612 cursor.advance("2026-01-02T00:00:00Z");
5613
5614 assert_eq!(
5615 cursor.into_last_safe_ts().as_deref(),
5616 Some("2026-01-01T00:00:00Z")
5617 );
5618 }
5619
5620 #[serial(cooldown_state)]
5621 #[test]
5622 fn mention_cursor_tracks_max_timestamp_until_gap() {
5623 let mut cursor = MentionCursor::default();
5624 cursor.advance("2026-01-01T00:00:00Z");
5625 cursor.advance("2026-01-01T00:05:00Z");
5626 cursor.block_on_gap();
5627 cursor.advance("2026-01-01T00:10:00Z");
5628
5629 assert_eq!(
5630 cursor.into_last_safe_ts().as_deref(),
5631 Some("2026-01-01T00:05:00Z")
5632 );
5633 }
5634
5635 struct AckTrackingBackend {
5639 ack_calls: Mutex<Vec<String>>,
5640 posted: Mutex<Vec<String>>,
5641 ack_failure: bool,
5642 inner: NoopBackend,
5643 }
5644
5645 impl AckTrackingBackend {
5646 fn new(ack_failure: bool) -> Arc<Self> {
5647 Arc::new(Self {
5648 ack_calls: Mutex::new(Vec::new()),
5649 posted: Mutex::new(Vec::new()),
5650 ack_failure,
5651 inner: NoopBackend,
5652 })
5653 }
5654
5655 fn fail_on_ack(&self) -> bool {
5656 self.ack_failure
5657 }
5658 }
5659
5660 #[async_trait]
5661 impl ExternalBackend for AckTrackingBackend {
5662 fn name(&self) -> &'static str {
5663 "ack-tracking"
5664 }
5665
5666 async fn acknowledge_mention(&self, comment_id: &str) -> anyhow::Result<()> {
5667 self.ack_calls.lock().await.push(comment_id.to_string());
5668 if self.fail_on_ack() {
5669 anyhow::bail!("network error: temporary failure posting reaction");
5671 }
5672 Ok(())
5673 }
5674
5675 async fn get_authenticated_user(&self) -> anyhow::Result<Option<String>> {
5677 Ok(Some("bot".into()))
5678 }
5679
5680 async fn get_mentions(&self, _since: &str) -> anyhow::Result<Vec<Mention>> {
5682 Ok(vec![])
5683 }
5684
5685 async fn create_task(&self, t: &str, b: &str, l: &[String]) -> anyhow::Result<ExternalId> {
5687 self.inner.create_task(t, b, l).await
5688 }
5689 async fn get_task(&self, id: &ExternalId) -> anyhow::Result<ExternalTask> {
5690 self.inner.get_task(id).await
5691 }
5692 async fn list_by_status(&self, s: Status) -> anyhow::Result<Vec<ExternalTask>> {
5693 self.inner.list_by_status(s).await
5694 }
5695 async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
5696 self.inner.list_routable().await
5697 }
5698 async fn post_comment(&self, id: &ExternalId, b: &str) -> anyhow::Result<()> {
5699 self.posted.lock().await.push(b.to_string());
5700 self.inner.post_comment(id, b).await
5701 }
5702 async fn set_labels(&self, id: &ExternalId, l: &[String]) -> anyhow::Result<()> {
5703 self.inner.set_labels(id, l).await
5704 }
5705 async fn remove_label(&self, id: &ExternalId, l: &str) -> anyhow::Result<()> {
5706 self.inner.remove_label(id, l).await
5707 }
5708 async fn get_sub_issues(&self, id: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
5709 self.inner.get_sub_issues(id).await
5710 }
5711 async fn create_sub_task(
5712 &self,
5713 p: &ExternalId,
5714 t: &str,
5715 b: &str,
5716 l: &[String],
5717 ) -> anyhow::Result<ExternalId> {
5718 self.inner.create_sub_task(p, t, b, l).await
5719 }
5720 async fn ensure_status_label(&self, l: &str) -> anyhow::Result<()> {
5721 self.inner.ensure_status_label(l).await
5722 }
5723 async fn has_open_issue_with_title(&self, t: &str, l: &str) -> anyhow::Result<bool> {
5724 self.inner.has_open_issue_with_title(t, l).await
5725 }
5726 async fn health_check(&self) -> anyhow::Result<()> {
5727 self.inner.health_check().await
5728 }
5729 async fn is_pr_merged(&self, b: &str) -> anyhow::Result<bool> {
5730 self.inner.is_pr_merged(b).await
5731 }
5732 async fn update_status(&self, id: &ExternalId, s: Status) -> anyhow::Result<()> {
5733 self.inner.update_status(id, s).await
5734 }
5735 }
5736
5737 #[serial(cooldown_state)]
5740 #[tokio::test]
5741 async fn create_mention_task_skips_on_ack_failure() {
5742 cooldown::reset_global_state().await;
5743 let backend: Arc<AckTrackingBackend> = AckTrackingBackend::new(true);
5744 let store: Arc<TaskStore> = Arc::new(TaskStore::open_memory().await.unwrap());
5745
5746 let initial_ts = "2026-01-01T00:00:00Z";
5748 store
5749 .kv_set("mentions_last_checked", initial_ts)
5750 .await
5751 .unwrap();
5752
5753 let mention = Mention {
5754 id: "mention-abc".into(),
5755 body: "@bot can you help?".into(),
5756 author: "alice".into(),
5757 created_at: "2026-01-02T00:00:00Z".into(),
5758 issue_url: Some("https://api.github.com/repos/owner/repo/issues/42".into()),
5759 };
5760
5761 let backend_trait: Arc<dyn ExternalBackend> = Arc::<AckTrackingBackend>::clone(&backend);
5762 let task_manager = Arc::new(TaskManager::new(Arc::clone(&backend_trait)));
5763
5764 scan_comments(
5765 &backend_trait,
5766 Some(&store),
5767 "owner/repo",
5768 &task_manager,
5769 Some(&[mention]),
5770 )
5771 .await
5772 .unwrap();
5773
5774 let calls = backend.ack_calls.lock().await;
5776 assert_eq!(
5777 &**calls,
5778 &["mention-abc"],
5779 "acknowledge_mention must be called"
5780 );
5781
5782 let all = store.list_all("owner/repo").await.unwrap();
5784 assert!(
5785 all.is_empty(),
5786 "no task should be created when acknowledge_mention fails"
5787 );
5788
5789 let cursor = store.kv_get("mentions_last_checked").await.unwrap();
5791 assert_eq!(
5792 cursor.as_deref(),
5793 Some(initial_ts),
5794 "cursor must not advance when acknowledge_mention fails"
5795 );
5796 }
5797
5798 #[serial(cooldown_state)]
5801 #[tokio::test]
5802 async fn execute_command_for_mention_skips_on_ack_failure() {
5803 cooldown::reset_global_state().await;
5804 let backend: Arc<AckTrackingBackend> = AckTrackingBackend::new(true);
5805 let store: Arc<TaskStore> = Arc::new(TaskStore::open_memory().await.unwrap());
5806
5807 let initial_ts = "2026-01-01T00:00:00Z";
5808 store
5809 .kv_set("mentions_last_checked", initial_ts)
5810 .await
5811 .unwrap();
5812
5813 let mention = Mention {
5815 id: "mention-def".into(),
5816 body: "@bot /close".into(),
5817 author: "alice".into(),
5818 created_at: "2026-01-02T00:00:00Z".into(),
5819 issue_url: Some("https://api.github.com/repos/owner/repo/issues/42".into()),
5820 };
5821
5822 let backend_trait: Arc<dyn ExternalBackend> = Arc::<AckTrackingBackend>::clone(&backend);
5823 let task_manager = Arc::new(TaskManager::new(Arc::clone(&backend_trait)));
5824
5825 scan_comments(
5826 &backend_trait,
5827 Some(&store),
5828 "owner/repo",
5829 &task_manager,
5830 Some(&[mention]),
5831 )
5832 .await
5833 .unwrap();
5834
5835 let calls = backend.ack_calls.lock().await;
5837 assert_eq!(&**calls, &["mention-def"]);
5838
5839 let all = store.list_all("owner/repo").await.unwrap();
5841 assert!(
5842 all.is_empty(),
5843 "no task should be created when acknowledge_mention fails for ExecuteCommandForMention"
5844 );
5845
5846 let cursor = store.kv_get("mentions_last_checked").await.unwrap();
5848 assert_eq!(
5849 cursor.as_deref(),
5850 Some(initial_ts),
5851 "cursor must not advance when acknowledge_mention fails for ExecuteCommandForMention"
5852 );
5853 }
5854
5855 #[derive(Default)]
5859 struct FlakyGh {
5860 get_issue_calls: AtomicUsize,
5861 }
5862
5863 #[async_trait]
5864 impl CommandGh for FlakyGh {
5865 async fn get_issue(&self, _repo: &str, number: &str) -> anyhow::Result<GitHubIssue> {
5866 if self.get_issue_calls.fetch_add(1, Ordering::SeqCst) == 0 {
5867 anyhow::bail!("network error: temporary failure fetching issue");
5868 }
5869 Ok(GitHubIssue {
5870 number: number.parse().unwrap_or(0),
5871 id: 1,
5872 title: "t".into(),
5873 body: None,
5874 state: "open".into(),
5875 labels: Vec::new(),
5876 user: GitHubUser {
5877 login: "alice".into(),
5878 },
5879 created_at: String::new(),
5880 updated_at: String::new(),
5881 html_url: String::new(),
5882 node_id: None,
5883 pull_request: None,
5884 author_association: Some("OWNER".into()),
5885 state_reason: None,
5886 })
5887 }
5888
5889 async fn is_collaborator(&self, _repo: &str, _username: &str) -> anyhow::Result<bool> {
5890 Ok(true)
5891 }
5892
5893 async fn close_issue(&self, _repo: &str, _number: &str) -> anyhow::Result<()> {
5894 Ok(())
5895 }
5896 }
5897
5898 #[serial(cooldown_state)]
5901 #[tokio::test]
5902 async fn slash_command_retries_after_transient_fetch_failure() {
5903 cooldown::reset_global_state().await;
5904 let backend: Arc<AckTrackingBackend> = AckTrackingBackend::new(false);
5905 let store: Arc<TaskStore> = Arc::new(TaskStore::open_memory().await.unwrap());
5906 let initial_ts = "2026-01-01T00:00:00Z";
5907
5908 let mention = Mention {
5909 id: "mention-ghi".into(),
5910 body: "@bot /unblock".into(),
5911 author: "alice".into(),
5912 created_at: "2026-01-02T00:00:00Z".into(),
5913 issue_url: Some("https://api.github.com/repos/owner/repo/issues/42".into()),
5914 };
5915
5916 let backend_trait: Arc<dyn ExternalBackend> = Arc::<AckTrackingBackend>::clone(&backend);
5917 let task_manager = Arc::new(TaskManager::new(Arc::clone(&backend_trait)));
5918 let gh = FlakyGh::default();
5919 let mut cursor = MentionCursor::default();
5920
5921 let advanced_safely = handle_slash_command(
5923 &backend_trait,
5924 Some(&store),
5925 "owner/repo",
5926 &task_manager,
5927 &gh,
5928 &mention,
5929 &OwnerCommand::Unblock,
5930 &mut cursor,
5931 )
5932 .await
5933 .unwrap();
5934
5935 assert!(
5936 !advanced_safely,
5937 "a transient fetch failure must stop cursor advancement"
5938 );
5939 assert!(
5940 cursor.last_safe_ts.is_none(),
5941 "cursor must stay before the comment so the next tick retries it"
5942 );
5943 assert!(
5944 store.list_all("owner/repo").await.unwrap().is_empty(),
5945 "no mention task on a transient failure"
5946 );
5947 assert!(
5948 backend.posted.lock().await.is_empty(),
5949 "nothing is posted to GitHub on a transient failure"
5950 );
5951 assert_eq!(
5952 filter_mentions_by_since(slice::from_ref(&mention), initial_ts).len(),
5953 1,
5954 "the comment is still in range for the next scan"
5955 );
5956
5957 let mut cursor = MentionCursor::default();
5959 let advanced_safely = handle_slash_command(
5960 &backend_trait,
5961 Some(&store),
5962 "owner/repo",
5963 &task_manager,
5964 &gh,
5965 &mention,
5966 &OwnerCommand::Unblock,
5967 &mut cursor,
5968 )
5969 .await
5970 .unwrap();
5971
5972 assert!(advanced_safely);
5973 assert_eq!(
5974 cursor.into_last_safe_ts().as_deref(),
5975 Some("2026-01-02T00:00:00Z"),
5976 "cursor advances once the command actually runs"
5977 );
5978 assert_eq!(
5979 backend.posted.lock().await.len(),
5980 1,
5981 "the command runs once, posting a single confirmation"
5982 );
5983 assert_eq!(
5984 store.list_all("owner/repo").await.unwrap().len(),
5985 1,
5986 "one sentinel mention task for the executed command"
5987 );
5988 }
5989
5990 #[serial(cooldown_state)]
5993 #[tokio::test]
5994 async fn execute_command_keeps_cursor_on_fetch_failure() {
5995 cooldown::reset_global_state().await;
5996 let backend: Arc<AckTrackingBackend> = AckTrackingBackend::new(false);
5997 let store: Arc<TaskStore> = Arc::new(TaskStore::open_memory().await.unwrap());
5998
5999 let initial_ts = "2026-01-01T00:00:00Z";
6000 store
6001 .kv_set("mentions_last_checked", initial_ts)
6002 .await
6003 .unwrap();
6004
6005 let comment = Mention {
6007 id: "comment-jkl".into(),
6008 body: "/unblock".into(),
6009 author: "alice".into(),
6010 created_at: "2026-01-02T00:00:00Z".into(),
6011 issue_url: Some("https://api.github.com/repos/owner/repo/issues/42".into()),
6012 };
6013
6014 let backend_trait: Arc<dyn ExternalBackend> = Arc::<AckTrackingBackend>::clone(&backend);
6015 let task_manager = Arc::new(TaskManager::new(Arc::clone(&backend_trait)));
6016
6017 scan_comments(
6018 &backend_trait,
6019 Some(&store),
6020 "owner/repo",
6021 &task_manager,
6022 Some(&[comment]),
6023 )
6024 .await
6025 .unwrap();
6026
6027 assert!(
6028 backend.posted.lock().await.is_empty(),
6029 "nothing is posted when the issue fetch fails"
6030 );
6031 assert_eq!(
6032 store
6033 .kv_get("mentions_last_checked")
6034 .await
6035 .unwrap()
6036 .as_deref(),
6037 Some(initial_ts),
6038 "cursor must not advance past a command that never ran"
6039 );
6040 }
6041
6042 fn make_task(block_reason: Option<String>) -> Task {
6045 Task {
6046 id: 1,
6047 external_id: None,
6048 repo: "owner/repo".to_string(),
6049 origin: String::new(),
6050 title: String::new(),
6051 body: String::new(),
6052 status: TaskStatus::Blocked,
6053 source: String::new(),
6054 source_id: String::new(),
6055 author: String::new(),
6056 url: String::new(),
6057 labels: vec![],
6058 agent: None,
6059 model: None,
6060 complexity: String::new(),
6061 estimate: 0,
6062 route_reason: String::new(),
6063 agent_profile: String::new(),
6064 selected_skills: String::new(),
6065 route_attempts: 0,
6066 attempts: 0,
6067 branch: String::new(),
6068 worktree: String::new(),
6069 worktree_cleaned: false,
6070 owner_node: None,
6071 summary: String::new(),
6072 last_error: String::new(),
6073 parent_id: None,
6074 block_reason,
6075 pr_number: Some(42),
6076 pr_review_context: String::new(),
6077 last_review_ts: String::new(),
6078 review_ts_map: String::new(),
6079 last_comment_review_ts: String::new(),
6080 merge_conflict_retries: 0,
6081 ci_merge_failures: 0,
6082 pr_create_failures: 0,
6083 push_failures: 0,
6084 network_retries: 0,
6085 timeout_retries: 0,
6086 review_agent_failures: 0,
6087 review_resets: 0,
6088 review_cycles: 0,
6089 review_invocations: 0,
6090 needs_review_refires: 0,
6091 input_tokens: 0,
6092 output_tokens: 0,
6093 total_cost_usd: 0.0,
6094 model_reroute_chain: String::new(),
6095 limit_reroute_chain: String::new(),
6096 memory: vec![],
6097 delegations: vec![],
6098 auto_unblock_count: 0,
6099 auto_unblock_last_at: String::new(),
6100 auto_unblock_last_reason: String::new(),
6101 ci_recovery_count: 0,
6102 no_code_reroutes: 0,
6103 no_code_last_agent: String::new(),
6104 created_at: String::new(),
6105 updated_at: String::new(),
6106 }
6107 }
6108
6109 #[serial(cooldown_state)]
6110 #[test]
6111 fn is_ci_failure_block_matches_ci_failure_limit() {
6112 let task = make_task(Some(
6113 "CI failure limit (3) reached during auto-merge".to_string(),
6114 ));
6115 assert!(is_ci_failure_block(&task));
6116 }
6117
6118 #[serial(cooldown_state)]
6119 #[test]
6120 fn is_ci_failure_block_matches_ci_checks_timeout() {
6121 let task = make_task(Some(
6122 "CI checks timed out after 3 auto-merge failures".to_string(),
6123 ));
6124 assert!(is_ci_failure_block(&task));
6125 }
6126
6127 #[serial(cooldown_state)]
6128 #[test]
6129 fn is_ci_failure_block_false_for_regular_block() {
6130 let task = make_task(Some("waiting on input".to_string()));
6131 assert!(!is_ci_failure_block(&task));
6132 }
6133
6134 #[serial(cooldown_state)]
6135 #[test]
6136 fn is_ci_failure_block_false_for_none_reason() {
6137 let task = make_task(None);
6138 assert!(!is_ci_failure_block(&task));
6139 }
6140
6141 #[serial(cooldown_state)]
6144 #[test]
6145 fn ci_failure_cooldown_zero_count_immediate() {
6146 let task = make_task(None);
6148 assert!(ci_failure_unblock_cooldown_elapsed(&task));
6149 }
6150
6151 #[serial(cooldown_state)]
6152 #[test]
6153 fn ci_failure_cooldown_empty_last_at_immediate() {
6154 let mut task = make_task(None);
6156 task.auto_unblock_count = 1;
6157 assert!(ci_failure_unblock_cooldown_elapsed(&task));
6158 }
6159
6160 #[serial(cooldown_state)]
6161 #[test]
6162 fn ci_failure_cooldown_invalid_timestamp_immediate() {
6163 let mut task = make_task(None);
6165 task.auto_unblock_count = 1;
6166 task.auto_unblock_last_at = "not-a-timestamp".to_string();
6167 assert!(ci_failure_unblock_cooldown_elapsed(&task));
6168 }
6169
6170 #[serial(cooldown_state)]
6171 #[test]
6172 fn ci_failure_cooldown_high_count_still_eventually_elapses() {
6173 let long_ago = (chrono::Utc::now() - chrono::Duration::days(30)).to_rfc3339();
6176 for count in [3i32, 5, 99] {
6177 let mut task = make_task(None);
6178 task.auto_unblock_count = count;
6179 task.auto_unblock_last_at = long_ago.clone();
6180 assert!(
6181 ci_failure_unblock_cooldown_elapsed(&task),
6182 "count={count} should eventually elapse"
6183 );
6184 }
6185 }
6186
6187 #[serial(cooldown_state)]
6188 #[test]
6189 fn ci_failure_cooldown_recent_timestamp_still_blocks() {
6190 let just_now = chrono::Utc::now().to_rfc3339();
6192 for count in [1i32, 2, 3, 99] {
6193 let mut task = make_task(None);
6194 task.auto_unblock_count = count;
6195 task.auto_unblock_last_at = just_now.clone();
6196 assert!(
6197 !ci_failure_unblock_cooldown_elapsed(&task),
6198 "count={count} should still be cooling down"
6199 );
6200 }
6201 }
6202
6203 #[serial(cooldown_state)]
6206 #[tokio::test]
6207 async fn global_ci_failure_sweep_returns_ok_with_no_blocked_tasks() {
6208 cooldown::reset_global_state().await;
6209 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6210 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6211 let task_manager = Arc::new(TaskManager::with_store(
6212 backend,
6213 Arc::clone(&store),
6214 "owner/repo".to_string(),
6215 ));
6216 auto_unblock_ci_failure_blocked_tasks_global(
6217 &task_manager,
6218 &store,
6219 &Arc::new(DashSet::new()),
6220 )
6221 .await
6222 .unwrap();
6223 }
6224
6225 #[serial(cooldown_state)]
6226 #[tokio::test]
6227 async fn global_ci_failure_sweep_skips_tasks_in_cooldown() {
6228 cooldown::reset_global_state().await;
6231 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6232 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6233 let task_manager = Arc::new(TaskManager::with_store(
6234 backend,
6235 Arc::clone(&store),
6236 "active/repo".to_string(),
6237 ));
6238
6239 let id = store
6241 .upsert_external(&crate::store::UpsertExternal {
6242 repo: "inactive/repo",
6243 ext_id: "10",
6244 title: "Old blocked task",
6245 body: "",
6246 author: "",
6247 url: "",
6248 labels: &[],
6249 origin: "github",
6250 })
6251 .await
6252 .unwrap();
6253 store.update_status(id, TaskStatus::Blocked).await.unwrap();
6254 let just_now = chrono::Utc::now().to_rfc3339();
6255 store
6256 .set_fields(
6257 id,
6258 &[
6259 (
6260 "block_reason",
6261 serde_json::json!("CI failure limit reached during auto-merge"),
6262 ),
6263 ("pr_number", serde_json::json!(99)),
6264 ("auto_unblock_count", serde_json::json!(1)),
6265 ("auto_unblock_last_at", serde_json::json!(just_now)),
6266 ],
6267 )
6268 .await
6269 .unwrap();
6270
6271 auto_unblock_ci_failure_blocked_tasks_global(
6272 &task_manager,
6273 &store,
6274 &Arc::new(DashSet::new()),
6275 )
6276 .await
6277 .unwrap();
6278
6279 let task = store.get(id).await.unwrap();
6281 assert_eq!(
6282 task.auto_unblock_count, 1,
6283 "task in cooldown must not be processed"
6284 );
6285 }
6286
6287 #[serial(cooldown_state)]
6288 #[tokio::test]
6289 async fn global_ci_failure_sweep_ignores_non_ci_failure_blocks() {
6290 cooldown::reset_global_state().await;
6291 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6292 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6293 let task_manager = Arc::new(TaskManager::with_store(
6294 backend,
6295 Arc::clone(&store),
6296 "active/repo".to_string(),
6297 ));
6298
6299 let id = store
6301 .upsert_external(&crate::store::UpsertExternal {
6302 repo: "inactive/repo",
6303 ext_id: "20",
6304 title: "Manual block",
6305 body: "",
6306 author: "",
6307 url: "",
6308 labels: &[],
6309 origin: "github",
6310 })
6311 .await
6312 .unwrap();
6313 store.update_status(id, TaskStatus::Blocked).await.unwrap();
6314 store
6315 .set_fields(
6316 id,
6317 &[("block_reason", serde_json::json!("waiting for human input"))],
6318 )
6319 .await
6320 .unwrap();
6321
6322 auto_unblock_ci_failure_blocked_tasks_global(
6323 &task_manager,
6324 &store,
6325 &Arc::new(DashSet::new()),
6326 )
6327 .await
6328 .unwrap();
6329
6330 let task = store.get(id).await.unwrap();
6332 assert_eq!(task.status, TaskStatus::Blocked);
6333 }
6334
6335 #[serial(cooldown_state)]
6338 #[tokio::test]
6339 async fn global_rebroadcast_sweep_returns_ok_with_no_blocked_tasks() {
6340 cooldown::reset_global_state().await;
6341 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6342 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6343 let task_manager = Arc::new(TaskManager::with_store(
6344 backend,
6345 Arc::clone(&store),
6346 "owner/repo".to_string(),
6347 ));
6348 let router = router_with_healthy_review_agent();
6349 auto_recover_rebroadcast_blocked_tasks_global(
6350 &router,
6351 &task_manager,
6352 &store,
6353 &sweep_repos(),
6354 )
6355 .await
6356 .unwrap();
6357 }
6358
6359 #[serial(cooldown_state)]
6360 #[tokio::test]
6361 async fn global_rebroadcast_sweep_ignores_non_rebroadcast_blocks() {
6362 cooldown::reset_global_state().await;
6363 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6364 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6365 let task_manager = Arc::new(TaskManager::with_store(
6366 backend,
6367 Arc::clone(&store),
6368 "active/repo".to_string(),
6369 ));
6370 let router = router_with_healthy_review_agent();
6371
6372 let id = store
6373 .upsert_external(&crate::store::UpsertExternal {
6374 repo: "inactive/repo",
6375 ext_id: "30",
6376 title: "Manual block",
6377 body: "",
6378 author: "",
6379 url: "",
6380 labels: &[],
6381 origin: "github",
6382 })
6383 .await
6384 .unwrap();
6385 store.update_status(id, TaskStatus::Blocked).await.unwrap();
6386 store
6387 .set_fields(
6388 id,
6389 &[("block_reason", serde_json::json!("waiting for human input"))],
6390 )
6391 .await
6392 .unwrap();
6393 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
6394 .bind(id)
6395 .execute(store.pool())
6396 .await
6397 .unwrap();
6398
6399 auto_recover_rebroadcast_blocked_tasks_global(
6400 &router,
6401 &task_manager,
6402 &store,
6403 &sweep_repos(),
6404 )
6405 .await
6406 .unwrap();
6407
6408 let task = store.get(id).await.unwrap();
6409 assert_eq!(task.status, TaskStatus::Blocked);
6410 assert_eq!(
6411 task.block_reason.as_deref(),
6412 Some("waiting for human input")
6413 );
6414 }
6415
6416 #[serial(cooldown_state)]
6417 #[tokio::test]
6418 async fn global_rebroadcast_sweep_skips_freshly_blocked_tasks() {
6419 cooldown::reset_global_state().await;
6420 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6421 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6422 let task_manager = Arc::new(TaskManager::with_store(
6423 backend,
6424 Arc::clone(&store),
6425 "active/repo".to_string(),
6426 ));
6427 let router = router_with_healthy_review_agent();
6428
6429 let id = store
6430 .upsert_external(&crate::store::UpsertExternal {
6431 repo: "inactive/repo",
6432 ext_id: "40",
6433 title: "Just escalated",
6434 body: "",
6435 author: "",
6436 url: "",
6437 labels: &[],
6438 origin: "github",
6439 })
6440 .await
6441 .unwrap();
6442 store.update_status(id, TaskStatus::Blocked).await.unwrap();
6443 store
6444 .set_fields(
6445 id,
6446 &[("block_reason", serde_json::json!(REBROADCAST_BLOCK_REASON))],
6447 )
6448 .await
6449 .unwrap();
6450 auto_recover_rebroadcast_blocked_tasks_global(
6454 &router,
6455 &task_manager,
6456 &store,
6457 &sweep_repos(),
6458 )
6459 .await
6460 .unwrap();
6461
6462 let task = store.get(id).await.unwrap();
6463 assert_eq!(task.status, TaskStatus::Blocked);
6464 }
6465
6466 #[serial(cooldown_state)]
6471 #[tokio::test]
6472 async fn global_rebroadcast_sweep_recovers_task_from_inactive_repo() {
6473 cooldown::reset_global_state().await;
6474 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6475 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6476 let task_manager = Arc::new(TaskManager::with_store(
6477 backend,
6478 Arc::clone(&store),
6479 "active/repo".to_string(),
6480 ));
6481 let router = router_with_healthy_review_agent();
6482
6483 let id = store
6484 .upsert_external(&crate::store::UpsertExternal {
6485 repo: "inactive/repo",
6486 ext_id: "458",
6487 title: "Rebroadcast blocked from inactive repo",
6488 body: "",
6489 author: "",
6490 url: "",
6491 labels: &[],
6492 origin: "github",
6493 })
6494 .await
6495 .unwrap();
6496 store.update_status(id, TaskStatus::Blocked).await.unwrap();
6497 store
6498 .set_fields(
6499 id,
6500 &[
6501 ("block_reason", serde_json::json!(REBROADCAST_BLOCK_REASON)),
6502 ("needs_review_refires", serde_json::json!(6)),
6503 ],
6504 )
6505 .await
6506 .unwrap();
6507 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
6508 .bind(id)
6509 .execute(store.pool())
6510 .await
6511 .unwrap();
6512
6513 auto_recover_rebroadcast_blocked_tasks_global(
6514 &router,
6515 &task_manager,
6516 &store,
6517 &sweep_repos(),
6518 )
6519 .await
6520 .unwrap();
6521
6522 let task = store.get(id).await.unwrap();
6523 assert_eq!(
6524 task.status,
6525 TaskStatus::NeedsReview,
6526 "cross-repo rebroadcast-blocked task must be recovered to needs_review"
6527 );
6528 assert!(
6529 task.block_reason.is_none(),
6530 "block_reason must be cleared on recovery"
6531 );
6532 assert_eq!(
6533 task.needs_review_refires, 0,
6534 "needs_review_refires must be reset to 0 on recovery"
6535 );
6536 }
6537
6538 #[serial(cooldown_state)]
6541 #[tokio::test]
6542 async fn global_rebroadcast_sweep_skips_reset_without_routable_review_agent() {
6543 cooldown::reset_global_state().await;
6544 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6545 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6546 let task_manager = Arc::new(TaskManager::with_store(
6547 backend,
6548 Arc::clone(&store),
6549 "active/repo".to_string(),
6550 ));
6551 let router = Router::new_for_test(RouterConfig::default(), vec!["claude".to_string()]);
6554
6555 let id = store
6556 .upsert_external(&crate::store::UpsertExternal {
6557 repo: "inactive/repo",
6558 ext_id: "459",
6559 title: "Rebroadcast blocked, no routable agents",
6560 body: "",
6561 author: "",
6562 url: "",
6563 labels: &[],
6564 origin: "github",
6565 })
6566 .await
6567 .unwrap();
6568 store.update_status(id, TaskStatus::Blocked).await.unwrap();
6569 store
6570 .set_fields(
6571 id,
6572 &[("block_reason", serde_json::json!(REBROADCAST_BLOCK_REASON))],
6573 )
6574 .await
6575 .unwrap();
6576 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
6577 .bind(id)
6578 .execute(store.pool())
6579 .await
6580 .unwrap();
6581
6582 auto_recover_rebroadcast_blocked_tasks_global(
6583 &router,
6584 &task_manager,
6585 &store,
6586 &sweep_repos(),
6587 )
6588 .await
6589 .unwrap();
6590
6591 let task = store.get(id).await.unwrap();
6592 assert_eq!(task.status, TaskStatus::Blocked);
6593 assert_eq!(task.block_reason.as_deref(), Some(REBROADCAST_BLOCK_REASON));
6594 }
6595}