1use super::EngineConfig;
12use super::ci_bootstrap;
13use super::cleanup::{check_merged_prs, cleanup_done_worktrees, prune_old_run_output};
14use super::commands::{CommandOutcome, CommandStoreOps, parse_command, validate_and_run_command};
15use super::review_poll::review_open_prs;
16use crate::backends::ExternalId;
17use crate::backends::ExternalTask;
18use crate::backends::{ExternalBackend, Mention, Status};
19use crate::config;
20use crate::engine;
21use crate::engine::auto_merge::{is_pr_behind, required_checks_state};
22use crate::engine::cleanup::cleanup_task_worktree;
23use crate::engine::commands::CommandGh;
24use crate::engine::commands::OwnerCommand;
25use crate::engine::cooldown;
26use crate::engine::cooldown::{
27 cooldown_reason, github_circuit_remaining_secs, is_agent_in_cooldown, is_github_circuit_open,
28 is_model_in_cooldown, refresh_degraded_agents,
29};
30use crate::engine::dependencies;
31use crate::engine::router::Router;
32use crate::engine::router::RouterConfig;
33use crate::engine::router::config as router_config;
34use crate::engine::runner::agents::patterns;
35use crate::engine::tasks;
36use crate::engine::tasks::{TaskManager, status_to_task_status};
37use crate::github::http::GhHttp;
38use crate::github::types::extract_issue_number_from_url;
39use crate::store;
40use crate::store::Task;
41use crate::store::TaskRun;
42use crate::store::TaskStatus;
43use crate::store::TaskStore;
44use crate::tmux::TmuxManager;
45use dashmap::{DashMap, DashSet};
46use futures::future;
47use std::collections::HashMap;
48use std::collections::HashSet;
49use std::sync::Arc;
50use std::time::Duration;
51use std::time::Instant;
52use tokio::sync::OnceCell;
53use tokio::sync::RwLock;
54use tokio::time;
55
56async fn kv_get_prefer_store(store: Option<&Arc<TaskStore>>, key: &str) -> Option<String> {
62 match try_kv_get_prefer_store(store, key).await {
63 Ok(opt) => opt,
64 Err(e) => {
65 tracing::error!(key, err = %e, "kv_get failed");
66 None
67 }
68 }
69}
70
71async fn try_kv_get_prefer_store(
77 store: Option<&Arc<TaskStore>>,
78 key: &str,
79) -> anyhow::Result<Option<String>> {
80 if let Some(s) = store {
81 Ok(s.kv_get(key).await?)
82 } else {
83 Ok(None)
84 }
85}
86
87async fn kv_set_prefer_store(store: Option<&Arc<TaskStore>>, key: &str, value: &str) {
93 if let Err(e) = try_kv_set_prefer_store(store, key, value).await {
94 tracing::error!(key, err = %e, "kv_set failed");
95 }
96}
97
98async fn try_kv_set_prefer_store(
104 store: Option<&Arc<TaskStore>>,
105 key: &str,
106 value: &str,
107) -> anyhow::Result<()> {
108 if let Some(s) = store {
109 s.kv_set(key, value).await?;
110 }
111 Ok(())
112}
113
114#[derive(Debug, Clone)]
115pub(crate) struct ReviewTaskSnapshot {
116 pub external: ExternalTask,
117 pub stored: Task,
118}
119
120async fn prefetch_review_tasks(
121 store: &Arc<TaskStore>,
122 repo: &str,
123) -> anyhow::Result<(
124 Vec<ReviewTaskSnapshot>,
125 Vec<ReviewTaskSnapshot>,
126 Vec<ReviewTaskSnapshot>,
127)> {
128 let mut in_review = store.list_by_status(repo, TaskStatus::InReview).await?;
130 in_review.extend(store.list_by_status(repo, TaskStatus::Approved).await?);
131 let in_review = in_review
132 .into_iter()
133 .map(|stored| ReviewTaskSnapshot {
134 external: tasks::store_task_to_external(&stored),
135 stored,
136 })
137 .collect();
138 let needs_review = store
139 .list_by_status(repo, TaskStatus::NeedsReview)
140 .await?
141 .into_iter()
142 .map(|stored| ReviewTaskSnapshot {
143 external: tasks::store_task_to_external(&stored),
144 stored,
145 })
146 .collect();
147
148 let blocked_candidates = store
152 .list_by_status(repo, TaskStatus::Blocked)
153 .await?
154 .into_iter()
155 .filter(|stored| stored.pr_number.is_some() || !stored.branch.is_empty())
156 .map(|stored| ReviewTaskSnapshot {
157 external: tasks::store_task_to_external(&stored),
158 stored,
159 })
160 .collect();
161
162 Ok((in_review, needs_review, blocked_candidates))
163}
164
165async fn fetch_comments_since(
166 backend: &Arc<dyn ExternalBackend>,
167 since: &str,
168) -> anyhow::Result<Vec<Mention>> {
169 backend.get_mentions(since).await
170}
171
172fn filter_mentions_by_since(comments: &[Mention], since: &str) -> Vec<Mention> {
173 comments
174 .iter()
175 .filter(|comment| comment.created_at.as_str() > since)
176 .cloned()
177 .collect()
178}
179
180#[derive(Debug, Clone, Copy, PartialEq, Eq)]
181enum FailureCategory {
182 FalseFailure,
183 RateLimit,
184 CreditExhausted,
185 SilentExit0,
186 Timeout,
187 ConnectionError,
188 CliFlagError,
189 AllAgentsExhausted,
190 ParseError,
191 MaxAttempts,
192 PushFailed,
193 PrCreateFailed,
194 ModelUnavailable,
195 Unknown,
196}
197
198impl FailureCategory {
199 fn is_recoverable(self) -> bool {
206 matches!(
207 self,
208 FailureCategory::FalseFailure
209 | FailureCategory::RateLimit
210 | FailureCategory::CreditExhausted
211 | FailureCategory::SilentExit0
212 | FailureCategory::Timeout
213 | FailureCategory::ConnectionError
214 | FailureCategory::CliFlagError
215 | FailureCategory::AllAgentsExhausted
216 | FailureCategory::ParseError
217 | FailureCategory::ModelUnavailable
218 )
219 }
220
221 fn as_str(self) -> &'static str {
222 match self {
223 FailureCategory::FalseFailure => "FalseFailure",
224 FailureCategory::RateLimit => "RateLimit",
225 FailureCategory::CreditExhausted => "CreditExhausted",
226 FailureCategory::SilentExit0 => "SilentExit0",
227 FailureCategory::Timeout => "Timeout",
228 FailureCategory::ConnectionError => "ConnectionError",
229 FailureCategory::CliFlagError => "CliFlagError",
230 FailureCategory::AllAgentsExhausted => "AllAgentsExhausted",
231 FailureCategory::ParseError => "ParseError",
232 FailureCategory::MaxAttempts => "MaxAttempts",
233 FailureCategory::PushFailed => "PushFailed",
234 FailureCategory::PrCreateFailed => "PrCreateFailed",
235 FailureCategory::ModelUnavailable => "ModelUnavailable",
236 FailureCategory::Unknown => "Unknown",
237 }
238 }
239}
240
241fn classify_failure(error: &str, outcome: &str) -> FailureCategory {
242 let lower = error.to_lowercase();
243
244 if outcome == "timeout" || lower.contains("timeout") {
245 return FailureCategory::Timeout;
246 }
247
248 if outcome == "billing_cycle_exhausted"
259 || outcome == "credit_exhausted"
260 || outcome == "org_level_disabled"
261 || cooldown::detect_credit_exhaustion(&lower).is_some()
262 {
263 return FailureCategory::CreditExhausted;
264 }
265
266 if outcome == "rate_limit" || lower.contains("rate limit") || lower.contains("usage limit") {
267 return FailureCategory::RateLimit;
268 }
269
270 if error.trim().is_empty() {
271 return FailureCategory::FalseFailure;
272 }
273
274 if lower.contains("all agents exhausted") {
275 return FailureCategory::AllAgentsExhausted;
276 }
277
278 if lower.contains("max attempts") || lower.contains("exceeded max attempts") {
279 return FailureCategory::MaxAttempts;
280 }
281
282 if lower.contains("output-format=stream-json") && lower.contains("requires --verbose") {
283 return FailureCategory::CliFlagError;
284 }
285
286 if lower.contains("model unavailable")
287 || lower.contains("model not found")
288 || (lower.contains("does not exist") && lower.contains("model"))
289 {
290 return FailureCategory::ModelUnavailable;
291 }
292
293 if lower.contains("push failed") {
294 return FailureCategory::PushFailed;
295 }
296
297 if lower.contains("create pr failed")
298 || lower.contains("failed to create pull request")
299 || lower.contains("pull request creation failed")
300 {
301 return FailureCategory::PrCreateFailed;
302 }
303
304 if lower.contains("exit 0") {
305 return FailureCategory::SilentExit0;
306 }
307
308 if outcome == "parse_error"
309 || outcome == "truncated"
310 || lower.contains("invalid response")
311 || lower.contains("parse error")
312 || lower.contains("truncated")
313 {
314 return FailureCategory::ParseError;
315 }
316
317 if patterns::detect_network_error(&lower).is_some() {
318 return FailureCategory::ConnectionError;
319 }
320
321 FailureCategory::Unknown
322}
323
324fn classify_failure_from_run(run: &TaskRun) -> Option<FailureCategory> {
325 if run.outcome == "success" && run.error.trim().is_empty() {
327 return None;
328 }
329
330 if run.outcome.trim().is_empty()
337 && run.error.trim().is_empty()
338 && run.exit_code.is_none()
339 && run.stdout.trim().is_empty()
340 && run.stderr.trim().is_empty()
341 {
342 return None;
343 }
344
345 let has_output = !run.stdout.trim().is_empty() || !run.stderr.trim().is_empty();
346 if run.exit_code == Some(0) && !has_output && run.outcome != "success" {
347 return Some(FailureCategory::SilentExit0);
348 }
349
350 let mut category = classify_failure(&run.error, &run.outcome);
351
352 if category == FailureCategory::FalseFailure {
361 let has_nonzero_exit = run.exit_code.is_some() && run.exit_code != Some(0);
362 let has_output = !run.stdout.trim().is_empty() || !run.stderr.trim().is_empty();
363 if has_nonzero_exit || has_output {
364 category = FailureCategory::Unknown;
365 }
366 }
367
368 Some(category)
369}
370
371async fn resolve_mention_parent_id(
381 store: &TaskStore,
382 repo: &str,
383 issue_num: &str,
384 is_pr: bool,
385) -> anyhow::Result<Option<i64>> {
386 if is_pr {
387 if let Ok(pr_num) = issue_num.parse::<i32>() {
388 store.resolve_id_by_pr_number(repo, pr_num).await
389 } else {
390 Ok(None)
391 }
392 } else {
393 store.resolve_id_by_external(repo, issue_num).await
394 }
395}
396
397async fn record_mention_task(
399 store: &TaskStore,
400 repo: &str,
401 mention: &Mention,
402 title: &str,
403 body: &str,
404 parent_id: Option<i64>,
405 cursor: &mut MentionCursor,
406) -> bool {
407 match store
408 .create_internal(repo, title, body, "mention", &mention.id, parent_id)
409 .await
410 {
411 Ok(task_id) => {
412 tracing::info!(task_id, mention_id = %mention.id, "created mention task");
413 cursor.advance(&mention.created_at);
414 true
415 }
416 Err(e) => {
417 tracing::warn!(mention_id = %mention.id, err = %e, "failed to create mention task");
418 false
419 }
420 }
421}
422
423fn advance_cursor(last_safe_ts: &mut Option<String>, ts: &str) {
425 if last_safe_ts.as_deref() < Some(ts) {
426 *last_safe_ts = Some(ts.to_string());
427 }
428}
429
430#[derive(Debug, Default)]
431struct MentionCursor {
432 last_safe_ts: Option<String>,
433 blocked_by_gap: bool,
434}
435
436impl MentionCursor {
437 fn advance(&mut self, ts: &str) {
438 if self.blocked_by_gap {
439 return;
440 }
441 advance_cursor(&mut self.last_safe_ts, ts);
442 }
443
444 fn block_on_gap(&mut self) {
445 self.blocked_by_gap = true;
446 }
447
448 fn into_last_safe_ts(self) -> Option<String> {
449 self.last_safe_ts
450 }
451}
452
453#[derive(Debug, PartialEq)]
455pub(crate) enum CommentAction {
456 Skip,
458 ExecuteCommand {
460 command: OwnerCommand,
461 issue_num: String,
462 },
463 ExecuteCommandForMention {
465 command: OwnerCommand,
466 issue_num: String,
467 },
468 CreateMentionTask { issue_num: Option<String> },
470}
471
472pub(crate) fn classify_comment(
477 body: &str,
478 issue_url: Option<&str>,
479 current_user: &str,
480 already_processed: bool,
481) -> CommentAction {
482 if already_processed {
483 return CommentAction::Skip;
484 }
485
486 let is_mention =
487 body.contains(current_user) || body.contains("@orch") || body.contains("@orchestrator");
488 let command = parse_command(body);
489 let issue_num = issue_url.and_then(extract_issue_number_from_url);
490
491 match (command, is_mention, issue_num) {
492 (Some(cmd), true, Some(num)) => CommentAction::ExecuteCommandForMention {
493 command: cmd,
494 issue_num: num,
495 },
496 (Some(cmd), false, Some(num)) => CommentAction::ExecuteCommand {
497 command: cmd,
498 issue_num: num,
499 },
500 (Some(_), true, None) => CommentAction::CreateMentionTask { issue_num: None },
502 (None, true, issue_num) => CommentAction::CreateMentionTask { issue_num },
504 (Some(_), false, None) | (None, false, _) => CommentAction::Skip,
506 }
507}
508
509#[expect(
522 clippy::too_many_arguments,
523 reason = "the sync tick hands over its shared state piece by piece"
524)]
525async fn handle_slash_command(
526 backend: &Arc<dyn ExternalBackend>,
527 store: Option<&Arc<TaskStore>>,
528 repo: &str,
529 task_manager: &Arc<TaskManager>,
530 gh: &dyn CommandGh,
531 mention: &Mention,
532 command: &OwnerCommand,
533 cursor: &mut MentionCursor,
534) -> anyhow::Result<bool> {
535 let Some(issue_num) = mention
536 .issue_url
537 .as_deref()
538 .and_then(extract_issue_number_from_url)
539 else {
540 tracing::warn!(comment_id = %mention.id, "slash command without valid issue number");
541 return Ok(true);
542 };
543
544 let store_opt: Option<Arc<dyn CommandStoreOps>> =
545 store.map(|s| Arc::clone(s) as Arc<dyn CommandStoreOps>);
546 let outcome = validate_and_run_command(
547 backend,
548 gh,
549 repo,
550 &issue_num,
551 command,
552 &mention.author,
553 &store_opt,
554 task_manager,
555 )
556 .await;
557
558 let is_pr = match &outcome {
560 CommandOutcome::Executed { is_pr }
561 | CommandOutcome::NotOpen { is_pr }
562 | CommandOutcome::NotCollaborator { is_pr } => *is_pr,
563 CommandOutcome::FetchFailed | CommandOutcome::CollaboratorCheckFailed => {
564 tracing::warn!(
567 comment_id = %mention.id,
568 command = %command,
569 "deferring @orch slash command after a transient GitHub error"
570 );
571 return Ok(false);
572 }
573 };
574
575 if let Some(s) = store {
578 let parent_id = match resolve_mention_parent_id(s, repo, &issue_num, is_pr).await {
579 Ok(parent_id) => parent_id,
580 Err(e) => {
581 tracing::warn!(
582 comment_id = %mention.id,
583 err = %e,
584 "deferring mention task due to parent lookup failure"
585 );
586 return Err(e);
587 }
588 };
589
590 let (title, body) = match &outcome {
591 CommandOutcome::NotOpen { .. } => (
592 format!(
593 "Skipped @orch command on closed #{issue_num} from @{}",
594 mention.author
595 ),
596 format!(
597 "Mention by @{}:\n\n{}\n\n**Skipped:** target is closed",
598 mention.author, mention.body
599 ),
600 ),
601 CommandOutcome::NotCollaborator { .. } => (
602 format!(
603 "Skipped @orch command on #{issue_num} from non-collaborator @{}",
604 mention.author
605 ),
606 format!(
607 "Mention by @{}:\n\n{}\n\n**Skipped:** author is not a collaborator",
608 mention.author, mention.body
609 ),
610 ),
611 CommandOutcome::Executed { .. } => (
612 format!(
613 "Respond to mention by @{} on task #{issue_num} — command executed",
614 mention.author
615 ),
616 format!(
617 "Mention by @{} on #{issue_num}:\n\n{}\n\n**Status:** Command executed",
618 mention.author, mention.body
619 ),
620 ),
621 CommandOutcome::FetchFailed | CommandOutcome::CollaboratorCheckFailed => {
622 tracing::debug!("skipping mention task record due to earlier command failure");
623 return Ok(true);
624 }
625 };
626
627 let created = record_mention_task(s, repo, mention, &title, &body, parent_id, cursor).await;
628 if !created {
629 return Ok(false);
630 }
631 }
632 Ok(true)
633}
634
635fn auto_unblock_cooldown_elapsed(count: i32, last_at: &str) -> bool {
636 if count == 0 {
637 return true;
638 }
639
640 let required = match count {
641 1 => chrono::Duration::minutes(30),
642 2 => chrono::Duration::hours(2),
643 _ => return false,
644 };
645
646 if last_at.trim().is_empty() {
647 return true;
648 }
649
650 match chrono::DateTime::parse_from_rfc3339(last_at) {
651 Ok(ts) => chrono::Utc::now() - ts.with_timezone(&chrono::Utc) >= required,
652 Err(_) => true,
653 }
654}
655
656fn is_ci_failure_block(task: &Task) -> bool {
657 let reason = task.block_reason.as_deref().unwrap_or("");
658 reason.contains("CI failure limit") || reason.contains("CI checks timed out")
659}
660
661fn ci_failure_unblock_cooldown_elapsed(task: &Task) -> bool {
662 if task.auto_unblock_count == 0 {
663 return true;
664 }
665
666 let required = match task.auto_unblock_count {
670 1 => chrono::Duration::minutes(10),
671 2 => chrono::Duration::hours(1),
672 3 => chrono::Duration::hours(6),
673 _ => chrono::Duration::hours(24),
674 };
675
676 let last_at = &task.auto_unblock_last_at;
677 if last_at.trim().is_empty() {
678 return true;
679 }
680
681 match chrono::DateTime::parse_from_rfc3339(last_at) {
682 Ok(ts) => chrono::Utc::now() - ts.with_timezone(&chrono::Utc) >= required,
683 Err(_) => true,
684 }
685}
686
687async fn try_unblock_ci_failure_task(
688 gh: &GhHttp,
689 repo: &str,
690 task_manager: &Arc<TaskManager>,
691 store: &Arc<TaskStore>,
692 task: &Task,
693 auto_merge_in_flight: &Arc<DashSet<String>>,
694) -> anyhow::Result<bool> {
695 let Some(pr_number) = task.pr_number.and_then(|n| u64::try_from(n).ok()) else {
696 tracing::debug!(
697 task_id = task.id,
698 "CI failure blocked task has no PR number"
699 );
700 return Ok(false);
701 };
702
703 let pr = match gh.get_pr(repo, pr_number).await {
704 Ok(p) => p,
705 Err(e) => {
706 tracing::warn!(task_id = task.id, pr_number, err = %e, "failed to get PR status for CI failure block");
707 return Ok(false);
708 }
709 };
710
711 tracing::debug!(
712 task_id = task.id,
713 pr_number,
714 merged = pr.merged,
715 state = ?pr.state,
716 "verifying PR state for CI failure blocked task"
717 );
718
719 if pr.merged.unwrap_or(false) || pr.state.eq_ignore_ascii_case("closed") {
720 tracing::info!(
721 task_id = task.id,
722 pr_number,
723 merged = pr.merged,
724 state = %pr.state,
725 "PR merged/closed — unblocking CI failure blocked task"
726 );
727 let ext_id = task
728 .external_id
729 .clone()
730 .unwrap_or_else(|| format!("internal:{}", task.id));
731 task_manager
735 .mark_task_done_by_store_id(task.id, &task.repo, &ExternalId(ext_id))
736 .await?;
737 let fields = vec![
738 ("block_reason", serde_json::Value::Null),
739 ("auto_unblock_count", serde_json::json!(0)),
740 ("auto_unblock_last_at", serde_json::json!("")),
741 ("auto_unblock_last_reason", serde_json::json!("")),
742 ];
743 store.set_fields(task.id, &fields).await?;
744 return Ok(true);
745 }
746
747 let task_id_str = task
751 .external_id
752 .clone()
753 .unwrap_or_else(|| format!("internal:{}", task.id));
754 if auto_merge_in_flight.contains(&task_id_str) {
755 tracing::debug!(
756 task_id = task.id,
757 pr_number,
758 "CI-failure recovery merge already in flight — skipping"
759 );
760 return Ok(false);
761 }
762
763 let branch_update_result = gh.update_pr_branch(repo, pr_number).await;
769 match branch_update_result {
770 Ok(()) => {
771 tracing::info!(
772 task_id = task.id,
773 pr_number,
774 "updated CI-failure-blocked PR branch against base to retrigger CI"
775 );
776 }
777 Err(ref e) => {
778 tracing::warn!(
780 task_id = task.id,
781 pr_number,
782 err = %e,
783 outcome = "branch_update_failed",
784 "failed to update CI-failure-blocked PR branch (may already be up to date)"
785 );
786 }
787 }
788
789 if let Err(e) = gh.enable_auto_merge(repo, pr_number).await {
790 tracing::warn!(task_id = task.id, pr_number, err = %e, "failed to re-enable auto-merge after branch update");
791 }
792
793 let pr_state = pr.state.as_str();
796 let new_reason = if let Err(ref e) = branch_update_result {
797 format!(
798 "CI failure limit reached during auto-merge (PR #{pr_number} still open, state: {pr_state}, branch update failed: {e})"
799 )
800 } else {
801 format!(
802 "CI failure limit reached during auto-merge (PR #{pr_number} still open, state: {pr_state})"
803 )
804 };
805 let now = chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string();
806 let new_count = task.auto_unblock_count.saturating_add(1);
809 let fields = vec![
810 ("block_reason", serde_json::json!(&new_reason)),
811 ("auto_unblock_count", serde_json::json!(new_count)),
812 ("auto_unblock_last_at", serde_json::json!(&now)),
813 ];
814 store.set_fields_silent(task.id, &fields).await?;
818
819 auto_merge_in_flight.insert(task_id_str.clone());
823 let gh_clone = gh.clone();
824 let repo_string = repo.to_string();
825 let task_manager_clone = Arc::clone(task_manager);
826 let store_clone = Arc::clone(store);
827 let task_clone = task.clone();
828 let in_flight_clone = Arc::clone(auto_merge_in_flight);
829
830 tokio::spawn(async move {
831 let _guard = scopeguard::guard((), |()| {
832 in_flight_clone.remove(&task_id_str);
833 });
834
835 match poll_and_merge_recovered_pr(&gh_clone, &repo_string, pr_number, task_clone.id).await {
836 Ok(true) => {
837 tracing::info!(
838 task_id = task_clone.id,
839 pr_number,
840 "CI-failure-blocked PR merged during recovery sweep"
841 );
842 let ext_id = task_clone
843 .external_id
844 .clone()
845 .unwrap_or_else(|| format!("internal:{}", task_clone.id));
846 let fields = [
847 ("block_reason", serde_json::Value::Null),
848 ("auto_unblock_count", serde_json::json!(0)),
849 ("auto_unblock_last_at", serde_json::json!("")),
850 ("auto_unblock_last_reason", serde_json::json!("")),
851 ];
852 if let Err(e) = task_manager_clone
853 .update_task_status_and_result_by_store_id(
854 task_clone.id,
855 &task_clone.repo,
856 &ExternalId(ext_id),
857 Status::Done,
858 &fields,
859 )
860 .await
861 {
862 tracing::warn!(
863 task_id = task_clone.id,
864 pr_number,
865 err = %e,
866 "failed to mark CI-failure-blocked task done after recovery merge"
867 );
868 }
869 if let Err(e) =
870 cleanup_task_worktree(&format!("{}", task_clone.id), &repo_string, &store_clone)
871 .await
872 {
873 tracing::warn!(
874 task_id = task_clone.id,
875 pr_number,
876 err = %e,
877 "cleanup after CI-failure recovery merge failed"
878 );
879 }
880 }
881 Ok(false) => {
882 tracing::debug!(
883 task_id = task_clone.id,
884 pr_number,
885 "CI-failure-blocked PR not merged this recovery tick — poll completed without merge"
886 );
887 }
888 Err(e) => {
889 tracing::warn!(
890 task_id = task_clone.id,
891 pr_number,
892 err = %e,
893 "error while polling CI-failure-blocked PR for recovery"
894 );
895 }
896 }
897 });
898
899 Ok(false)
900}
901
902async fn poll_and_merge_recovered_pr(
913 gh: &GhHttp,
914 repo: &str,
915 pr_number: u64,
916 task_id: i64,
917) -> anyhow::Result<bool> {
918 const MAX_BEHIND_RETRIES: u32 = 3;
922
923 let max_wait_secs: u64 = config::get("workflow.ci_poll_max_wait_secs")
924 .ok()
925 .and_then(|s| s.parse().ok())
926 .unwrap_or(600);
927 let base_interval_secs: u64 = config::get("workflow.ci_poll_interval_secs")
928 .ok()
929 .and_then(|s| s.parse().ok())
930 .unwrap_or(15);
931 let max_wait = Duration::from_secs(max_wait_secs);
932 let start = Instant::now();
933 let mut poll_count: u32 = 0;
934 let mut behind_retries: u32 = 0;
935
936 let initial_pr = gh.get_pr(repo, pr_number).await?;
937 let base_branch = initial_pr.base.ref_.clone();
938 let required_contexts = gh
939 .get_required_status_check_contexts(repo, &base_branch)
940 .await?;
941 let repo_has_workflows = if required_contexts.is_empty() {
942 gh.has_workflows(repo).await?
943 } else {
944 true
945 };
946
947 loop {
948 let pr = gh.get_pr(repo, pr_number).await?;
949 if pr.merged.unwrap_or(false) || pr.state.eq_ignore_ascii_case("closed") {
950 tracing::info!(
951 task_id = task_id,
952 pr_number,
953 "PR merged/closed during recovery poll"
954 );
955 return Ok(true);
956 }
957
958 if pr.mergeable == Some(false) {
959 tracing::info!(
960 task_id = task_id,
961 pr_number,
962 outcome = "poll_exited_without_merge",
963 reason = "merge_conflicts",
964 "PR has merge conflicts after recovery update — leaving for next recovery cycle"
965 );
966 return Ok(false);
967 }
968
969 if is_pr_behind(&pr) {
970 if behind_retries >= MAX_BEHIND_RETRIES {
971 tracing::info!(
972 task_id = task_id,
973 pr_number,
974 behind_retries,
975 max_behind_retries = MAX_BEHIND_RETRIES,
976 outcome = "poll_exited_without_merge",
977 reason = "behind_retry_exhausted",
978 "PR still behind base after recovery retries — leaving for next recovery cycle"
979 );
980 return Ok(false);
981 }
982 behind_retries += 1;
983 match gh.update_pr_branch(repo, pr_number).await {
984 Ok(()) => {
985 tracing::info!(
986 task_id = task_id,
987 pr_number,
988 behind_retries,
989 "PR was behind base — updated branch and continuing recovery poll"
990 );
991 continue;
992 }
993 Err(e) => {
994 tracing::warn!(
995 task_id = task_id,
996 pr_number,
997 behind_retries,
998 err = %e,
999 outcome = "branch_update_failed",
1000 "failed to update behind PR branch during recovery poll"
1001 );
1002 return Ok(false);
1003 }
1004 }
1005 }
1006
1007 let head_sha = pr.head.sha.clone();
1008 let (state, total, passing, failing, pending) = if required_contexts.is_empty() {
1009 gh.get_combined_status(
1010 repo,
1011 &head_sha,
1012 repo_has_workflows,
1013 pr.mergeable_state.as_deref(),
1014 )
1015 .await?
1016 } else {
1017 let check_runs = gh.get_check_runs(repo, &head_sha).await?;
1018 let statuses = gh.get_commit_status_contexts(repo, &head_sha).await?;
1019 required_checks_state(&required_contexts, &check_runs, &statuses)
1020 };
1021
1022 tracing::info!(
1023 task_id = task_id,
1024 pr_number,
1025 state = %state,
1026 total,
1027 passing,
1028 failing,
1029 pending,
1030 "CI recovery status check"
1031 );
1032
1033 match state.as_str() {
1034 "success" => {
1035 tracing::info!(
1036 task_id = task_id,
1037 pr_number,
1038 "CI green after recovery — attempting direct merge"
1039 );
1040 match gh.merge_pr(repo, pr_number, true, &head_sha).await {
1041 Ok(()) => {
1042 tracing::info!(
1043 task_id = task_id,
1044 pr_number,
1045 "direct merge succeeded after recovery"
1046 );
1047 return Ok(true);
1048 }
1049 Err(e) => {
1050 let err = e.to_string().to_ascii_lowercase();
1051 tracing::warn!(
1052 task_id = task_id,
1053 pr_number,
1054 err = %e,
1055 "direct merge failed after recovery"
1056 );
1057 if err.contains("already merged")
1060 || err.contains("no merge commit")
1061 || err.contains("pull request already merged")
1062 {
1063 return Ok(true);
1064 }
1065 if err.contains("head branch was modified")
1067 || err.contains("sha does not match")
1068 {
1069 tracing::info!(
1070 task_id = task_id,
1071 pr_number,
1072 outcome = "poll_exited_without_merge",
1073 reason = "head_moved",
1074 "merge rejected because the head sha changed"
1075 );
1076 return Ok(false);
1077 }
1078 if (err.contains("head branch is out of date")
1082 || err.contains("behind")
1083 || err.contains("409"))
1084 && behind_retries < MAX_BEHIND_RETRIES
1085 {
1086 behind_retries += 1;
1087 match gh.update_pr_branch(repo, pr_number).await {
1088 Ok(()) => {
1089 tracing::info!(
1090 task_id = task_id,
1091 pr_number,
1092 behind_retries,
1093 "merge rejected due to out-of-date branch — updated and continuing recovery poll"
1094 );
1095 continue;
1096 }
1097 Err(update_err) => {
1098 tracing::warn!(
1099 task_id = task_id,
1100 pr_number,
1101 behind_retries,
1102 err = %update_err,
1103 outcome = "branch_update_failed",
1104 "failed to update out-of-date PR branch during recovery poll"
1105 );
1106 }
1107 }
1108 }
1109 tracing::info!(
1110 task_id = task_id,
1111 pr_number,
1112 outcome = "poll_exited_without_merge",
1113 reason = "direct_merge_failed",
1114 "direct merge did not succeed — leaving for next recovery cycle"
1115 );
1116 return Ok(false);
1117 }
1118 }
1119 }
1120 "failure" => {
1121 tracing::info!(
1122 task_id = task_id,
1123 pr_number,
1124 outcome = "poll_exited_without_merge",
1125 reason = "ci_failure",
1126 "CI still failing after recovery — will retry later"
1127 );
1128 return Ok(false);
1129 }
1130 _ => {}
1131 }
1132
1133 if start.elapsed() >= max_wait {
1134 tracing::info!(
1135 task_id = task_id,
1136 pr_number,
1137 outcome = "poll_exited_without_merge",
1138 reason = "poll_timeout",
1139 "CI still pending after recovery poll timeout — will retry later"
1140 );
1141 return Ok(false);
1142 }
1143
1144 poll_count += 1;
1145 let multiplier = (2_u64).pow(poll_count.saturating_sub(1)).min(4);
1146 let sleep_secs = base_interval_secs.saturating_mul(multiplier);
1147 time::sleep(Duration::from_secs(sleep_secs)).await;
1148 }
1149}
1150
1151const REBROADCAST_BLOCK_REASON: &str = "review agent rebroadcast escalated after repeated retries";
1156
1157pub(crate) async fn auto_recover_rebroadcast_blocked_tasks_global(
1172 router: &Router,
1173 task_manager: &Arc<TaskManager>,
1174 store: &Arc<TaskStore>,
1175 active_repos: &HashSet<String>,
1176) -> anyhow::Result<()> {
1177 const MIN_BLOCK_AGE_MINUTES: i64 = 5;
1178
1179 let all_blocked = match store.list_all_by_status_global(TaskStatus::Blocked).await {
1180 Ok(tasks) => tasks,
1181 Err(e) => {
1182 tracing::warn!(err = %e, "global rebroadcast-blocked sweep: failed to list blocked tasks");
1183 return Ok(());
1184 }
1185 };
1186
1187 let now = chrono::Utc::now();
1188 let candidates: Vec<_> = all_blocked
1189 .into_iter()
1190 .filter(|t| {
1191 if t.block_reason.as_deref() != Some(REBROADCAST_BLOCK_REASON) {
1192 return false;
1193 }
1194 if !active_repos.contains(&t.repo) {
1196 return false;
1197 }
1198 match chrono::DateTime::parse_from_rfc3339(&t.updated_at) {
1200 Ok(updated_at) => {
1201 let age = now - updated_at.with_timezone(&chrono::Utc);
1202 age.num_minutes() >= MIN_BLOCK_AGE_MINUTES
1203 }
1204 Err(_) => true,
1205 }
1206 })
1207 .collect();
1208
1209 if candidates.is_empty() {
1210 return Ok(());
1211 }
1212
1213 let gh = match GhHttp::new() {
1214 Ok(g) => Some(g),
1215 Err(e) => {
1216 tracing::warn!(err = %e, "global rebroadcast-blocked sweep: failed to create GhHttp — skipping PR-state checks this tick");
1217 None
1218 }
1219 };
1220
1221 let any_routable =
1224 !router.available_agents.is_empty() && router.healthy_agent_count("review") > 0;
1225
1226 for task in candidates {
1227 let ext_id = task
1228 .external_id
1229 .clone()
1230 .unwrap_or_else(|| format!("internal:{}", task.id));
1231
1232 if let (Some(gh), Some(pr_number)) =
1233 (&gh, task.pr_number.and_then(|n| u64::try_from(n).ok()))
1234 {
1235 match gh.get_pr(&task.repo, pr_number).await {
1236 Ok(pr) => {
1237 if pr.merged.unwrap_or(false) {
1238 tracing::info!(
1239 task_id = task.id,
1240 repo = %task.repo,
1241 pr_number,
1242 "global rebroadcast-blocked sweep: PR already merged — marking task done"
1243 );
1244 if let Err(e) = task_manager
1245 .mark_task_done_by_store_id(
1246 task.id,
1247 &task.repo,
1248 &ExternalId(ext_id.clone()),
1249 )
1250 .await
1251 {
1252 tracing::warn!(task_id = task.id, repo = %task.repo, err = %e, "global rebroadcast-blocked sweep: failed to mark merged task done");
1253 continue;
1254 }
1255 let fields: &[(&str, serde_json::Value)] = &[
1256 ("block_reason", serde_json::Value::Null),
1257 ("needs_review_refires", serde_json::json!(0)),
1258 ];
1259 if let Err(e) = store.set_fields(task.id, fields).await {
1260 tracing::warn!(task_id = task.id, err = %e, "global rebroadcast-blocked sweep: failed to clear fields after marking done");
1261 }
1262 continue;
1263 }
1264 if pr.state.eq_ignore_ascii_case("closed") {
1265 tracing::info!(
1266 task_id = task.id,
1267 repo = %task.repo,
1268 pr_number,
1269 "global rebroadcast-blocked sweep: PR closed unmerged — leaving task blocked, not re-dispatching"
1270 );
1271 let new_reason = format!(
1272 "{REBROADCAST_BLOCK_REASON} (PR #{pr_number} closed unmerged — no further auto-recovery)"
1273 );
1274 let fields: &[(&str, serde_json::Value)] =
1275 &[("block_reason", serde_json::json!(new_reason))];
1276 if let Err(e) = store.set_fields_silent(task.id, fields).await {
1280 tracing::warn!(task_id = task.id, err = %e, "global rebroadcast-blocked sweep: failed to annotate closed-unmerged block reason");
1281 }
1282 continue;
1283 }
1284 }
1286 Err(e) => {
1287 tracing::warn!(task_id = task.id, repo = %task.repo, pr_number, err = %e, "global rebroadcast-blocked sweep: failed to get PR status — skipping this tick");
1288 continue;
1289 }
1290 }
1291 }
1292
1293 if !any_routable {
1294 continue;
1295 }
1296
1297 if let Err(e) = task_manager
1298 .update_task_status_by_store_id(
1299 task.id,
1300 &task.repo,
1301 &ExternalId(ext_id.clone()),
1302 Status::NeedsReview,
1303 )
1304 .await
1305 {
1306 tracing::warn!(task_id = task.id, repo = %task.repo, err = %e, "global rebroadcast-blocked sweep: failed to reset task to needs_review — skipping");
1307 continue;
1308 }
1309
1310 let fields: &[(&str, serde_json::Value)] = &[
1311 ("block_reason", serde_json::Value::Null),
1312 ("needs_review_refires", serde_json::json!(0)),
1313 ];
1314 if let Err(e) = store.set_fields(task.id, fields).await {
1315 tracing::warn!(task_id = task.id, err = %e, "global rebroadcast-blocked sweep: failed to clear block_reason/needs_review_refires after status reset");
1316 }
1317
1318 tracing::info!(
1319 task_id = task.id,
1320 repo = %task.repo,
1321 ext_id = %ext_id,
1322 "global rebroadcast-blocked sweep: auto-recovered rebroadcast-blocked task — review agents now available"
1323 );
1324 }
1325
1326 Ok(())
1327}
1328
1329async fn auto_unblock_blocked_tasks(
1330 repo: &str,
1331 task_manager: &Arc<TaskManager>,
1332 store: &Arc<TaskStore>,
1333 dispatching: &Arc<DashMap<String, String>>,
1334 auto_merge_in_flight: &Arc<DashSet<String>>,
1335) -> anyhow::Result<()> {
1336 let blocked = match store.list_by_status(repo, TaskStatus::Blocked).await {
1337 Ok(tasks) => tasks,
1338 Err(e) => {
1339 tracing::warn!(err = %e, "failed to list blocked tasks — skipping auto-unblock this tick");
1340 return Ok(());
1341 }
1342 };
1343
1344 if blocked.is_empty() {
1345 return Ok(());
1346 }
1347
1348 let task_ids: Vec<i64> = blocked.iter().map(|t| t.id).collect();
1350 let mut runs_by_task = match store.get_runs_batch(&task_ids).await {
1351 Ok(map) => map,
1352 Err(e) => {
1353 tracing::warn!(err = %e, "get_runs_batch failed — skipping auto-unblock this tick");
1354 return Ok(());
1355 }
1356 };
1357
1358 let (ci_failure_tasks, other_blocked): (Vec<_>, Vec<_>) =
1360 blocked.into_iter().partition(is_ci_failure_block);
1361
1362 if !ci_failure_tasks.is_empty() {
1364 let gh = match GhHttp::new() {
1365 Ok(g) => Some(g),
1366 Err(e) => {
1367 tracing::warn!(err = %e, "failed to create GhHttp — skipping CI failure unblock this tick");
1368 None
1369 }
1370 };
1371 for task in ci_failure_tasks {
1372 if ci_failure_unblock_cooldown_elapsed(&task)
1373 && let Some(ref gh) = gh
1374 && let Err(e) = try_unblock_ci_failure_task(
1375 gh,
1376 repo,
1377 task_manager,
1378 store,
1379 &task,
1380 auto_merge_in_flight,
1381 )
1382 .await
1383 {
1384 tracing::warn!(task_id = task.id, err = %e, "CI failure unblock check failed");
1385 }
1386 }
1387 }
1388
1389 for task in other_blocked {
1391 if task.block_reason.is_some() {
1392 continue;
1393 }
1394
1395 let task_id_str = task
1396 .external_id
1397 .clone()
1398 .unwrap_or_else(|| format!("internal:{}", task.id));
1399 let dispatch_key = format!("{repo}/{task_id_str}");
1400 if dispatching.contains_key(&dispatch_key) {
1401 continue;
1402 }
1403
1404 let runs = runs_by_task.remove(&task.id).unwrap_or_default();
1405 let relevant_runs: Vec<_> = runs
1406 .into_iter()
1407 .filter(|r| r.run_type == "agent" || r.run_type == "review")
1408 .collect();
1409 let mut failures = Vec::new();
1410 let mut has_review_failure = false;
1411 for run in relevant_runs.iter().rev() {
1412 if let Some(category) = classify_failure_from_run(run) {
1413 if run.run_type == "review" {
1414 has_review_failure = true;
1415 }
1416 failures.push(category);
1417 }
1418 if failures.len() >= 3 {
1419 break;
1420 }
1421 }
1422
1423 if failures.is_empty() || !failures.iter().all(|f| f.is_recoverable()) {
1424 continue;
1425 }
1426
1427 let reason_key = failures.first().map_or("", |f| f.as_str());
1429
1430 let current_reason = task.auto_unblock_last_reason.clone();
1433 let reason_changed = reason_key != current_reason;
1434
1435 let cooldown_count = if reason_changed {
1438 0
1439 } else {
1440 task.auto_unblock_count
1441 };
1442
1443 if cooldown_count >= 3 {
1447 continue;
1448 }
1449
1450 if !auto_unblock_cooldown_elapsed(cooldown_count, &task.auto_unblock_last_at) {
1451 continue;
1452 }
1453
1454 let new_count = cooldown_count + 1;
1458 let now = chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string();
1459
1460 let ext_id = task
1461 .external_id
1462 .clone()
1463 .unwrap_or_else(|| format!("internal:{}", task.id));
1464 let new_status = if has_review_failure {
1465 Status::NeedsReview
1466 } else {
1467 Status::New
1468 };
1469 let new_status_str = if has_review_failure {
1470 "needs_review"
1471 } else {
1472 "new"
1473 };
1474
1475 if let Err(e) = task_manager
1478 .update_task_status(&ExternalId(ext_id), new_status)
1479 .await
1480 {
1481 tracing::warn!(
1482 task_id = task.id,
1483 err = %e,
1484 "auto-unblock failed to update status — counter not incremented"
1485 );
1486 continue;
1487 }
1488
1489 let mut fields: Vec<(&str, serde_json::Value)> = vec![
1491 ("auto_unblock_count", serde_json::json!(new_count)),
1492 ("auto_unblock_last_at", serde_json::json!(now)),
1493 ("auto_unblock_last_reason", serde_json::json!(reason_key)),
1494 ("agent", serde_json::Value::Null),
1495 ("model", serde_json::Value::Null),
1496 ];
1497 fields.push(("no_code_reroutes", serde_json::json!(0)));
1498 fields.push(("no_code_last_agent", serde_json::json!("")));
1499 if has_review_failure {
1500 fields.push(("review_agent_failures", serde_json::json!(0)));
1501 fields.push(("review_invocations", serde_json::json!(0)));
1502 fields.push(("review_resets", serde_json::json!(0)));
1503 }
1504 if let Err(e) = store.set_fields(task.id, &fields).await {
1505 tracing::warn!(task_id = task.id, err = %e, "failed to set auto_unblock fields after successful status update");
1506 }
1509
1510 if let Err(e) = store
1511 .append_activity(
1512 task.id,
1513 "auto_unblock",
1514 Some("blocked"),
1515 Some(new_status_str),
1516 None,
1517 None,
1518 Some(&serde_json::json!({
1519 "reason": reason_key,
1520 "count": if reason_changed { 1 } else { task.auto_unblock_count + 1 },
1521 "failures": failures.iter().map(|f| format!("{f:?}")).collect::<Vec<_>>(),
1522 })),
1523 )
1524 .await
1525 {
1526 tracing::warn!(task_id = task.id, err = %e, "failed to log auto_unblock activity");
1527 }
1528
1529 tracing::info!(
1530 task_id = task.id,
1531 failures = ?failures,
1532 review_failure = has_review_failure,
1533 "auto-unblocked task with recoverable failures"
1534 );
1535 }
1536
1537 Ok(())
1538}
1539
1540pub(crate) async fn auto_unblock_ci_failure_blocked_tasks_global(
1551 task_manager: &Arc<TaskManager>,
1552 store: &Arc<TaskStore>,
1553 auto_merge_in_flight: &Arc<DashSet<String>>,
1554) -> anyhow::Result<()> {
1555 let all_blocked = match store.list_all_by_status_global(TaskStatus::Blocked).await {
1556 Ok(tasks) => tasks,
1557 Err(e) => {
1558 tracing::warn!(err = %e, "global CI-failure blocked sweep: failed to list blocked tasks");
1559 return Ok(());
1560 }
1561 };
1562
1563 let ci_failure_tasks: Vec<_> = all_blocked
1564 .into_iter()
1565 .filter(is_ci_failure_block)
1566 .collect();
1567 if ci_failure_tasks.is_empty() {
1568 return Ok(());
1569 }
1570
1571 let gh = match GhHttp::new() {
1572 Ok(g) => g,
1573 Err(e) => {
1574 tracing::warn!(err = %e, "global CI-failure blocked sweep: failed to create GhHttp");
1575 return Ok(());
1576 }
1577 };
1578
1579 for task in ci_failure_tasks {
1580 if !ci_failure_unblock_cooldown_elapsed(&task) {
1581 continue;
1582 }
1583 if let Err(e) = try_unblock_ci_failure_task(
1584 &gh,
1585 &task.repo,
1586 task_manager,
1587 store,
1588 &task,
1589 auto_merge_in_flight,
1590 )
1591 .await
1592 {
1593 tracing::warn!(
1594 task_id = task.id,
1595 repo = %task.repo,
1596 err = %e,
1597 "global CI-failure blocked sweep: unblock check failed"
1598 );
1599 }
1600 }
1601
1602 Ok(())
1603}
1604
1605pub(crate) async fn refire_and_escalate_stale_needs_review_global(
1619 task_manager: &Arc<TaskManager>,
1620 store: &Arc<TaskStore>,
1621 router: &Router,
1622 dispatching: &Arc<DashMap<String, String>>,
1623 active_repos: &HashSet<String>,
1624) -> anyhow::Result<()> {
1625 const MIN_STALE_NEEDS_REVIEW_MINUTES: i64 = 1;
1626 const MAX_NEEDS_REVIEW_REFIRE_ATTEMPTS: u64 = 5;
1627
1628 let needs_review_tasks = match store
1629 .list_all_by_status_global(TaskStatus::NeedsReview)
1630 .await
1631 {
1632 Ok(tasks) => tasks,
1633 Err(e) => {
1634 tracing::warn!(err = %e, "global NeedsReview refire sweep: failed to list needs_review tasks");
1635 return Ok(());
1636 }
1637 };
1638
1639 if needs_review_tasks.is_empty() {
1640 return Ok(());
1641 }
1642
1643 let all_review_agents_cooled = router.healthy_agent_count("review") == 0;
1646 if all_review_agents_cooled {
1647 tracing::info!(
1648 count = needs_review_tasks.len(),
1649 "global NeedsReview refire sweep: no review agent available — skipping this pass"
1650 );
1651 return Ok(());
1652 }
1653
1654 for stored in needs_review_tasks {
1655 if !active_repos.contains(&stored.repo) {
1657 continue;
1658 }
1659
1660 let external = tasks::store_task_to_external(&stored);
1661 let ext_id = external.id.clone();
1662 let repo = stored.repo.clone();
1663
1664 let age_minutes = match chrono::DateTime::parse_from_rfc3339(&external.updated_at) {
1665 Ok(updated_at) => {
1666 let age = chrono::Utc::now() - updated_at.with_timezone(&chrono::Utc);
1667 if age.num_minutes() < MIN_STALE_NEEDS_REVIEW_MINUTES {
1668 continue;
1669 }
1670 Some(age.num_minutes())
1671 }
1672 Err(_) => None,
1673 };
1674
1675 let dispatch_key = format!("{}/{}", repo, ext_id.0);
1677 if dispatching.contains_key(&dispatch_key) {
1678 continue;
1679 }
1680
1681 let current_refires = u64::try_from(stored.needs_review_refires).unwrap_or(0);
1682 let new_refires = current_refires + 1;
1683
1684 if new_refires > MAX_NEEDS_REVIEW_REFIRE_ATTEMPTS {
1685 tracing::warn!(
1686 task_id = stored.id,
1687 repo = %repo,
1688 new_refires,
1689 "global NeedsReview refire sweep: escalating to Blocked after repeated refires"
1690 );
1691 let fields = [
1692 ("block_reason", serde_json::json!(REBROADCAST_BLOCK_REASON)),
1693 (
1694 "last_error",
1695 serde_json::json!(format!("escalated after {} retries", new_refires)),
1696 ),
1697 ];
1698 if let Err(e) = task_manager
1699 .update_task_status_and_result_by_store_id(
1700 stored.id,
1701 &repo,
1702 &ext_id,
1703 Status::Blocked,
1704 &fields,
1705 )
1706 .await
1707 {
1708 tracing::error!(task_id = stored.id, repo = %repo, err = %e, "global NeedsReview refire sweep: escalation to Blocked failed — skipping to avoid silent auto-unblock loop");
1709 continue;
1710 }
1711 if let Err(e) = store::store_increment_by_id(
1712 &Some(Arc::clone(store)),
1713 stored.id,
1714 "needs_review_refires",
1715 )
1716 .await
1717 {
1718 tracing::warn!(
1719 task_id = stored.id,
1720 repo = %repo,
1721 err = %e,
1722 "global NeedsReview refire sweep: failed to increment needs_review_refires after escalation"
1723 );
1724 }
1725 continue;
1726 }
1727
1728 let required_minutes =
1730 MIN_STALE_NEEDS_REVIEW_MINUTES * (1i64 << u32::try_from(current_refires).unwrap_or(0));
1731
1732 let should_fire = match age_minutes {
1733 Some(age) => age >= required_minutes,
1734 None => true,
1735 };
1736
1737 if !should_fire {
1738 continue;
1739 }
1740
1741 if let Err(e) = task_manager
1742 .update_task_status_by_store_id(stored.id, &repo, &ext_id, Status::NeedsReview)
1743 .await
1744 {
1745 tracing::warn!(
1746 task_id = stored.id,
1747 repo = %repo,
1748 err = %e,
1749 "global NeedsReview refire sweep: failed to re-fire NeedsReview event — not incrementing counter"
1750 );
1751 continue;
1752 }
1753
1754 let fired_refires = match store::store_increment_by_id(
1755 &Some(Arc::clone(store)),
1756 stored.id,
1757 "needs_review_refires",
1758 )
1759 .await
1760 {
1761 Ok(v) => v,
1762 Err(e) => {
1763 tracing::warn!(
1764 task_id = stored.id,
1765 repo = %repo,
1766 err = %e,
1767 "global NeedsReview refire sweep: failed to increment needs_review_refires after re-fire"
1768 );
1769 current_refires + 1
1770 }
1771 };
1772
1773 tracing::info!(
1774 task_id = stored.id,
1775 repo = %repo,
1776 age_minutes,
1777 refires = fired_refires,
1778 required_minutes,
1779 "global NeedsReview refire sweep: re-firing NeedsReview event for stale task"
1780 );
1781 }
1782
1783 Ok(())
1784}
1785
1786#[expect(
1793 clippy::too_many_arguments,
1794 reason = "the engine passes its shared handles one by one"
1795)]
1796pub(crate) async fn sync_tick(
1797 backend: &Arc<dyn ExternalBackend>,
1798 tmux: &Arc<TmuxManager>,
1799 repo: &str,
1800 config: &EngineConfig,
1801 router: &Arc<RwLock<Router>>,
1802 task_manager: &Arc<TaskManager>,
1803 store: &Arc<TaskStore>,
1804 dispatching: &Arc<DashMap<String, String>>,
1805 auto_merge_in_flight: &Arc<DashSet<String>>,
1806) -> anyhow::Result<()> {
1807 if !store.is_leader() {
1809 return Ok(());
1810 }
1811
1812 tracing::debug!("sync tick");
1813
1814 let mut phases: Vec<(&str, u128)> = Vec::new();
1815 let mut at = Instant::now();
1816
1817 let gh_circuit_open = is_github_circuit_open();
1820
1821 if gh_circuit_open {
1822 let remaining = github_circuit_remaining_secs();
1823 tracing::debug!(
1824 remaining_secs = remaining,
1825 "GitHub 5xx circuit breaker open — skipping non-critical sync work"
1826 );
1827 if let Err(e) =
1830 try_kv_set_prefer_store(Some(store), "metrics:orch.github_5xx_circuit.open", "1").await
1831 {
1832 tracing::error!(err = %e, "failed to persist circuit breaker state — may cause thundering herd after restart");
1833 }
1834 } else if let Err(e) =
1835 try_kv_set_prefer_store(Some(store), "metrics:orch.github_5xx_circuit.open", "0").await
1836 {
1837 tracing::error!(err = %e, "failed to clear circuit breaker state");
1838 }
1839
1840 if !gh_circuit_open {
1843 if let Err(e) = ingest_external_tasks(backend, repo, store).await {
1844 tracing::debug!(err = %e, "external task ingest failed");
1845 }
1846 phases.push(("ingest", at.elapsed().as_millis()));
1847 at = Instant::now();
1848
1849 if let Err(e) = backend.reconcile_project_statuses().await {
1853 tracing::warn!(err = %e, "project status reconciliation failed");
1854 }
1855 phases.push(("board_reconcile", at.elapsed().as_millis()));
1856 }
1857 at = Instant::now();
1858
1859 if !gh_circuit_open {
1861 let backend = Arc::clone(backend);
1862 let repo = repo.to_string();
1863 let task_manager = Arc::clone(task_manager);
1864 let store = Arc::clone(store);
1865 tokio::spawn(async move {
1866 if let Err(e) = cleanup_done_worktrees(&backend, &repo, &task_manager, &store).await {
1867 tracing::warn!(err = %e, "worktree cleanup failed");
1868 }
1869 if let Err(e) = prune_old_run_output(&store).await {
1870 tracing::warn!(err = %e, "task run pruning failed");
1871 }
1872 });
1873 }
1874
1875 let (in_review_tasks, needs_review_tasks, blocked_review_candidates) =
1876 match prefetch_review_tasks(store, repo).await {
1877 Ok(tasks) => tasks,
1878 Err(e) => {
1879 tracing::warn!(err = %e, "failed to prefetch review tasks");
1880 (vec![], vec![], vec![])
1881 }
1882 };
1883 phases.push(("prefetch_review", at.elapsed().as_millis()));
1884 at = Instant::now();
1885
1886 if !gh_circuit_open {
1887 let mention_fallback = chrono::Utc::now() - chrono::Duration::hours(24);
1888 let mention_since = kv_get_prefer_store(Some(store), "mentions_last_checked")
1889 .await
1890 .unwrap_or_else(|| mention_fallback.format("%Y-%m-%dT%H:%M:%SZ").to_string());
1891 let shared_since = mention_since.clone();
1892
1893 let cached_router_config = router.read().await.config.clone();
1894 let mut combined_needs = needs_review_tasks.clone();
1896 combined_needs.extend(blocked_review_candidates.clone());
1897
1898 let (merge_result, comments_result, review_result) = tokio::join!(
1899 check_merged_prs(
1900 backend,
1901 repo,
1902 store,
1903 task_manager,
1904 &in_review_tasks,
1905 &combined_needs,
1906 ),
1907 fetch_comments_since(backend, &shared_since),
1908 async {
1909 let gh = match GhHttp::new() {
1910 Ok(g) => g,
1911 Err(e) => {
1912 tracing::warn!(err = %e, "failed to create GitHub HTTP client for review poll");
1913 return Ok(());
1914 }
1915 };
1916 Box::pin(review_open_prs(
1917 backend,
1918 repo,
1919 config,
1920 task_manager,
1921 store,
1922 dispatching,
1923 auto_merge_in_flight,
1924 &in_review_tasks,
1925 &gh,
1926 &cached_router_config,
1927 ))
1928 .await
1929 }
1930 );
1931
1932 if let Err(e) = merge_result {
1933 tracing::warn!(err = %e, "PR merge check failed");
1934 }
1935
1936 match comments_result {
1937 Ok(comments) => {
1938 let filtered = filter_mentions_by_since(&comments, &mention_since);
1939
1940 if let Err(e) =
1941 scan_comments(backend, Some(store), repo, task_manager, Some(&filtered)).await
1942 {
1943 tracing::warn!(err = %e, "comment scan failed");
1944 }
1945 }
1946 Err(e) => {
1947 tracing::warn!(err = %e, "comment fetch failed");
1948 }
1949 }
1950
1951 if let Err(e) = review_result {
1952 tracing::warn!(err = %e, "PR review failed");
1953 }
1954
1955 if let Err(e) = ci_bootstrap::apply_pending(store, task_manager, repo).await {
1956 tracing::warn!(err = %e, "CI bootstrap ruleset apply failed");
1957 }
1958
1959 phases.push(("merge_comments_review", at.elapsed().as_millis()));
1960 }
1961 at = Instant::now();
1962
1963 let in_progress_tasks: Vec<_> = match store.list_by_status(repo, TaskStatus::InProgress).await {
1973 Ok(tasks) => tasks
1974 .into_iter()
1975 .map(|stored| {
1976 let external = tasks::store_task_to_external(&stored);
1977 (stored, external)
1978 })
1979 .collect(),
1980 Err(e) => {
1981 tracing::warn!(err = %e, "failed to list in_progress tasks — skipping stale InProgress check");
1982 vec![]
1983 }
1984 };
1985
1986 if !in_progress_tasks.is_empty() {
1987 let live_sessions: HashSet<String> = tmux
1989 .list_sessions()
1990 .await
1991 .unwrap_or_default()
1992 .into_iter()
1993 .map(|s| s.name)
1994 .collect();
1995
1996 for (_stored, external) in &in_progress_tasks {
1997 const MIN_STALE_MINUTES: i64 = 5;
1999
2000 let task_id = &external.id.0;
2001
2002 let dispatch_key = format!("{repo}/{task_id}");
2004 {
2005 if dispatching.contains_key(&dispatch_key) {
2006 tracing::debug!(
2007 task_id = %task_id,
2008 "task locked by dispatch flow, skipping stale check"
2009 );
2010 continue;
2011 }
2012 }
2013
2014 match chrono::DateTime::parse_from_rfc3339(&external.updated_at) {
2017 Ok(updated_at) => {
2018 let age = chrono::Utc::now() - updated_at.with_timezone(&chrono::Utc);
2019 if age.num_minutes() < MIN_STALE_MINUTES {
2020 tracing::debug!(
2021 task_id = %task_id,
2022 age_seconds = age.num_seconds(),
2023 "InProgress task is too young to be considered stale, skipping"
2024 );
2025 continue;
2026 }
2027 }
2028 Err(e) => {
2029 tracing::warn!(
2030 task_id = %task_id,
2031 ts = %external.updated_at,
2032 err = %e,
2033 "invalid updated_at timestamp — treating task as potentially stale"
2034 );
2035 }
2036 }
2037
2038 let agent_session = tmux.session_name(repo, task_id);
2039 if live_sessions.contains(&agent_session) {
2040 continue;
2042 }
2043
2044 {
2048 tracing::warn!(
2049 task_id = %task_id,
2050 session = %agent_session,
2051 "InProgress task has no active agent session — resetting to Routed"
2052 );
2053 match task_manager
2054 .update_task_status_if(&external.id, Status::Routed, Status::InProgress)
2055 .await
2056 {
2057 Err(e) => {
2058 tracing::error!(task_id = %task_id, err = %e, "failed to reset stale InProgress task — task may be stuck in InProgress indefinitely");
2059 }
2060 Ok(false) => {
2061 tracing::debug!(
2062 task_id = %task_id,
2063 "stale InProgress reset skipped — task already transitioned (concurrent Done/Blocked/NeedsReview)"
2064 );
2065 }
2066 Ok(true) => {
2067 }
2069 }
2070 }
2071 }
2072 }
2073
2074 phases.push(("stale_in_progress", at.elapsed().as_millis()));
2075 at = Instant::now();
2076 if let Err(e) =
2078 auto_unblock_blocked_tasks(repo, task_manager, store, dispatching, auto_merge_in_flight)
2079 .await
2080 {
2081 tracing::warn!(err = %e, "auto-unblock failed");
2082 }
2083
2084 phases.push(("auto_unblock", at.elapsed().as_millis()));
2085
2086 engine::log_phase_timings("sync", repo, &phases);
2087 Ok(())
2088}
2089
2090pub(crate) async fn sync_shared_tail(router: &Arc<RwLock<Router>>, store: Option<&Arc<TaskStore>>) {
2095 let Some(store) = store else {
2096 return;
2097 };
2098 if !store.is_leader() {
2099 return;
2100 }
2101
2102 let (available_agents, config) = {
2106 let r = router.read().await;
2107 (r.available_agents.clone(), r.config.clone())
2108 };
2109 let model_checker = |agent: &str| -> bool {
2110 for comp in &["simple", "medium", "complex", "review"] {
2111 if config.has_available_model_for_complexity(agent, comp) {
2112 return true;
2113 }
2114 }
2115 false
2116 };
2117 refresh_degraded_agents(
2118 store,
2119 &available_agents,
2120 &model_checker,
2121 router_config::health_check_window_hours(),
2122 router_config::degraded_rate_limit_threshold(),
2123 )
2124 .await;
2125
2126 emit_degraded_agents_if_needed(&available_agents, &config, Some(store)).await;
2127}
2128
2129async fn scan_comments(
2139 backend: &Arc<dyn ExternalBackend>,
2140 store: Option<&Arc<TaskStore>>,
2141 repo: &str,
2142 task_manager: &Arc<TaskManager>,
2143 prefetched_comments: Option<&[Mention]>,
2144) -> anyhow::Result<()> {
2145 let current_user = match backend.get_authenticated_user().await {
2146 Ok(Some(u)) => format!("@{u}"),
2147 Ok(None) => {
2148 tracing::debug!("backend does not support user identity, skipping comment scan");
2149 return Ok(());
2150 }
2151 Err(e) => {
2152 tracing::warn!(err = %e, "failed to get current user, skipping comment scan");
2153 return Ok(());
2154 }
2155 };
2156
2157 let comments = if let Some(c) = prefetched_comments {
2158 c.to_vec()
2159 } else {
2160 let fallback = chrono::Utc::now() - chrono::Duration::hours(24);
2161 let since_str = match kv_get_prefer_store(store, "mentions_last_checked").await {
2162 Some(ts) => ts,
2163 None => fallback.format("%Y-%m-%dT%H:%M:%SZ").to_string(),
2164 };
2165 match backend.get_mentions(&since_str).await {
2166 Ok(m) => m,
2167 Err(e) => {
2168 tracing::warn!(err = %e, "failed to get comments");
2169 return Ok(());
2170 }
2171 }
2172 };
2173
2174 let existing_mentions: HashSet<String> = if let Some(s) = store {
2176 match s.list_source_ids_by_source(repo, "mention", 30).await {
2177 Ok(ids) => ids.into_iter().collect(),
2178 Err(e) => {
2179 tracing::warn!(err = %e, "failed to load mention source IDs — skipping to prevent duplicates");
2180 return Ok(());
2181 }
2182 }
2183 } else {
2184 HashSet::new()
2185 };
2186
2187 let gh_cell: OnceCell<GhHttp> = OnceCell::new();
2191 let get_gh = || async { gh_cell.get_or_try_init(|| async { GhHttp::new() }).await };
2192
2193 let actions: Vec<CommentAction> = comments
2196 .iter()
2197 .map(|c| {
2198 classify_comment(
2199 &c.body,
2200 c.issue_url.as_deref(),
2201 ¤t_user,
2202 existing_mentions.contains(&c.id),
2203 )
2204 })
2205 .collect();
2206
2207 let pr_check_inputs: Vec<(usize, String)> = actions
2211 .iter()
2212 .enumerate()
2213 .filter_map(|(i, action)| {
2214 if let CommentAction::CreateMentionTask {
2215 issue_num: Some(num),
2216 } = action
2217 {
2218 Some((i, num.clone()))
2219 } else {
2220 None
2221 }
2222 })
2223 .collect();
2224
2225 let is_pr_values: Vec<bool> = if pr_check_inputs.is_empty() {
2227 Vec::new()
2228 } else {
2229 let gh = get_gh().await?;
2230 future::join_all(
2231 pr_check_inputs
2232 .iter()
2233 .map(|(_, num)| async move { gh.is_pull_request(repo, num).await }),
2234 )
2235 .await
2236 };
2237
2238 let is_pr_map: HashMap<usize, bool> = pr_check_inputs
2240 .iter()
2241 .map(|(i, _)| *i)
2242 .zip(is_pr_values)
2243 .collect();
2244
2245 let mut cursor = MentionCursor::default();
2246
2247 for (comment_idx, (comment, action)) in comments.iter().zip(actions).enumerate() {
2248 match action {
2249 CommentAction::Skip => {
2250 cursor.advance(&comment.created_at);
2251 }
2252
2253 CommentAction::ExecuteCommand { command, issue_num } => {
2254 let store_opt: Option<Arc<dyn CommandStoreOps>> =
2255 store.map(|s| Arc::clone(s) as Arc<dyn CommandStoreOps>);
2256 let gh = get_gh().await?;
2257 let outcome = validate_and_run_command(
2258 backend,
2259 gh,
2260 repo,
2261 &issue_num,
2262 &command,
2263 &comment.author,
2264 &store_opt,
2265 task_manager,
2266 )
2267 .await;
2268 match outcome {
2269 CommandOutcome::FetchFailed | CommandOutcome::CollaboratorCheckFailed => {
2272 tracing::warn!(
2273 comment_id = %comment.id,
2274 command = %command,
2275 "deferring slash command after a transient GitHub error"
2276 );
2277 cursor.block_on_gap();
2278 break;
2279 }
2280 CommandOutcome::Executed { .. }
2283 | CommandOutcome::NotOpen { .. }
2284 | CommandOutcome::NotCollaborator { .. } => cursor.advance(&comment.created_at),
2285 }
2286 }
2287
2288 CommentAction::ExecuteCommandForMention { command, .. } => {
2289 if let Err(e) = backend.acknowledge_mention(&comment.id).await {
2290 tracing::debug!(err = %e, mention_id = %comment.id, "failed to acknowledge mention");
2291 } else {
2295 let gh = get_gh().await?;
2298 let advanced_safely = handle_slash_command(
2299 backend,
2300 store,
2301 repo,
2302 task_manager,
2303 gh,
2304 comment,
2305 &command,
2306 &mut cursor,
2307 )
2308 .await?;
2309 if !advanced_safely {
2310 cursor.block_on_gap();
2311 break;
2312 }
2313 }
2314 }
2315
2316 CommentAction::CreateMentionTask { issue_num } => {
2317 if let Err(e) = backend.acknowledge_mention(&comment.id).await {
2318 tracing::debug!(err = %e, mention_id = %comment.id, "failed to acknowledge mention");
2319 } else {
2323 let is_pr = is_pr_map.get(&comment_idx).copied().unwrap_or(false);
2328
2329 let (title, task_body) = match (&issue_num, is_pr) {
2330 (Some(num), false) => (
2331 format!("Respond to mention by @{} on task #{num}", comment.author),
2332 format!(
2333 "Mention by @{} on GitHub issue #{num}:\n\n{}\n\n---\nThis mention was posted on issue #{num}. Review the full issue to understand the context before responding.",
2334 comment.author, comment.body
2335 ),
2336 ),
2337 (Some(num), true) => (
2338 format!("Respond to mention by @{} on PR #{num}", comment.author),
2339 format!(
2340 "Mention by @{} on GitHub PR #{num}:\n\n{}\n\n---\nThis mention was posted on PR #{num}. Review the full PR (diff, description, and comments) to understand the context before responding.",
2341 comment.author, comment.body
2342 ),
2343 ),
2344 (None, _) => (
2345 format!("Respond to mention by @{}", comment.author),
2346 format!("Mention by @{}:\n\n{}", comment.author, comment.body),
2347 ),
2348 };
2349
2350 if let Some(s) = store {
2351 let parent_id = match issue_num {
2354 Some(ref num) => {
2355 match resolve_mention_parent_id(s, repo, num, is_pr).await {
2356 Ok(id) => id,
2357 Err(e) => {
2358 tracing::warn!(
2359 mention_id = %comment.id,
2360 err = %e,
2361 "deferring mention task due to parent lookup failure"
2362 );
2363 return Err(anyhow::anyhow!(
2364 "deferring mention {} due to parent lookup failure",
2365 comment.id
2366 ));
2367 }
2368 }
2369 }
2370 None => None,
2371 };
2372 if !record_mention_task(
2373 s,
2374 repo,
2375 comment,
2376 &title,
2377 &task_body,
2378 parent_id,
2379 &mut cursor,
2380 )
2381 .await
2382 {
2383 cursor.block_on_gap();
2384 break;
2385 }
2386 }
2387 }
2388 }
2389 }
2390 }
2391
2392 if let Some(ts) = cursor.into_last_safe_ts() {
2393 kv_set_prefer_store(store, "mentions_last_checked", &ts).await;
2394 }
2395
2396 Ok(())
2397}
2398
2399struct DegradedAgentDetail {
2401 agent: String,
2403 reason: String,
2407 cooled_models: Vec<String>,
2410}
2411
2412struct AgentPoolHealth {
2414 agent: String,
2415 cooled_models: Vec<String>,
2416 persistent_failed_models: Vec<String>,
2417 total_configured_models: usize,
2418}
2419
2420pub(crate) async fn emit_degraded_agents_if_needed(
2429 available_agents: &[String],
2430 config: &RouterConfig,
2431 store: Option<&Arc<TaskStore>>,
2432) {
2433 const STALE_AGENTS_KV: &str = "metrics:orch.model_pool_stale_persistent.agents";
2436
2437 const COMPLEXITIES: &[&str] = &["simple", "medium", "complex", "review"];
2439 const HEAVILY_COOLED_RATIO_THRESHOLD: f64 = 0.5;
2440
2441 let mut details: Vec<DegradedAgentDetail> = Vec::new();
2442 let mut pool_health: Vec<AgentPoolHealth> = Vec::new();
2443
2444 for agent in available_agents {
2445 let agent_in_cd = is_agent_in_cooldown(agent);
2446
2447 let mut cooled_models: Vec<String> = Vec::new();
2449 let mut persistent_failed_models: Vec<String> = Vec::new();
2450 let mut seen_models = HashSet::new();
2451 for comp in COMPLEXITIES {
2452 if let Some(pool) = config.model_pool_for_complexity(agent, comp) {
2453 for model in pool {
2454 if seen_models.insert(model.clone()) && is_model_in_cooldown(agent, &model) {
2455 let key = format!("{agent}:{model}");
2456 if matches!(
2457 cooldown_reason(&key).as_deref(),
2458 Some("persistent_model_error" | "model_not_found")
2459 ) {
2460 persistent_failed_models.push(model.clone());
2461 }
2462 cooled_models.push(model);
2463 }
2464 }
2465 }
2466 }
2467 let total_configured_models = seen_models.len();
2468 pool_health.push(AgentPoolHealth {
2469 agent: agent.clone(),
2470 cooled_models: cooled_models.clone(),
2471 persistent_failed_models: persistent_failed_models.clone(),
2472 total_configured_models,
2473 });
2474
2475 let has_model = COMPLEXITIES
2478 .iter()
2479 .any(|comp| config.has_available_model_for_complexity(agent, comp));
2480
2481 if agent_in_cd || !has_model {
2482 let reason = if agent_in_cd {
2483 cooldown_reason(agent).unwrap_or_else(|| "agent_cooldown".to_string())
2484 } else {
2485 "no_available_model".to_string()
2486 };
2487 details.push(DegradedAgentDetail {
2488 agent: agent.clone(),
2489 reason,
2490 cooled_models,
2491 });
2492 }
2493 }
2494
2495 let count = details.len();
2496
2497 if let Err(e) = try_kv_set_prefer_store(
2500 store,
2501 "metrics:orch.agents_degraded.count",
2502 &count.to_string(),
2503 )
2504 .await
2505 {
2506 tracing::warn!(err = %e, "failed to persist degraded agents count metric");
2507 }
2508
2509 if count >= 3 {
2510 let degraded_agents: Vec<&str> = details.iter().map(|d| d.agent.as_str()).collect();
2512 let cooled_models_dim: Vec<String> = details
2513 .iter()
2514 .filter(|d| !d.cooled_models.is_empty())
2515 .map(|d| format!("{}:[{}]", d.agent, d.cooled_models.join(",")))
2516 .collect();
2517 let reasons_dim: Vec<String> = details
2518 .iter()
2519 .map(|d| format!("{}={}", d.agent, d.reason))
2520 .collect();
2521
2522 tracing::warn!(
2523 degraded_count = count,
2524 degraded_agents = ?degraded_agents,
2525 cooled_models = %cooled_models_dim.join("; "),
2526 cooldown_reasons = %reasons_dim.join("; "),
2527 "multi-agent degradation detected"
2528 );
2529
2530 if let Err(e) =
2532 try_kv_set_prefer_store(store, "metrics:orch.agents_degraded.alert", "1").await
2533 {
2534 tracing::warn!(err = %e, "failed to persist degraded agents alert metric");
2535 }
2536 } else if let Err(e) =
2537 try_kv_set_prefer_store(store, "metrics:orch.agents_degraded.alert", "0").await
2538 {
2539 tracing::warn!(err = %e, "failed to clear degraded agents alert metric");
2541 }
2542
2543 let heavily_cooled_agents: Vec<String> = pool_health
2547 .iter()
2548 .filter_map(|d| {
2549 if d.total_configured_models == 0 || d.persistent_failed_models.is_empty() {
2550 return None;
2551 }
2552 #[expect(
2553 clippy::cast_precision_loss,
2554 reason = "model counts are far below 2^52"
2555 )]
2556 let ratio = d.cooled_models.len() as f64 / d.total_configured_models as f64;
2557 (ratio >= HEAVILY_COOLED_RATIO_THRESHOLD).then(|| {
2558 format!(
2559 "{}:{}/{}:{}",
2560 d.agent,
2561 d.cooled_models.len(),
2562 d.total_configured_models,
2563 d.persistent_failed_models.join(",")
2564 )
2565 })
2566 })
2567 .collect();
2568
2569 let current_dim = heavily_cooled_agents.join(",");
2570 let prev_dim = kv_get_prefer_store(store, STALE_AGENTS_KV)
2571 .await
2572 .unwrap_or_default();
2573
2574 if heavily_cooled_agents.is_empty() {
2575 if !prev_dim.is_empty() {
2576 tracing::info!("agent model pool stale condition cleared");
2577 }
2578 if let Err(e) = try_kv_set_prefer_store(store, STALE_AGENTS_KV, "").await {
2579 tracing::warn!(err = %e, "failed to clear stale model pool agents state");
2580 }
2581 if let Err(e) =
2582 try_kv_set_prefer_store(store, "metrics:orch.model_pool_stale_persistent.alert", "0")
2583 .await
2584 {
2585 tracing::warn!(err = %e, "failed to clear stale model pool alert metric");
2586 }
2587 } else {
2588 if current_dim != prev_dim {
2589 tracing::warn!(
2590 affected_agents = ?heavily_cooled_agents,
2591 "agent model pool appears stale: persistent model failures in heavily cooled pool"
2592 );
2593 }
2594 if let Err(e) = try_kv_set_prefer_store(store, STALE_AGENTS_KV, ¤t_dim).await {
2595 tracing::warn!(err = %e, "failed to persist stale model pool agents state");
2596 }
2597 if let Err(e) =
2598 try_kv_set_prefer_store(store, "metrics:orch.model_pool_stale_persistent.alert", "1")
2599 .await
2600 {
2601 tracing::warn!(err = %e, "failed to persist stale model pool alert metric");
2602 }
2603 }
2604}
2605
2606pub(crate) async fn ingest_external_tasks(
2614 backend: &Arc<dyn ExternalBackend>,
2615 repo: &str,
2616 store: &Arc<TaskStore>,
2617) -> anyhow::Result<()> {
2618 #[derive(Clone)]
2619 struct DuplicateTarget {
2620 id: String,
2621 status_label: String,
2622 }
2623
2624 let mut existing_by_title: HashMap<String, DuplicateTarget> = HashMap::new();
2625 let mut accepted_by_title: HashMap<String, DuplicateTarget> = HashMap::new();
2626
2627 let cutoff = (chrono::Utc::now() - chrono::Duration::hours(24))
2628 .format("%Y-%m-%dT%H:%M:%SZ")
2629 .to_string();
2630 match store.list_for_doctor(repo, &cutoff).await {
2631 Ok(recent) => {
2632 let mut insert_target = |task: &Task| {
2633 if task.origin == "internal" {
2634 return;
2635 }
2636 let Some(ext_id) = task.external_id.as_ref() else {
2637 return;
2638 };
2639 let status_label = task
2640 .labels
2641 .iter()
2642 .find(|l| l.starts_with("status:"))
2643 .cloned()
2644 .unwrap_or_default();
2645 existing_by_title
2646 .entry(task.title.clone())
2647 .or_insert(DuplicateTarget {
2648 id: ext_id.clone(),
2649 status_label,
2650 });
2651 };
2652
2653 for task in recent.iter().filter(|t| t.status != TaskStatus::Done) {
2655 insert_target(task);
2656 }
2657 for task in recent.iter().filter(|t| t.status == TaskStatus::Done) {
2658 insert_target(task);
2659 }
2660 }
2661 Err(e) => {
2662 tracing::warn!(repo, err = %e, "ingest: failed to load recent tasks for dedup");
2663 }
2664 }
2665
2666 let all_tasks: Vec<(ExternalTask, Option<Status>)> =
2676 match backend.list_active_open_issues().await {
2677 Ok(tasks) => tasks,
2678 Err(e) => {
2679 tracing::warn!(
2680 repo,
2681 err = %e,
2682 "ingest: failed to list open issues — no new issue is ingested this cycle"
2683 );
2684 return Ok(());
2685 }
2686 };
2687
2688 let mut id_status_pairs: Vec<(i64, Status)> = Vec::new();
2691 let mut new_task_node_ids: Vec<(i64, String)> = Vec::new();
2694 let existing_ext_ids: HashSet<String> = match store.existing_external_ids(repo).await {
2699 Ok(s) => s,
2700 Err(e) => {
2701 tracing::warn!(repo = repo, err = %e, "ingest: failed to load existing external ids — proceeding");
2702 HashSet::new()
2703 }
2704 };
2705 for (task, status) in &all_tasks {
2706 let is_new = !existing_ext_ids.contains(&task.id.0);
2709
2710 if is_new {
2711 let duplicate_target = existing_by_title
2712 .get(&task.title)
2713 .or_else(|| accepted_by_title.get(&task.title));
2714 if let Some(target) = duplicate_target
2715 && target.id != task.id.0
2716 {
2717 let label = target.status_label.as_str();
2718 let should_dedupe =
2719 match backend.has_open_issue_with_title(&task.title, label).await {
2720 Ok(true) => true,
2721 Ok(false) => false,
2722 Err(e) => {
2723 tracing::warn!(
2724 task_id = task.id.0,
2725 err = %e,
2726 "ingest: dedup check failed — defaulting to keep issue"
2727 );
2728 false
2729 }
2730 };
2731
2732 if should_dedupe {
2733 let comment = format!(
2734 "Duplicate of #{id} — closing to avoid duplicate work.{footer}",
2735 id = target.id,
2736 footer = engine::orch_footer()
2737 );
2738 if let Err(e) = backend.post_comment(&task.id, &comment).await {
2739 tracing::warn!(
2740 task_id = task.id.0,
2741 err = %e,
2742 "ingest: failed to comment on duplicate issue"
2743 );
2744 }
2745 match backend.update_status(&task.id, Status::Done).await {
2746 Ok(()) => {
2747 tracing::info!(
2748 task_id = task.id.0,
2749 duplicate_of = target.id,
2750 "ingest: closed duplicate issue"
2751 );
2752 continue;
2753 }
2754 Err(e) => {
2755 tracing::warn!(
2756 task_id = task.id.0,
2757 err = %e,
2758 "ingest: failed to close duplicate issue — falling through to ingest normally"
2759 );
2760 }
2761 }
2762 }
2763 }
2764 }
2765
2766 match store.ensure_external_task(repo, task).await {
2767 Ok(store_id) => {
2768 if let Ok(number) = task.id.0.parse::<i64>()
2769 && let Err(err) = dependencies::sync_issue_dependencies(
2770 backend.as_ref(),
2771 store,
2772 store_id,
2773 repo,
2774 Some(number),
2775 &task.body,
2776 )
2777 .await
2778 {
2779 tracing::warn!(
2780 task_id = task.id.0,
2781 err = %err,
2782 "ingest: dependency sync failed"
2783 );
2784 }
2785 if let Some(s) = status {
2786 id_status_pairs.push((store_id, *s));
2787 }
2788 if is_new {
2790 let status_label = task
2791 .labels
2792 .iter()
2793 .find(|l| l.starts_with("status:"))
2794 .cloned()
2795 .unwrap_or_default();
2796 accepted_by_title
2797 .entry(task.title.clone())
2798 .or_insert(DuplicateTarget {
2799 id: task.id.0.clone(),
2800 status_label,
2801 });
2802 if let Err(e) = backend.acknowledge_issue(&task.id).await {
2803 tracing::debug!(
2804 task_id = task.id.0,
2805 err = %e,
2806 "ingest: acknowledgment failed"
2807 );
2808 }
2809 let task_status = status.unwrap_or(Status::New);
2812 match backend.sync_to_project(&task.id, task_status).await {
2813 Ok(Some(node_id)) => {
2814 new_task_node_ids.push((store_id, node_id));
2815 }
2816 Ok(None) => {
2817 }
2819 Err(e) => {
2820 tracing::debug!(
2821 task_id = task.id.0,
2822 err = %e,
2823 "ingest: project board sync failed"
2824 );
2825 }
2826 }
2827 }
2828 }
2829 Err(e) => {
2830 tracing::debug!(task_id = task.id.0, ?e, "ingest: upsert failed");
2831 }
2832 }
2833 }
2834
2835 if !id_status_pairs.is_empty() {
2841 let all_ids: Vec<i64> = id_status_pairs.iter().map(|(id, _)| *id).collect();
2842 let existing_map = match store.get_batch(&all_ids).await {
2843 Ok(map) => map,
2844 Err(e) => {
2845 tracing::warn!(err = %e, task_count = all_ids.len(), "ingest: get_batch failed — skipping initial status sync for newly ingested tasks this tick");
2846 return Ok(());
2847 }
2848 };
2849 for (store_id, status) in id_status_pairs {
2850 if let Some(existing) = existing_map.get(&store_id)
2851 && existing.status == TaskStatus::New
2852 {
2853 let db_status = status_to_task_status(status);
2854 if db_status != TaskStatus::New
2855 && let Err(e) = store.update_status(store_id, db_status).await
2856 {
2857 tracing::debug!(?e, "ingest: status sync failed");
2858 }
2859 }
2860 }
2861 }
2862
2863 if !new_task_node_ids.is_empty() {
2866 let node_ids: Vec<String> = new_task_node_ids.iter().map(|(_, n)| n.clone()).collect();
2867 match backend.get_project_item_estimates(&node_ids).await {
2868 Ok(estimates) => {
2869 for (store_id, node_id) in new_task_node_ids {
2870 if let Some(&estimate) = estimates.get(&node_id)
2871 && estimate > 0
2872 && let Err(e) = store.set_estimate_if_zero(store_id, estimate).await
2873 {
2874 tracing::debug!(
2875 store_id,
2876 estimate,
2877 err = %e,
2878 "ingest: failed to set estimate from project"
2879 );
2880 }
2881 }
2882 }
2883 Err(e) => {
2884 tracing::debug!(
2885 err = %e,
2886 "ingest: project estimate fetch failed — skipping estimate sync this tick"
2887 );
2888 }
2889 }
2890 }
2891
2892 Ok(())
2893}
2894
2895#[cfg(test)]
2896mod tests {
2897 use super::*;
2898 use super::{CommentAction, MentionCursor, classify_comment};
2899 use crate::backends::test_helpers::NoopBackend;
2900 use crate::backends::{ExternalId, ExternalTask, Mention, Status};
2901 use crate::engine::commands::OwnerCommand;
2902 use crate::engine::cooldown::record_persistent_model_failure;
2903 use crate::engine::cooldown::set_agent_cooldown;
2904 use crate::engine::dispatch_guard::DispatchGuard;
2905 use crate::engine::router::Router;
2906 use crate::engine::router::RouterConfig;
2907 use crate::github::types::GitHubIssue;
2908 use crate::github::types::GitHubUser;
2909 use crate::store::NewTask;
2910 use crate::store::{RunTokenUsage, TaskStore};
2911 use async_trait::async_trait;
2912 use serial_test::serial;
2913 use std::slice;
2914 use std::sync::Arc;
2915 use std::sync::atomic::AtomicUsize;
2916 use std::sync::atomic::Ordering;
2917 use tokio::sync::Mutex;
2918 use tokio::sync::oneshot;
2919
2920 struct IngestMockBackend {
2924 tasks: Vec<(String, Vec<ExternalTask>)>,
2926 dedup_result: bool,
2927 comments: Mutex<Vec<(String, String)>>,
2928 status_updates: Mutex<Vec<(String, Status)>>,
2929 project_syncs: Mutex<Vec<(String, Status)>>,
2930 project_reconciliations: AtomicUsize,
2931 list_calls: AtomicUsize,
2933 }
2934
2935 impl IngestMockBackend {
2936 fn with_tasks(tasks: Vec<(Status, ExternalTask)>) -> Arc<Self> {
2937 Self::with_tasks_and_dedup(tasks, false)
2938 }
2939
2940 fn with_tasks_and_dedup(
2941 tasks: Vec<(Status, ExternalTask)>,
2942 dedup_result: bool,
2943 ) -> Arc<Self> {
2944 let mut grouped: Vec<(String, Vec<ExternalTask>)> = Vec::new();
2945 for (status, task) in tasks {
2946 let label = status.as_label().to_string();
2947 if let Some(entry) = grouped.iter_mut().find(|(l, _)| l == &label) {
2948 entry.1.push(task);
2949 } else {
2950 grouped.push((label, vec![task]));
2951 }
2952 }
2953 Arc::new(Self {
2954 tasks: grouped,
2955 dedup_result,
2956 comments: Mutex::new(Vec::new()),
2957 status_updates: Mutex::new(Vec::new()),
2958 project_syncs: Mutex::new(Vec::new()),
2959 project_reconciliations: AtomicUsize::new(0),
2960 list_calls: AtomicUsize::new(0),
2961 })
2962 }
2963 }
2964
2965 #[async_trait]
2966 impl ExternalBackend for IngestMockBackend {
2967 fn name(&self) -> &'static str {
2968 "ingest-mock"
2969 }
2970 async fn create_task(&self, _: &str, _: &str, _: &[String]) -> anyhow::Result<ExternalId> {
2971 Ok(ExternalId("new".into()))
2972 }
2973 async fn get_task(&self, _: &ExternalId) -> anyhow::Result<ExternalTask> {
2974 anyhow::bail!("not implemented")
2975 }
2976 async fn list_by_status(&self, status: Status) -> anyhow::Result<Vec<ExternalTask>> {
2977 let label = status.as_label().to_string();
2978 Ok(self
2979 .tasks
2980 .iter()
2981 .find(|(l, _)| l == &label)
2982 .map(|(_, t)| t.clone())
2983 .unwrap_or_default())
2984 }
2985 async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
2986 Ok(vec![])
2987 }
2988 async fn list_active_open_issues(
2989 &self,
2990 ) -> anyhow::Result<Vec<(ExternalTask, Option<Status>)>> {
2991 self.list_calls.fetch_add(1, Ordering::Relaxed);
2992 let mut result = Vec::new();
2993 for (label, tasks) in &self.tasks {
2994 let status = match label.as_str() {
2995 "status:routed" => Some(Status::Routed),
2996 "status:in_progress" => Some(Status::InProgress),
2997 "status:needs_review" => Some(Status::NeedsReview),
2998 "status:in_review" => Some(Status::InReview),
2999 "status:approved" => Some(Status::Approved),
3000 "status:blocked" => Some(Status::Blocked),
3001 _ => None,
3002 };
3003 for task in tasks {
3004 result.push((task.clone(), status));
3005 }
3006 }
3007 Ok(result)
3008 }
3009 async fn post_comment(&self, id: &ExternalId, body: &str) -> anyhow::Result<()> {
3010 self.comments
3011 .lock()
3012 .await
3013 .push((id.0.clone(), body.to_string()));
3014 Ok(())
3015 }
3016 async fn set_labels(&self, _: &ExternalId, _: &[String]) -> anyhow::Result<()> {
3017 Ok(())
3018 }
3019 async fn remove_label(&self, _: &ExternalId, _: &str) -> anyhow::Result<()> {
3020 Ok(())
3021 }
3022 async fn get_sub_issues(&self, _: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
3023 Ok(vec![])
3024 }
3025 async fn health_check(&self) -> anyhow::Result<()> {
3026 Ok(())
3027 }
3028 async fn get_mentions(&self, _: &str) -> anyhow::Result<Vec<Mention>> {
3029 Ok(vec![])
3030 }
3031 async fn update_status(&self, id: &ExternalId, status: Status) -> anyhow::Result<()> {
3032 self.status_updates
3033 .lock()
3034 .await
3035 .push((id.0.clone(), status));
3036 Ok(())
3037 }
3038
3039 async fn has_open_issue_with_title(&self, _: &str, _: &str) -> anyhow::Result<bool> {
3040 Ok(self.dedup_result)
3041 }
3042
3043 async fn sync_to_project(
3044 &self,
3045 id: &ExternalId,
3046 status: Status,
3047 ) -> anyhow::Result<Option<String>> {
3048 self.project_syncs.lock().await.push((id.0.clone(), status));
3049 Ok(None) }
3051
3052 async fn reconcile_project_statuses(&self) -> anyhow::Result<()> {
3053 self.project_reconciliations.fetch_add(1, Ordering::Relaxed);
3054 Ok(())
3055 }
3056 }
3057
3058 fn make_ext_task(id: &str, title: &str) -> ExternalTask {
3059 ExternalTask {
3060 id: ExternalId(id.to_string()),
3061 title: title.to_string(),
3062 body: String::new(),
3063 state: "open".to_string(),
3064 labels: vec![],
3065 author: "user".to_string(),
3066 created_at: "2026-01-01T00:00:00Z".to_string(),
3067 updated_at: "2026-01-01T00:00:00Z".to_string(),
3068 url: format!("https://github.com/owner/repo/issues/{id}"),
3069 }
3070 }
3071
3072 #[serial(cooldown_state)]
3073 #[tokio::test]
3074 async fn ingest_upserts_tasks_into_store() {
3075 cooldown::reset_global_state().await;
3076 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![
3077 (Status::New, make_ext_task("1", "First issue")),
3078 (Status::InProgress, make_ext_task("2", "Second issue")),
3079 ]);
3080 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3081
3082 ingest_external_tasks(&backend, "owner/repo", &store)
3083 .await
3084 .unwrap();
3085
3086 let all = store.list_all("owner/repo").await.unwrap();
3088 assert_eq!(all.len(), 2);
3089
3090 let task1 = store
3091 .get_by_external_id("owner/repo", "1")
3092 .await
3093 .unwrap()
3094 .unwrap();
3095 assert_eq!(task1.title, "First issue");
3096 assert_eq!(task1.status, TaskStatus::New);
3097
3098 let task2 = store
3099 .get_by_external_id("owner/repo", "2")
3100 .await
3101 .unwrap()
3102 .unwrap();
3103 assert_eq!(task2.title, "Second issue");
3104 assert_eq!(task2.status, TaskStatus::InProgress);
3105 }
3106
3107 #[serial(cooldown_state)]
3108 #[tokio::test]
3109 async fn ingest_takes_every_open_issue_whatever_its_age_or_the_last_ingest() {
3110 cooldown::reset_global_state().await;
3114 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3115 store
3116 .kv_set("issues_last_ingested:owner/repo", "2026-10-26T00:00:00Z")
3117 .await
3118 .unwrap();
3119
3120 let mut old = make_ext_task("73", "Filed weeks ago, never ingested");
3121 old.updated_at = "2026-01-01T00:00:00Z".to_string();
3122 let backend = IngestMockBackend::with_tasks(vec![(Status::New, old)]);
3123 let backend_dyn: Arc<dyn ExternalBackend> = Arc::<IngestMockBackend>::clone(&backend);
3124 ingest_external_tasks(&backend_dyn, "owner/repo", &store)
3125 .await
3126 .unwrap();
3127 assert!(
3128 store
3129 .get_by_external_id("owner/repo", "73")
3130 .await
3131 .unwrap()
3132 .is_some()
3133 );
3134
3135 let backend = IngestMockBackend::with_tasks(vec![(
3137 Status::New,
3138 make_ext_task("78", "Opened after the restart"),
3139 )]);
3140 let backend_dyn: Arc<dyn ExternalBackend> = Arc::<IngestMockBackend>::clone(&backend);
3141 ingest_external_tasks(&backend_dyn, "owner/repo", &store)
3142 .await
3143 .unwrap();
3144 assert_eq!(
3145 backend.list_calls.load(Ordering::Relaxed),
3146 1,
3147 "one sync is one listing"
3148 );
3149 assert!(
3150 store
3151 .get_by_external_id("owner/repo", "78")
3152 .await
3153 .unwrap()
3154 .is_some()
3155 );
3156 assert_eq!(
3157 store
3158 .kv_get("issues_last_ingested:owner/repo")
3159 .await
3160 .unwrap()
3161 .as_deref(),
3162 Some("2026-10-26T00:00:00Z"),
3163 "the ingest must keep no cursor, so a stale one is never read or advanced"
3164 );
3165 }
3166
3167 #[serial(cooldown_state)]
3168 #[tokio::test]
3169 async fn ingest_updates_existing_tasks() {
3170 cooldown::reset_global_state().await;
3171 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![(
3172 Status::Routed,
3173 make_ext_task("42", "Updated title"),
3174 )]);
3175 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3176
3177 store
3179 .create(&NewTask {
3180 external_id: Some("42".to_string()),
3181 repo: "owner/repo".to_string(),
3182 origin: "github".to_string(),
3183 title: "Original title".to_string(),
3184 ..Default::default()
3185 })
3186 .await
3187 .unwrap();
3188
3189 ingest_external_tasks(&backend, "owner/repo", &store)
3190 .await
3191 .unwrap();
3192
3193 let task = store
3195 .get_by_external_id("owner/repo", "42")
3196 .await
3197 .unwrap()
3198 .unwrap();
3199 assert_eq!(task.title, "Updated title");
3200 assert_eq!(task.status, TaskStatus::Routed);
3201 }
3202
3203 #[serial(cooldown_state)]
3204 #[tokio::test]
3205 async fn ingest_closes_duplicate_titles() {
3206 cooldown::reset_global_state().await;
3207
3208 let backend = IngestMockBackend::with_tasks_and_dedup(
3209 vec![(Status::New, make_ext_task("2", "Duplicate issue"))],
3210 true,
3211 );
3212 let backend_dyn: Arc<dyn ExternalBackend> = Arc::<IngestMockBackend>::clone(&backend);
3213 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3214
3215 store
3216 .create(&NewTask {
3217 external_id: Some("1".to_string()),
3218 repo: "owner/repo".to_string(),
3219 origin: "github".to_string(),
3220 title: "Duplicate issue".to_string(),
3221 labels: vec!["status:new".to_string()],
3222 ..Default::default()
3223 })
3224 .await
3225 .unwrap();
3226
3227 ingest_external_tasks(&backend_dyn, "owner/repo", &store)
3228 .await
3229 .unwrap();
3230
3231 let all = store.list_all("owner/repo").await.unwrap();
3232 assert_eq!(all.len(), 1);
3233
3234 let comments = backend.comments.lock().await;
3235 assert_eq!(comments.len(), 1);
3236 assert!(comments[0].1.contains("Duplicate of #1"));
3237
3238 let status_updates = backend.status_updates.lock().await;
3239 assert_eq!(status_updates.len(), 1);
3240 assert_eq!(status_updates[0].0, "2");
3241 assert_eq!(status_updates[0].1, Status::Done);
3242 }
3243
3244 #[serial(cooldown_state)]
3245 #[tokio::test]
3246 async fn ingest_handles_empty_backend() {
3247 cooldown::reset_global_state().await;
3248 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
3249 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3250
3251 let result = ingest_external_tasks(&backend, "owner/repo", &store).await;
3253 assert!(result.is_ok());
3254
3255 let all = store.list_all("owner/repo").await.unwrap();
3256 assert!(all.is_empty());
3257 }
3258
3259 #[serial(cooldown_state)]
3260 #[tokio::test]
3261 async fn ingest_syncs_status_correctly_across_statuses() {
3262 cooldown::reset_global_state().await;
3263 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![
3264 (Status::New, make_ext_task("1", "New")),
3265 (Status::Routed, make_ext_task("2", "Routed")),
3266 (Status::InProgress, make_ext_task("3", "InProgress")),
3267 (Status::NeedsReview, make_ext_task("4", "NeedsReview")),
3268 (Status::InReview, make_ext_task("5", "InReview")),
3269 (Status::Blocked, make_ext_task("6", "Blocked")),
3270 ]);
3271 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3272
3273 ingest_external_tasks(&backend, "owner/repo", &store)
3274 .await
3275 .unwrap();
3276
3277 let counts = store.status_counts("owner/repo").await.unwrap();
3278 assert_eq!(counts.get("new"), Some(&1));
3279 assert_eq!(counts.get("routed"), Some(&1));
3280 assert_eq!(counts.get("in_progress"), Some(&1));
3281 assert_eq!(counts.get("needs_review"), Some(&1));
3282 assert_eq!(counts.get("in_review"), Some(&1));
3283 assert_eq!(counts.get("blocked"), Some(&1));
3284 }
3285
3286 #[serial(cooldown_state)]
3287 #[tokio::test]
3288 async fn ingest_does_not_overwrite_store_authoritative_status() {
3289 cooldown::reset_global_state().await;
3290 let backend: Arc<dyn ExternalBackend> =
3292 IngestMockBackend::with_tasks(vec![(Status::New, make_ext_task("99", "My task"))]);
3293 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3294
3295 let id = store
3297 .create(&NewTask {
3298 external_id: Some("99".to_string()),
3299 repo: "owner/repo".to_string(),
3300 origin: "github".to_string(),
3301 title: "My task".to_string(),
3302 ..Default::default()
3303 })
3304 .await
3305 .unwrap();
3306 store.update_status(id, TaskStatus::Routed).await.unwrap();
3307
3308 ingest_external_tasks(&backend, "owner/repo", &store)
3310 .await
3311 .unwrap();
3312
3313 let task = store.get(id).await.unwrap();
3314 assert_eq!(
3315 task.status,
3316 TaskStatus::Routed,
3317 "ingest must not overwrite store-authoritative status"
3318 );
3319 }
3320
3321 #[serial(cooldown_state)]
3322 #[tokio::test]
3323 async fn ingest_sync_to_project_called_only_for_new_tasks() {
3324 cooldown::reset_global_state().await;
3325
3326 let backend = IngestMockBackend::with_tasks(vec![
3328 (Status::New, make_ext_task("10", "Existing task")),
3329 (Status::New, make_ext_task("11", "New task")),
3330 ]);
3331 let backend_dyn: Arc<dyn ExternalBackend> = Arc::<IngestMockBackend>::clone(&backend);
3332 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3333
3334 store
3336 .create(&NewTask {
3337 external_id: Some("10".to_string()),
3338 repo: "owner/repo".to_string(),
3339 origin: "github".to_string(),
3340 title: "Existing task".to_string(),
3341 ..Default::default()
3342 })
3343 .await
3344 .unwrap();
3345
3346 ingest_external_tasks(&backend_dyn, "owner/repo", &store)
3347 .await
3348 .unwrap();
3349
3350 let syncs = backend.project_syncs.lock().await;
3351 assert_eq!(
3353 syncs.len(),
3354 1,
3355 "sync_to_project must be called exactly once (for new task only), got: {syncs:?}"
3356 );
3357 assert_eq!(syncs[0].0, "11", "sync_to_project must target the new task");
3358 }
3359
3360 #[serial(cooldown_state)]
3361 #[tokio::test]
3362 async fn ingest_handles_mixed_labeled_and_unlabeled_open_issues() {
3363 struct MixedBackend;
3367
3368 #[async_trait]
3369 impl ExternalBackend for MixedBackend {
3370 fn name(&self) -> &'static str {
3371 "mixed"
3372 }
3373 async fn create_task(
3374 &self,
3375 _: &str,
3376 _: &str,
3377 _: &[String],
3378 ) -> anyhow::Result<ExternalId> {
3379 Ok(ExternalId("x".into()))
3380 }
3381 async fn get_task(&self, _: &ExternalId) -> anyhow::Result<ExternalTask> {
3382 anyhow::bail!("not implemented")
3383 }
3384 async fn list_by_status(&self, _: Status) -> anyhow::Result<Vec<ExternalTask>> {
3385 Ok(vec![])
3386 }
3387 async fn list_active_open_issues(
3389 &self,
3390 ) -> anyhow::Result<Vec<(ExternalTask, Option<Status>)>> {
3391 Ok(vec![
3392 (make_ext_task("100", "Unlabeled routable"), None),
3393 (
3394 make_ext_task("101", "In progress issue"),
3395 Some(Status::InProgress),
3396 ),
3397 ])
3398 }
3399 async fn post_comment(&self, _: &ExternalId, _: &str) -> anyhow::Result<()> {
3400 Ok(())
3401 }
3402 async fn set_labels(&self, _: &ExternalId, _: &[String]) -> anyhow::Result<()> {
3403 Ok(())
3404 }
3405 async fn remove_label(&self, _: &ExternalId, _: &str) -> anyhow::Result<()> {
3406 Ok(())
3407 }
3408 async fn get_sub_issues(&self, _: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
3409 Ok(vec![])
3410 }
3411 async fn health_check(&self) -> anyhow::Result<()> {
3412 Ok(())
3413 }
3414 async fn get_mentions(&self, _: &str) -> anyhow::Result<Vec<Mention>> {
3415 Ok(vec![])
3416 }
3417 async fn update_status(&self, _: &ExternalId, _: Status) -> anyhow::Result<()> {
3418 Ok(())
3419 }
3420 async fn has_open_issue_with_title(&self, _: &str, _: &str) -> anyhow::Result<bool> {
3421 Ok(false)
3422 }
3423 async fn sync_to_project(
3424 &self,
3425 _: &ExternalId,
3426 _: Status,
3427 ) -> anyhow::Result<Option<String>> {
3428 Ok(None)
3429 }
3430 }
3431
3432 cooldown::reset_global_state().await;
3433
3434 let backend: Arc<dyn ExternalBackend> = Arc::new(MixedBackend);
3435 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3436
3437 ingest_external_tasks(&backend, "owner/repo", &store)
3438 .await
3439 .unwrap();
3440
3441 let all = store.list_all("owner/repo").await.unwrap();
3442 assert_eq!(all.len(), 2, "both issues must be ingested");
3443
3444 let unlabeled = store
3445 .get_by_external_id("owner/repo", "100")
3446 .await
3447 .unwrap()
3448 .unwrap();
3449 assert_eq!(
3450 unlabeled.status,
3451 TaskStatus::New,
3452 "unlabeled issue should start as New"
3453 );
3454
3455 let in_progress = store
3456 .get_by_external_id("owner/repo", "101")
3457 .await
3458 .unwrap()
3459 .unwrap();
3460 assert_eq!(
3461 in_progress.status,
3462 TaskStatus::InProgress,
3463 "labeled in_progress issue must be synced to InProgress"
3464 );
3465 }
3466
3467 #[serial(cooldown_state)]
3470 #[tokio::test]
3471 async fn kv_get_prefer_store_reads_from_store() {
3472 cooldown::reset_global_state().await;
3473 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3474
3475 store.kv_set("k1", "store_val").await.unwrap();
3476
3477 let opt = Some(&store);
3478 let val = kv_get_prefer_store(opt, "k1").await;
3479 assert_eq!(val.as_deref(), Some("store_val"));
3480 }
3481
3482 #[serial(cooldown_state)]
3483 #[tokio::test]
3484 async fn kv_get_prefer_store_returns_none_without_store() {
3485 cooldown::reset_global_state().await;
3486 let opt: Option<&Arc<TaskStore>> = None;
3487 let val = kv_get_prefer_store(opt, "k2").await;
3488 assert_eq!(val, None);
3489 }
3490
3491 #[serial(cooldown_state)]
3492 #[tokio::test]
3493 async fn kv_set_prefer_store_writes_to_store() {
3494 cooldown::reset_global_state().await;
3495 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3496
3497 let opt = Some(&store);
3498 kv_set_prefer_store(opt, "k3", "val3").await;
3499
3500 assert_eq!(store.kv_get("k3").await.unwrap().as_deref(), Some("val3"));
3501 }
3502
3503 #[serial(cooldown_state)]
3504 #[tokio::test]
3505 async fn kv_set_prefer_store_noop_without_store() {
3506 cooldown::reset_global_state().await;
3507 let opt: Option<&Arc<TaskStore>> = None;
3508 kv_set_prefer_store(opt, "k4", "val4").await;
3510 }
3511
3512 #[serial(cooldown_state)]
3515 #[tokio::test]
3516 async fn try_kv_get_prefer_store_returns_value_from_store() {
3517 cooldown::reset_global_state().await;
3518 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3519 store.kv_set("try_k1", "try_val1").await.unwrap();
3520
3521 let opt = Some(&store);
3522 let result = try_kv_get_prefer_store(opt, "try_k1").await;
3523 assert_eq!(result.unwrap().as_deref(), Some("try_val1"));
3524 }
3525
3526 #[serial(cooldown_state)]
3527 #[tokio::test]
3528 async fn try_kv_get_prefer_store_returns_none_for_missing_key() {
3529 cooldown::reset_global_state().await;
3530 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3531 let opt = Some(&store);
3532
3533 let result = try_kv_get_prefer_store(opt, "nonexistent_key").await;
3534 assert_eq!(result.unwrap(), None); }
3536
3537 #[serial(cooldown_state)]
3538 #[tokio::test]
3539 async fn try_kv_get_prefer_store_returns_none_without_store() {
3540 cooldown::reset_global_state().await;
3541 let opt: Option<&Arc<TaskStore>> = None;
3542 let result = try_kv_get_prefer_store(opt, "try_k2").await;
3543 assert_eq!(result.unwrap(), None);
3544 }
3545
3546 #[serial(cooldown_state)]
3547 #[tokio::test]
3548 async fn try_kv_set_prefer_store_writes_to_store() {
3549 cooldown::reset_global_state().await;
3550 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3551 let opt = Some(&store);
3552
3553 let result = try_kv_set_prefer_store(opt, "try_k3", "try_val3").await;
3554 assert!(result.is_ok());
3555 assert_eq!(
3556 store.kv_get("try_k3").await.unwrap().as_deref(),
3557 Some("try_val3")
3558 );
3559 }
3560
3561 #[serial(cooldown_state)]
3562 #[tokio::test]
3563 async fn try_kv_set_prefer_store_ok_without_store() {
3564 cooldown::reset_global_state().await;
3565 let opt: Option<&Arc<TaskStore>> = None;
3566 let result = try_kv_set_prefer_store(opt, "try_k4", "try_val4").await;
3568 assert!(result.is_ok());
3569 }
3570
3571 #[serial(cooldown_state)]
3574 #[test]
3575 fn dispatching_set_blocks_duplicate_processing() {
3576 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3577
3578 let repo = "owner/repo";
3579 let task_id = "42";
3580 let dispatch_key = format!("{repo}/{task_id}");
3581
3582 assert!(!dispatching.contains_key(&dispatch_key));
3584
3585 dispatching.insert(dispatch_key.clone(), task_id.to_string());
3587
3588 assert!(
3590 dispatching.contains_key(&dispatch_key),
3591 "task should be locked while dispatching"
3592 );
3593
3594 dispatching.remove(&dispatch_key);
3596
3597 assert!(
3599 !dispatching.contains_key(&dispatch_key),
3600 "task should be unlocked after review completes"
3601 );
3602 }
3603
3604 #[serial(cooldown_state)]
3605 #[test]
3606 fn dispatching_set_does_not_block_other_tasks() {
3607 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3608
3609 let repo = "owner/repo";
3610
3611 let key_42 = format!("{repo}/42");
3613 dispatching.insert(key_42.clone(), "42".to_string());
3614
3615 let key_43 = format!("{repo}/43");
3617 assert!(
3618 dispatching.contains_key(&key_42),
3619 "task 42 should be locked"
3620 );
3621 assert!(
3622 !dispatching.contains_key(&key_43),
3623 "task 43 should not be locked"
3624 );
3625 }
3626
3627 #[serial(cooldown_state)]
3640 #[tokio::test]
3641 async fn dispatch_key_held_during_review_agent_execution() {
3642 cooldown::reset_global_state().await;
3643
3644 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3645
3646 let dispatch_key = "owner/repo/42".to_string();
3647
3648 dispatching.insert(dispatch_key.clone(), "42".to_string());
3650
3651 let (agent_started_tx, agent_started_rx) = oneshot::channel::<()>();
3652 let (check_done_tx, check_done_rx) = oneshot::channel::<()>();
3653
3654 let dispatching_agent = Arc::clone(&dispatching);
3655 let key_agent = dispatch_key.clone();
3656
3657 let review_task = tokio::spawn(async move {
3659 agent_started_tx.send(()).unwrap();
3660 check_done_rx.await.unwrap();
3662 dispatching_agent.remove(&key_agent);
3664 });
3665
3666 agent_started_rx.await.unwrap();
3669 assert!(
3670 dispatching.contains_key(&dispatch_key),
3671 "dispatch_key must be visible in the dispatching set while the review \
3672 agent is running — without the a6d8b9a fix, review_open_prs would see \
3673 the key absent and re-dispatch the task, silently dropping \
3674 CHANGES_REQUESTED feedback"
3675 );
3676
3677 check_done_tx.send(()).unwrap();
3679 review_task.await.unwrap();
3680
3681 assert!(
3683 !dispatching.contains_key(&dispatch_key),
3684 "dispatch_key must be removed from the dispatching set after the \
3685 review agent completes so subsequent sync ticks can process the task"
3686 );
3687 }
3688
3689 #[serial(cooldown_state)]
3702 #[tokio::test]
3703 async fn dispatch_guard_releases_key_on_panic() {
3704 cooldown::reset_global_state().await;
3705
3706 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3707 let key = "owner/repo/42".to_string();
3708
3709 dispatching.insert(key.clone(), "42".to_string());
3713
3714 let guard = DispatchGuard::new(Arc::clone(&dispatching), key.clone());
3716
3717 let handle = tokio::spawn(async move {
3719 let _guard = guard; panic!("simulated panic inside review_and_merge");
3721 });
3722
3723 let _ = handle.await;
3725
3726 assert!(
3728 !dispatching.contains_key(&key),
3729 "dispatch key must be removed from the dispatching set even when the \
3730 spawned review task panics — without DispatchGuard the key leaks and \
3731 the task gets stuck in a permanent review loop"
3732 );
3733 }
3734
3735 #[serial(cooldown_state)]
3736 #[tokio::test]
3737 async fn sync_tick_reconciles_board_without_ingesting_or_transitioning_tasks() {
3738 cooldown::reset_global_state().await;
3739 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3740 let mock = IngestMockBackend::with_tasks(vec![]);
3741 let backend: Arc<dyn ExternalBackend> = Arc::<IngestMockBackend>::clone(&mock);
3742 let task_manager = Arc::new(TaskManager::with_store(
3743 Arc::clone(&backend),
3744 Arc::clone(&store),
3745 "owner/repo".to_string(),
3746 ));
3747 let dispatching = Arc::new(DashMap::new());
3748 Box::pin(sync_tick(
3749 &backend,
3750 &Arc::new(TmuxManager::new()),
3751 "owner/repo",
3752 &EngineConfig::default(),
3753 &Arc::new(RwLock::new(Router::new_for_test(
3754 RouterConfig::default(),
3755 vec!["claude".to_string()],
3756 ))),
3757 &task_manager,
3758 &store,
3759 &dispatching,
3760 &Arc::new(DashSet::new()),
3761 ))
3762 .await
3763 .unwrap();
3764 assert_eq!(mock.project_reconciliations.load(Ordering::Relaxed), 1);
3765 assert!(mock.project_syncs.lock().await.is_empty());
3766 assert!(mock.status_updates.lock().await.is_empty());
3767 assert!(dispatching.is_empty());
3768 assert!(
3769 store
3770 .existing_external_ids("owner/repo")
3771 .await
3772 .unwrap()
3773 .is_empty()
3774 );
3775 }
3776
3777 #[serial(cooldown_state)]
3778 #[tokio::test]
3779 async fn needs_review_refire_increments_and_fires_when_old_enough() {
3780 cooldown::reset_global_state().await;
3781 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3782 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
3783 let task_manager = Arc::new(TaskManager::with_store(
3784 backend,
3785 Arc::clone(&store),
3786 "owner/repo".to_string(),
3787 ));
3788 let router = reviewer_router();
3789 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3790
3791 let id = store
3792 .upsert_external(&crate::store::UpsertExternal {
3793 repo: "owner/repo",
3794 ext_id: "201",
3795 title: "Needs review old",
3796 body: "",
3797 author: "",
3798 url: "",
3799 labels: &[],
3800 origin: "github",
3801 })
3802 .await
3803 .unwrap();
3804 store
3805 .update_status(id, TaskStatus::NeedsReview)
3806 .await
3807 .unwrap();
3808 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
3809 .bind(id)
3810 .execute(store.pool())
3811 .await
3812 .unwrap();
3813
3814 let t = store.get(id).await.unwrap();
3815 assert_eq!(t.needs_review_refires, 0);
3816
3817 refire_and_escalate_stale_needs_review_global(
3818 &task_manager,
3819 &store,
3820 &router,
3821 &dispatching,
3822 &sweep_repos(),
3823 )
3824 .await
3825 .unwrap();
3826
3827 let t2 = store.get(id).await.unwrap();
3828 assert_eq!(t2.needs_review_refires, 1);
3829 assert_eq!(t2.status, TaskStatus::NeedsReview);
3830 }
3831
3832 #[serial(cooldown_state)]
3833 #[tokio::test]
3834 async fn needs_review_backoff_delays_fire_but_increments_counter() {
3835 cooldown::reset_global_state().await;
3836 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3837 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
3838 let task_manager = Arc::new(TaskManager::with_store(
3839 backend,
3840 Arc::clone(&store),
3841 "owner/repo".to_string(),
3842 ));
3843 let router = reviewer_router();
3844 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3845
3846 let id = store
3847 .upsert_external(&crate::store::UpsertExternal {
3848 repo: "owner/repo",
3849 ext_id: "202",
3850 title: "Needs review backoff",
3851 body: "",
3852 author: "",
3853 url: "",
3854 labels: &[],
3855 origin: "github",
3856 })
3857 .await
3858 .unwrap();
3859 store
3860 .update_status(id, TaskStatus::NeedsReview)
3861 .await
3862 .unwrap();
3863
3864 store
3865 .set_fields(id, &[("needs_review_refires", serde_json::json!(1))])
3866 .await
3867 .unwrap();
3868
3869 let now_minus_1 = (chrono::Utc::now() - chrono::Duration::minutes(1))
3871 .format("%Y-%m-%dT%H:%M:%SZ")
3872 .to_string();
3873 sqlx::query("UPDATE tasks SET updated_at = ? WHERE id = ?")
3874 .bind(now_minus_1.clone())
3875 .bind(id)
3876 .execute(store.pool())
3877 .await
3878 .unwrap();
3879
3880 let before = store.get(id).await.unwrap();
3881 assert_eq!(before.needs_review_refires, 1);
3882
3883 refire_and_escalate_stale_needs_review_global(
3884 &task_manager,
3885 &store,
3886 &router,
3887 &dispatching,
3888 &sweep_repos(),
3889 )
3890 .await
3891 .unwrap();
3892
3893 let after = store.get(id).await.unwrap();
3894 assert_eq!(after.needs_review_refires, 1);
3896 assert_eq!(after.updated_at, now_minus_1);
3897 assert_eq!(after.status, TaskStatus::NeedsReview);
3898 }
3899
3900 fn reviewer_router() -> Router {
3902 let mut config = RouterConfig::default();
3903 config.model_map.insert(
3904 "review".to_string(),
3905 HashMap::from([("claude".to_string(), vec!["sonnet".to_string()])]),
3906 );
3907 Router::new_for_test(config, vec!["claude".to_string()])
3908 }
3909
3910 #[serial(cooldown_state)]
3911 #[tokio::test]
3912 async fn needs_review_escalates_to_blocked_after_max_refires() {
3913 cooldown::reset_global_state().await;
3914 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3915 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
3916 let task_manager = Arc::new(TaskManager::with_store(
3917 backend,
3918 Arc::clone(&store),
3919 "owner/repo".to_string(),
3920 ));
3921 let router = reviewer_router();
3922 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3923
3924 let id = store
3925 .upsert_external(&crate::store::UpsertExternal {
3926 repo: "owner/repo",
3927 ext_id: "203",
3928 title: "Needs review escalate",
3929 body: "",
3930 author: "",
3931 url: "",
3932 labels: &[],
3933 origin: "github",
3934 })
3935 .await
3936 .unwrap();
3937 store
3938 .update_status(id, TaskStatus::NeedsReview)
3939 .await
3940 .unwrap();
3941
3942 store
3944 .set_fields(id, &[("needs_review_refires", serde_json::json!(5))])
3945 .await
3946 .unwrap();
3947 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
3948 .bind(id)
3949 .execute(store.pool())
3950 .await
3951 .unwrap();
3952
3953 refire_and_escalate_stale_needs_review_global(
3954 &task_manager,
3955 &store,
3956 &router,
3957 &dispatching,
3958 &sweep_repos(),
3959 )
3960 .await
3961 .unwrap();
3962
3963 let after = store.get(id).await.unwrap();
3964 assert_eq!(after.status, TaskStatus::Blocked);
3965 assert!(after.block_reason.is_some());
3966 }
3967
3968 #[serial(cooldown_state)]
3971 #[tokio::test]
3972 async fn needs_review_allows_max_refires_before_escalation() {
3973 cooldown::reset_global_state().await;
3974 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3975 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
3976 let task_manager = Arc::new(TaskManager::with_store(
3977 backend,
3978 Arc::clone(&store),
3979 "owner/repo".to_string(),
3980 ));
3981 let router = reviewer_router();
3982 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3983
3984 let id = store
3985 .upsert_external(&crate::store::UpsertExternal {
3986 repo: "owner/repo",
3987 ext_id: "204",
3988 title: "Max refires test",
3989 body: "",
3990 author: "",
3991 url: "",
3992 labels: &[],
3993 origin: "github",
3994 })
3995 .await
3996 .unwrap();
3997 store
3998 .update_status(id, TaskStatus::NeedsReview)
3999 .await
4000 .unwrap();
4001
4002 store
4004 .set_fields(id, &[("needs_review_refires", serde_json::json!(4))])
4005 .await
4006 .unwrap();
4007 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4008 .bind(id)
4009 .execute(store.pool())
4010 .await
4011 .unwrap();
4012
4013 refire_and_escalate_stale_needs_review_global(
4014 &task_manager,
4015 &store,
4016 &router,
4017 &dispatching,
4018 &sweep_repos(),
4019 )
4020 .await
4021 .unwrap();
4022
4023 let after = store.get(id).await.unwrap();
4024 assert_eq!(after.status, TaskStatus::NeedsReview);
4025 assert_eq!(after.needs_review_refires, 5);
4026 assert!(after.block_reason.is_none());
4027 }
4028
4029 fn sweep_repos() -> HashSet<String> {
4031 ["owner/repo", "inactive/repo", "some/repo", "active/repo"]
4032 .into_iter()
4033 .map(String::from)
4034 .collect()
4035 }
4036
4037 #[serial(cooldown_state)]
4039 #[tokio::test]
4040 async fn global_sweeps_leave_tasks_of_a_removed_repo_alone() {
4041 cooldown::reset_global_state().await;
4042 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4043 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4044 let task_manager = Arc::new(TaskManager::with_store(
4045 backend,
4046 Arc::clone(&store),
4047 "active/repo".to_string(),
4048 ));
4049 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4050 let router = router_with_healthy_review_agent();
4051 let active: HashSet<String> = ["active/repo".to_string()].into_iter().collect();
4052
4053 let review_id = store
4054 .create_internal("removed/repo", "Needs review", "", "cron", "1", None)
4055 .await
4056 .unwrap();
4057 store
4058 .update_status(review_id, TaskStatus::NeedsReview)
4059 .await
4060 .unwrap();
4061 let blocked_id = store
4062 .create_internal("removed/repo", "Rebroadcast blocked", "", "cron", "2", None)
4063 .await
4064 .unwrap();
4065 store
4066 .update_status(blocked_id, TaskStatus::Blocked)
4067 .await
4068 .unwrap();
4069 store
4070 .set_block_reason(blocked_id, Some(REBROADCAST_BLOCK_REASON))
4071 .await
4072 .unwrap();
4073 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z'")
4074 .execute(store.pool())
4075 .await
4076 .unwrap();
4077
4078 refire_and_escalate_stale_needs_review_global(
4079 &task_manager,
4080 &store,
4081 &router,
4082 &dispatching,
4083 &active,
4084 )
4085 .await
4086 .unwrap();
4087 auto_recover_rebroadcast_blocked_tasks_global(&router, &task_manager, &store, &active)
4088 .await
4089 .unwrap();
4090
4091 let review = store.get(review_id).await.unwrap();
4092 assert_eq!(review.status, TaskStatus::NeedsReview);
4093 assert_eq!(review.needs_review_refires, 0);
4094 let blocked = store.get(blocked_id).await.unwrap();
4095 assert_eq!(blocked.status, TaskStatus::Blocked);
4096 assert_eq!(
4097 blocked.block_reason.as_deref(),
4098 Some(REBROADCAST_BLOCK_REASON)
4099 );
4100 }
4101
4102 fn router_with_healthy_review_agent() -> Router {
4105 let mut config = RouterConfig::default();
4106 config
4107 .model_map
4108 .entry("review".to_string())
4109 .or_default()
4110 .insert("claude".to_string(), vec!["sonnet".to_string()]);
4111 let mut router = Router::new(config);
4112 router.available_agents = vec!["claude".to_string()];
4113 router
4114 }
4115
4116 #[serial(cooldown_state)]
4121 #[tokio::test]
4122 async fn global_refire_sweep_refires_stale_task_from_inactive_repo() {
4123 cooldown::reset_global_state().await;
4124 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4125 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4126 let task_manager = Arc::new(TaskManager::with_store(
4128 backend,
4129 Arc::clone(&store),
4130 "active/repo".to_string(),
4131 ));
4132 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4133 let router = router_with_healthy_review_agent();
4134
4135 let id = store
4136 .upsert_external(&crate::store::UpsertExternal {
4137 repo: "inactive/repo",
4138 ext_id: "490",
4139 title: "Stale needs_review from inactive repo",
4140 body: "",
4141 author: "",
4142 url: "",
4143 labels: &[],
4144 origin: "github",
4145 })
4146 .await
4147 .unwrap();
4148 store
4149 .update_status(id, TaskStatus::NeedsReview)
4150 .await
4151 .unwrap();
4152 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4153 .bind(id)
4154 .execute(store.pool())
4155 .await
4156 .unwrap();
4157
4158 refire_and_escalate_stale_needs_review_global(
4159 &task_manager,
4160 &store,
4161 &router,
4162 &dispatching,
4163 &sweep_repos(),
4164 )
4165 .await
4166 .unwrap();
4167
4168 let task = store.get(id).await.unwrap();
4169 assert_eq!(
4170 task.needs_review_refires, 1,
4171 "cross-repo task must be refired"
4172 );
4173 assert_eq!(task.status, TaskStatus::NeedsReview);
4174 }
4175
4176 #[serial(cooldown_state)]
4179 #[tokio::test]
4180 async fn global_refire_sweep_escalates_cross_repo_task_to_blocked() {
4181 cooldown::reset_global_state().await;
4182 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4183 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4184 let task_manager = Arc::new(TaskManager::with_store(
4185 backend,
4186 Arc::clone(&store),
4187 "active/repo".to_string(),
4188 ));
4189 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4190 let router = router_with_healthy_review_agent();
4191
4192 let id = store
4193 .upsert_external(&crate::store::UpsertExternal {
4194 repo: "inactive/repo",
4195 ext_id: "493",
4196 title: "Exhausted refires from inactive repo",
4197 body: "",
4198 author: "",
4199 url: "",
4200 labels: &[],
4201 origin: "github",
4202 })
4203 .await
4204 .unwrap();
4205 store
4206 .update_status(id, TaskStatus::NeedsReview)
4207 .await
4208 .unwrap();
4209 store
4210 .set_fields(id, &[("needs_review_refires", serde_json::json!(5))])
4211 .await
4212 .unwrap();
4213 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4214 .bind(id)
4215 .execute(store.pool())
4216 .await
4217 .unwrap();
4218
4219 refire_and_escalate_stale_needs_review_global(
4220 &task_manager,
4221 &store,
4222 &router,
4223 &dispatching,
4224 &sweep_repos(),
4225 )
4226 .await
4227 .unwrap();
4228
4229 let task = store.get(id).await.unwrap();
4230 assert_eq!(task.status, TaskStatus::Blocked);
4231 assert_eq!(
4232 task.block_reason.as_deref(),
4233 Some("review agent rebroadcast escalated after repeated retries")
4234 );
4235 }
4236
4237 #[serial(cooldown_state)]
4242 #[tokio::test]
4243 async fn global_refire_sweep_skips_when_all_review_agents_cooled() {
4244 cooldown::reset_global_state().await;
4245 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4246 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4247 let task_manager = Arc::new(TaskManager::with_store(
4248 backend,
4249 Arc::clone(&store),
4250 "active/repo".to_string(),
4251 ));
4252 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4253 let router = Router::new_for_test(RouterConfig::default(), vec!["claude".to_string()]);
4256
4257 let id = store
4258 .upsert_external(&crate::store::UpsertExternal {
4259 repo: "inactive/repo",
4260 ext_id: "494",
4261 title: "Stale needs_review while agents cooled",
4262 body: "",
4263 author: "",
4264 url: "",
4265 labels: &[],
4266 origin: "github",
4267 })
4268 .await
4269 .unwrap();
4270 store
4271 .update_status(id, TaskStatus::NeedsReview)
4272 .await
4273 .unwrap();
4274 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4275 .bind(id)
4276 .execute(store.pool())
4277 .await
4278 .unwrap();
4279
4280 refire_and_escalate_stale_needs_review_global(
4281 &task_manager,
4282 &store,
4283 &router,
4284 &dispatching,
4285 &sweep_repos(),
4286 )
4287 .await
4288 .unwrap();
4289
4290 let task = store.get(id).await.unwrap();
4291 assert_eq!(
4292 task.needs_review_refires, 0,
4293 "counter must not be touched while cooled"
4294 );
4295 assert_eq!(task.status, TaskStatus::NeedsReview);
4296 }
4297
4298 #[serial(cooldown_state)]
4301 #[tokio::test]
4302 async fn global_refire_sweep_skips_when_no_reviewer_available() {
4303 cooldown::reset_global_state().await;
4304 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4305 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4306 let task_manager = Arc::new(TaskManager::with_store(
4307 backend,
4308 Arc::clone(&store),
4309 "active/repo".to_string(),
4310 ));
4311 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4312 let router = Router::new_for_test(RouterConfig::default(), vec![]);
4313
4314 let id = store
4315 .upsert_external(&crate::store::UpsertExternal {
4316 repo: "some/repo",
4317 ext_id: "495",
4318 title: "needs_review with no reviewer",
4319 body: "",
4320 author: "",
4321 url: "",
4322 labels: &[],
4323 origin: "github",
4324 })
4325 .await
4326 .unwrap();
4327 store
4328 .update_status(id, TaskStatus::NeedsReview)
4329 .await
4330 .unwrap();
4331 store
4332 .set_fields(id, &[("needs_review_refires", serde_json::json!(5))])
4333 .await
4334 .unwrap();
4335 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4336 .bind(id)
4337 .execute(store.pool())
4338 .await
4339 .unwrap();
4340
4341 refire_and_escalate_stale_needs_review_global(
4342 &task_manager,
4343 &store,
4344 &router,
4345 &dispatching,
4346 &sweep_repos(),
4347 )
4348 .await
4349 .unwrap();
4350
4351 let task = store.get(id).await.unwrap();
4352 assert_eq!(task.status, TaskStatus::NeedsReview);
4353 assert_eq!(task.needs_review_refires, 5);
4354 }
4355
4356 #[serial(cooldown_state)]
4362 #[tokio::test]
4363 async fn auto_recover_rebroadcast_blocked_task_after_review_cycle() {
4364 cooldown::reset_global_state().await;
4365 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4366 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4367 let task_manager = Arc::new(TaskManager::with_store(
4368 backend,
4369 Arc::clone(&store),
4370 "owner/repo".to_string(),
4371 ));
4372 let router = router_with_healthy_review_agent();
4373
4374 let id = store
4375 .upsert_external(&crate::store::UpsertExternal {
4376 repo: "owner/repo",
4377 ext_id: "511",
4378 title: "Rebroadcast blocked after review",
4379 body: "",
4380 author: "",
4381 url: "",
4382 labels: &[],
4383 origin: "github",
4384 })
4385 .await
4386 .unwrap();
4387 store.update_status(id, TaskStatus::Blocked).await.unwrap();
4388 store
4389 .set_block_reason(
4390 id,
4391 Some("review agent rebroadcast escalated after repeated retries"),
4392 )
4393 .await
4394 .unwrap();
4395 store
4396 .set_fields(id, &[("review_cycles", serde_json::json!(1))])
4397 .await
4398 .unwrap();
4399 store
4400 .set_fields(id, &[("needs_review_refires", serde_json::json!(6))])
4401 .await
4402 .unwrap();
4403 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4404 .bind(id)
4405 .execute(store.pool())
4406 .await
4407 .unwrap();
4408
4409 auto_recover_rebroadcast_blocked_tasks_global(
4410 &router,
4411 &task_manager,
4412 &store,
4413 &sweep_repos(),
4414 )
4415 .await
4416 .unwrap();
4417
4418 let task = store.get(id).await.unwrap();
4419 assert_eq!(
4420 task.status,
4421 TaskStatus::NeedsReview,
4422 "blocked rebroadcast task must be recovered to needs_review"
4423 );
4424 assert!(
4425 task.block_reason.is_none(),
4426 "block_reason must be cleared on recovery"
4427 );
4428 assert_eq!(
4429 task.needs_review_refires, 0,
4430 "needs_review_refires must be reset to 0 on recovery"
4431 );
4432 }
4433
4434 #[serial(cooldown_state)]
4437 #[tokio::test]
4438 async fn stale_in_review_recovery_skips_fresh_tasks() {
4439 cooldown::reset_global_state().await;
4440 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4441 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4442 let task_manager = Arc::new(TaskManager::with_store(
4443 Arc::clone(&backend),
4444 Arc::clone(&store),
4445 "owner/repo".to_string(),
4446 ));
4447 let tmux = Arc::new(TmuxManager::new());
4448 let router = Arc::new(RwLock::new(Router::new_for_test(
4449 RouterConfig::default(),
4450 vec!["claude".to_string()],
4451 )));
4452 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4453 let auto_merge_in_flight: Arc<DashSet<String>> = Arc::new(DashSet::new());
4454
4455 let id = store
4456 .upsert_external(&crate::store::UpsertExternal {
4457 repo: "owner/repo",
4458 ext_id: "66",
4459 title: "Fresh review",
4460 body: "",
4461 author: "",
4462 url: "",
4463 labels: &[],
4464 origin: "github",
4465 })
4466 .await
4467 .unwrap();
4468 store.update_status(id, TaskStatus::InReview).await.unwrap();
4469 store
4470 .set_fields(id, &[("branch", serde_json::json!("feature/66"))])
4471 .await
4472 .unwrap();
4473 Box::pin(sync_tick(
4476 &backend,
4477 &tmux,
4478 "owner/repo",
4479 &EngineConfig::default(),
4480 &router,
4481 &task_manager,
4482 &store,
4483 &dispatching,
4484 &auto_merge_in_flight,
4485 ))
4486 .await
4487 .unwrap();
4488
4489 let task = store.get(id).await.unwrap();
4491 assert_eq!(task.status, TaskStatus::InReview);
4492 }
4493
4494 #[serial(cooldown_state)]
4495 #[tokio::test]
4496 async fn auto_unblock_routes_recoverable_failure() {
4497 cooldown::reset_global_state().await;
4498 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4499 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4500 let task_manager = Arc::new(TaskManager::with_store(
4501 backend,
4502 Arc::clone(&store),
4503 "owner/repo".to_string(),
4504 ));
4505 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4506
4507 let id = store
4508 .upsert_external(&crate::store::UpsertExternal {
4509 repo: "owner/repo",
4510 ext_id: "77",
4511 title: "Blocked task",
4512 body: "",
4513 author: "",
4514 url: "",
4515 labels: &[],
4516 origin: "github",
4517 })
4518 .await
4519 .unwrap();
4520 store.update_status(id, TaskStatus::Blocked).await.unwrap();
4521 store
4522 .set_fields(
4523 id,
4524 &[
4525 ("agent", serde_json::json!("codex")),
4526 ("model", serde_json::json!("gpt-5")),
4527 ],
4528 )
4529 .await
4530 .unwrap();
4531
4532 let run_id = store
4533 .start_run(&crate::store::StartRun {
4534 task_id: id,
4535 attempt: 1,
4536 run_type: "agent",
4537 agent: "codex",
4538 model: "gpt-5",
4539 command: "codex --model gpt-5",
4540 prompt: "do thing",
4541 })
4542 .await
4543 .unwrap();
4544 store
4545 .complete_run(&crate::store::CompleteRun {
4546 run_id,
4547 exit_code: Some(1),
4548 stdout: "",
4549 stderr: "",
4550 parsed: "",
4551 outcome: "rate_limit",
4552 error: "rate limit exceeded",
4553 tokens: RunTokenUsage::default(),
4554 })
4555 .await
4556 .unwrap();
4557
4558 auto_unblock_blocked_tasks(
4559 "owner/repo",
4560 &task_manager,
4561 &store,
4562 &dispatching,
4563 &Arc::new(DashSet::new()),
4564 )
4565 .await
4566 .unwrap();
4567
4568 let task = store.get(id).await.unwrap();
4569 assert_eq!(task.status, TaskStatus::New);
4570 assert!(task.agent.is_none());
4571 assert!(task.model.is_none());
4572 assert_eq!(task.auto_unblock_count, 1);
4573 assert_ne!(task.auto_unblock_last_at, "");
4574 assert_eq!(task.auto_unblock_last_reason, "RateLimit");
4575 }
4576
4577 #[serial(cooldown_state)]
4578 #[tokio::test]
4579 async fn auto_unblock_skips_manual_block() {
4580 cooldown::reset_global_state().await;
4581 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4582 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4583 let task_manager = Arc::new(TaskManager::with_store(
4584 backend,
4585 Arc::clone(&store),
4586 "owner/repo".to_string(),
4587 ));
4588 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4589
4590 let id = store
4591 .upsert_external(&crate::store::UpsertExternal {
4592 repo: "owner/repo",
4593 ext_id: "88",
4594 title: "Manually blocked",
4595 body: "",
4596 author: "",
4597 url: "",
4598 labels: &[],
4599 origin: "github",
4600 })
4601 .await
4602 .unwrap();
4603 store.update_status(id, TaskStatus::Blocked).await.unwrap();
4604 store
4605 .set_block_reason(id, Some("waiting on input"))
4606 .await
4607 .unwrap();
4608
4609 let run_id = store
4610 .start_run(&crate::store::StartRun {
4611 task_id: id,
4612 attempt: 1,
4613 run_type: "agent",
4614 agent: "codex",
4615 model: "gpt-5",
4616 command: "codex --model gpt-5",
4617 prompt: "do thing",
4618 })
4619 .await
4620 .unwrap();
4621 store
4622 .complete_run(&crate::store::CompleteRun {
4623 run_id,
4624 exit_code: Some(1),
4625 stdout: "",
4626 stderr: "",
4627 parsed: "",
4628 outcome: "rate_limit",
4629 error: "rate limit exceeded",
4630 tokens: RunTokenUsage::default(),
4631 })
4632 .await
4633 .unwrap();
4634
4635 auto_unblock_blocked_tasks(
4636 "owner/repo",
4637 &task_manager,
4638 &store,
4639 &dispatching,
4640 &Arc::new(DashSet::new()),
4641 )
4642 .await
4643 .unwrap();
4644
4645 let task = store.get(id).await.unwrap();
4646 assert_eq!(task.status, TaskStatus::Blocked);
4647 assert_eq!(task.auto_unblock_count, 0);
4648 }
4649
4650 #[serial(cooldown_state)]
4653 #[test]
4654 fn failure_category_as_str() {
4655 assert_eq!(FailureCategory::RateLimit.as_str(), "RateLimit");
4658 assert_eq!(FailureCategory::Timeout.as_str(), "Timeout");
4659 assert_eq!(
4660 FailureCategory::ModelUnavailable.as_str(),
4661 "ModelUnavailable"
4662 );
4663 assert_eq!(FailureCategory::MaxAttempts.as_str(), "MaxAttempts");
4664 assert_eq!(FailureCategory::Unknown.as_str(), "Unknown");
4665 assert_eq!(FailureCategory::SilentExit0.as_str(), "SilentExit0");
4666 assert_eq!(FailureCategory::ConnectionError.as_str(), "ConnectionError");
4667 assert_eq!(FailureCategory::CliFlagError.as_str(), "CliFlagError");
4668 assert_eq!(
4669 FailureCategory::AllAgentsExhausted.as_str(),
4670 "AllAgentsExhausted"
4671 );
4672 assert_eq!(FailureCategory::ParseError.as_str(), "ParseError");
4673 assert_eq!(FailureCategory::FalseFailure.as_str(), "FalseFailure");
4674 assert_eq!(FailureCategory::PushFailed.as_str(), "PushFailed");
4675 assert_eq!(FailureCategory::PrCreateFailed.as_str(), "PrCreateFailed");
4676 }
4677
4678 #[serial(cooldown_state)]
4681 #[test]
4682 fn classify_failure_model_unavailable_phrase_is_recoverable() {
4683 let category = classify_failure(
4684 "model unavailable (anthropic/claude-sonnet-4-6): Model not found: anthropic/claude-sonnet-4-6",
4685 "error",
4686 );
4687 assert_eq!(
4688 category,
4689 FailureCategory::ModelUnavailable,
4690 "\"model unavailable\" error must classify as ModelUnavailable"
4691 );
4692 assert!(
4693 category.is_recoverable(),
4694 "ModelUnavailable must be recoverable so auto-unblock can re-route"
4695 );
4696 }
4697
4698 #[serial(cooldown_state)]
4700 #[test]
4701 fn classify_failure_kimi_billing_cycle_is_credit_exhausted_not_rate_limit() {
4702 let error = "kimi rate limit: Failed to authenticate. API Error: 403 You've reached \
4709 your usage limit for this billing cycle. Your quota will be refreshed soon.";
4710 let category = classify_failure(error, "rate_limit");
4711 assert_eq!(
4712 category,
4713 FailureCategory::CreditExhausted,
4714 "billing-cycle exhaustion must classify as CreditExhausted, not RateLimit"
4715 );
4716 assert!(category.is_recoverable());
4717 }
4718
4719 #[serial(cooldown_state)]
4720 #[test]
4721 fn classify_failure_billing_cycle_exhausted_outcome_is_credit_exhausted() {
4722 let category = classify_failure("some billing message", "billing_cycle_exhausted");
4725 assert_eq!(category, FailureCategory::CreditExhausted);
4726 }
4727
4728 #[serial(cooldown_state)]
4729 #[test]
4730 fn classify_failure_generic_rate_limit_is_still_rate_limit() {
4731 let category = classify_failure("429 Too Many Requests", "rate_limit");
4734 assert_eq!(category, FailureCategory::RateLimit);
4735 }
4736
4737 #[serial(cooldown_state)]
4739 #[test]
4740 fn classify_failure_pr_create_failed_specific_patterns() {
4741 assert_eq!(
4743 classify_failure("create pr failed", ""),
4744 FailureCategory::PrCreateFailed
4745 );
4746 assert_eq!(
4747 classify_failure("failed to create pull request", ""),
4748 FailureCategory::PrCreateFailed
4749 );
4750 assert_eq!(
4751 classify_failure("pull request creation failed", ""),
4752 FailureCategory::PrCreateFailed
4753 );
4754
4755 assert_ne!(
4757 classify_failure("network failed while fetching pull request status", ""),
4758 FailureCategory::PrCreateFailed
4759 );
4760 assert_ne!(
4761 classify_failure("connection failed, retrying pull request check", ""),
4762 FailureCategory::PrCreateFailed
4763 );
4764 assert_ne!(
4765 classify_failure("pull request created but failed to merge", ""),
4766 FailureCategory::PrCreateFailed
4767 );
4768 }
4769
4770 #[serial(cooldown_state)]
4771 #[test]
4772 fn classify_failure_model_not_found_is_recoverable() {
4773 let category = classify_failure("Model not found: gpt-5-ultra", "error");
4774 assert_eq!(
4775 category,
4776 FailureCategory::ModelUnavailable,
4777 "\"model not found\" error must classify as ModelUnavailable"
4778 );
4779 assert!(category.is_recoverable());
4780 }
4781
4782 #[serial(cooldown_state)]
4783 #[test]
4784 fn classify_failure_model_does_not_exist_is_recoverable() {
4785 let category = classify_failure("The model `claude-opus-99` does not exist", "error");
4786 assert_eq!(category, FailureCategory::ModelUnavailable);
4787 assert!(category.is_recoverable());
4788 }
4789
4790 #[serial(cooldown_state)]
4791 #[tokio::test]
4792 async fn auto_unblock_routes_task_blocked_by_model_unavailable() {
4793 cooldown::reset_global_state().await;
4794 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4795 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4796 let task_manager = Arc::new(TaskManager::with_store(
4797 backend,
4798 Arc::clone(&store),
4799 "owner/repo".to_string(),
4800 ));
4801 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4802
4803 let id = store
4804 .upsert_external(&crate::store::UpsertExternal {
4805 repo: "owner/repo",
4806 ext_id: "1202",
4807 title: "Task blocked by model unavailable",
4808 body: "",
4809 author: "",
4810 url: "",
4811 labels: &[],
4812 origin: "github",
4813 })
4814 .await
4815 .unwrap();
4816 store.update_status(id, TaskStatus::Blocked).await.unwrap();
4817
4818 let run_id = store
4819 .start_run(&crate::store::StartRun {
4820 task_id: id,
4821 attempt: 1,
4822 run_type: "agent",
4823 agent: "opencode",
4824 model: "anthropic/claude-sonnet-4-6",
4825 command: "opencode",
4826 prompt: "fix the bug",
4827 })
4828 .await
4829 .unwrap();
4830 store
4831 .complete_run(&crate::store::CompleteRun {
4832 run_id,
4833 exit_code: Some(1),
4834 stdout: "",
4835 stderr: "",
4836 parsed: "",
4837 outcome: "error",
4838 error: "model unavailable (anthropic/claude-sonnet-4-6): Model not found: anthropic/claude-sonnet-4-6",
4839 tokens: RunTokenUsage::default(),
4840 })
4841 .await
4842 .unwrap();
4843
4844 auto_unblock_blocked_tasks(
4845 "owner/repo",
4846 &task_manager,
4847 &store,
4848 &dispatching,
4849 &Arc::new(DashSet::new()),
4850 )
4851 .await
4852 .unwrap();
4853
4854 let task = store.get(id).await.unwrap();
4855 assert_eq!(
4856 task.status,
4857 TaskStatus::New,
4858 "task blocked by model unavailable must be auto-unblocked and re-routed"
4859 );
4860 assert_eq!(task.auto_unblock_count, 1);
4861 assert_eq!(task.auto_unblock_last_reason, "ModelUnavailable");
4862 }
4863
4864 #[serial(cooldown_state)]
4867 #[tokio::test]
4868 async fn auto_unblock_resets_count_when_reason_changes() {
4869 cooldown::reset_global_state().await;
4870 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4874 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4875 let task_manager = Arc::new(TaskManager::with_store(
4876 backend,
4877 Arc::clone(&store),
4878 "owner/repo".to_string(),
4879 ));
4880 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4881
4882 let id = store
4883 .upsert_external(&crate::store::UpsertExternal {
4884 repo: "owner/repo",
4885 ext_id: "1227",
4886 title: "Task with different failure reason",
4887 body: "",
4888 author: "",
4889 url: "",
4890 labels: &[],
4891 origin: "github",
4892 })
4893 .await
4894 .unwrap();
4895 store.update_status(id, TaskStatus::Blocked).await.unwrap();
4896 store
4898 .set_fields(
4899 id,
4900 &[
4901 ("auto_unblock_count", serde_json::json!(1)),
4902 (
4903 "auto_unblock_last_reason",
4904 serde_json::json!("ModelUnavailable"),
4905 ),
4906 ],
4907 )
4908 .await
4909 .unwrap();
4910
4911 let run_id = store
4913 .start_run(&crate::store::StartRun {
4914 task_id: id,
4915 attempt: 2,
4916 run_type: "agent",
4917 agent: "claude",
4918 model: "opus",
4919 command: "claude",
4920 prompt: "fix the bug",
4921 })
4922 .await
4923 .unwrap();
4924 store
4925 .complete_run(&crate::store::CompleteRun {
4926 run_id,
4927 exit_code: Some(1),
4928 stdout: "",
4929 stderr: "",
4930 parsed: "",
4931 outcome: "error",
4932 error: "rate limit reached for model opus",
4933 tokens: RunTokenUsage::default(),
4934 })
4935 .await
4936 .unwrap();
4937
4938 auto_unblock_blocked_tasks(
4939 "owner/repo",
4940 &task_manager,
4941 &store,
4942 &dispatching,
4943 &Arc::new(DashSet::new()),
4944 )
4945 .await
4946 .unwrap();
4947
4948 let task = store.get(id).await.unwrap();
4949 assert_eq!(
4950 task.status,
4951 TaskStatus::New,
4952 "task must be auto-unblocked even with count=1 from a different reason"
4953 );
4954 assert_eq!(task.auto_unblock_count, 1);
4956 assert_eq!(
4957 task.auto_unblock_last_reason, "RateLimit",
4958 "reason must be updated to the new failure category"
4959 );
4960 }
4961
4962 #[serial(cooldown_state)]
4968 #[tokio::test]
4969 async fn auto_unblock_leaves_max_attempts_blocked_task_alone() {
4970 cooldown::reset_global_state().await;
4971 let store = Arc::new(TaskStore::open_memory().await.unwrap());
4972 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4973 let task_manager = Arc::new(TaskManager::with_store(
4974 backend,
4975 Arc::clone(&store),
4976 "owner/repo".to_string(),
4977 ));
4978 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4979
4980 let id = store
4981 .upsert_external(&crate::store::UpsertExternal {
4982 repo: "owner/repo",
4983 ext_id: "3783",
4984 title: "Approved work sent back to the agent",
4985 body: "",
4986 author: "",
4987 url: "",
4988 labels: &[],
4989 origin: "github",
4990 })
4991 .await
4992 .unwrap();
4993 store.update_status(id, TaskStatus::Blocked).await.unwrap();
4994
4995 let run_id = store
4996 .start_run(&crate::store::StartRun {
4997 task_id: id,
4998 attempt: 11,
4999 run_type: "agent",
5000 agent: "claude",
5001 model: "opus",
5002 command: "claude",
5003 prompt: "fix the bug",
5004 })
5005 .await
5006 .unwrap();
5007 store
5008 .complete_run(&crate::store::CompleteRun {
5009 run_id,
5010 exit_code: Some(1),
5011 stdout: "",
5012 stderr: "",
5013 parsed: "",
5014 outcome: "error",
5015 error: "exceeded max attempts (11/10). Use `/retry` to reset.",
5016 tokens: RunTokenUsage::default(),
5017 })
5018 .await
5019 .unwrap();
5020
5021 auto_unblock_blocked_tasks(
5022 "owner/repo",
5023 &task_manager,
5024 &store,
5025 &dispatching,
5026 &Arc::new(DashSet::new()),
5027 )
5028 .await
5029 .unwrap();
5030
5031 let task = store.get(id).await.unwrap();
5032 assert_eq!(
5033 task.status,
5034 TaskStatus::Blocked,
5035 "max-attempts block must survive the next tick until a person acts"
5036 );
5037 assert_eq!(
5038 task.auto_unblock_count, 0,
5039 "no auto-unblock attempt should be recorded"
5040 );
5041 }
5042
5043 #[serial(cooldown_state)]
5044 #[tokio::test]
5045 async fn auto_unblock_stores_reason_on_first_unblock() {
5046 cooldown::reset_global_state().await;
5047 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5050 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
5051 let task_manager = Arc::new(TaskManager::with_store(
5052 backend,
5053 Arc::clone(&store),
5054 "owner/repo".to_string(),
5055 ));
5056 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
5057
5058 let id = store
5059 .upsert_external(&crate::store::UpsertExternal {
5060 repo: "owner/repo",
5061 ext_id: "1227b",
5062 title: "Task for first-time reason storage",
5063 body: "",
5064 author: "",
5065 url: "",
5066 labels: &[],
5067 origin: "github",
5068 })
5069 .await
5070 .unwrap();
5071 store.update_status(id, TaskStatus::Blocked).await.unwrap();
5072
5073 let run_id = store
5074 .start_run(&crate::store::StartRun {
5075 task_id: id,
5076 attempt: 1,
5077 run_type: "agent",
5078 agent: "claude",
5079 model: "opus",
5080 command: "claude",
5081 prompt: "fix the bug",
5082 })
5083 .await
5084 .unwrap();
5085 store
5086 .complete_run(&crate::store::CompleteRun {
5087 run_id,
5088 exit_code: Some(1),
5089 stdout: "",
5090 stderr: "",
5091 parsed: "",
5092 outcome: "timeout",
5093 error: "task timed out",
5094 tokens: RunTokenUsage::default(),
5095 })
5096 .await
5097 .unwrap();
5098
5099 auto_unblock_blocked_tasks(
5100 "owner/repo",
5101 &task_manager,
5102 &store,
5103 &dispatching,
5104 &Arc::new(DashSet::new()),
5105 )
5106 .await
5107 .unwrap();
5108
5109 let task = store.get(id).await.unwrap();
5110 assert_eq!(task.status, TaskStatus::New);
5111 assert_eq!(task.auto_unblock_count, 1);
5112 assert_eq!(
5113 task.auto_unblock_last_reason, "Timeout",
5114 "reason must be stored on first auto-unblock"
5115 );
5116 }
5117
5118 #[serial(cooldown_state)]
5121 #[test]
5122 fn classify_failure_sparse_checkout_not_parse_error() {
5123 let category = classify_failure("error: cannot reset paths in a sparse checkout", "error");
5126 assert_ne!(
5127 category,
5128 FailureCategory::ParseError,
5129 "sparse checkout errors must NOT be classified as ParseError"
5130 );
5131 assert_eq!(
5132 category,
5133 FailureCategory::Unknown,
5134 "sparse checkout errors should be Unknown (not recoverable auto-unblock)"
5135 );
5136 }
5137
5138 #[serial(cooldown_state)]
5139 #[test]
5140 fn classify_failure_sparse_index_not_parse_error() {
5141 let category = classify_failure("failed due to sparse index configuration", "error");
5143 assert_ne!(category, FailureCategory::ParseError);
5144 assert_eq!(category, FailureCategory::Unknown);
5145 }
5146
5147 #[serial(cooldown_state)]
5154 #[test]
5155 fn classify_failure_truncated_is_recoverable() {
5156 let category = classify_failure(
5157 "truncated: review output was cut off before completion \
5158 (output/reasoning token budget exceeded)",
5159 "truncated",
5160 );
5161 assert_eq!(
5162 category,
5163 FailureCategory::ParseError,
5164 "truncated outcome must classify as a recoverable parse-style failure"
5165 );
5166 assert!(
5167 category.is_recoverable(),
5168 "truncated failures must be recoverable so auto-unblock can retry"
5169 );
5170 }
5171
5172 #[serial(cooldown_state)]
5173 #[test]
5174 fn dispatching_key_includes_repo_for_cross_project_isolation() {
5175 let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
5176
5177 let key_a = "owner/repo-a/42".to_string();
5179 dispatching.insert(key_a.clone(), "42".to_string());
5180
5181 let key_b = "owner/repo-b/42".to_string();
5183 assert!(dispatching.contains_key(&key_a));
5184 assert!(
5185 !dispatching.contains_key(&key_b),
5186 "same task ID in different repo should not be blocked"
5187 );
5188 }
5189
5190 #[serial(cooldown_state)]
5191 #[tokio::test]
5192 async fn emit_degraded_agents_writes_metric_when_three_or_more_agents_degraded() {
5193 cooldown::reset_global_state().await;
5194
5195 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5196
5197 let mut config = RouterConfig::default();
5199 config
5200 .model_map
5201 .entry("medium".to_string())
5202 .or_default()
5203 .insert(
5204 "test-agent-3a".to_string(),
5205 vec!["test-model-a".to_string()],
5206 );
5207 config
5208 .model_map
5209 .entry("medium".to_string())
5210 .or_default()
5211 .insert(
5212 "test-agent-3b".to_string(),
5213 vec!["test-model-b".to_string()],
5214 );
5215 config
5216 .model_map
5217 .entry("medium".to_string())
5218 .or_default()
5219 .insert(
5220 "test-agent-3c".to_string(),
5221 vec!["test-model-c".to_string()],
5222 );
5223 config
5224 .model_map
5225 .entry("medium".to_string())
5226 .or_default()
5227 .insert(
5228 "test-agent-3d".to_string(),
5229 vec!["test-model-d".to_string()],
5230 );
5231 let mut router = Router::new(config);
5232 router.available_agents = vec![
5233 "test-agent-3a".to_string(),
5234 "test-agent-3b".to_string(),
5235 "test-agent-3c".to_string(),
5236 "test-agent-3d".to_string(),
5237 ];
5238
5239 set_agent_cooldown("test-agent-3a", 3600).await;
5241 set_agent_cooldown("test-agent-3b", 3600).await;
5242 set_agent_cooldown("test-agent-3c", 3600).await;
5243
5244 emit_degraded_agents_if_needed(&router.available_agents, &router.config, Some(&store))
5246 .await;
5247
5248 let count_val = store
5250 .kv_get("metrics:orch.agents_degraded.count")
5251 .await
5252 .unwrap();
5253 assert_eq!(count_val.as_deref(), Some("3"));
5254
5255 let alert_val = store
5257 .kv_get("metrics:orch.agents_degraded.alert")
5258 .await
5259 .unwrap();
5260 assert_eq!(alert_val.as_deref(), Some("1"));
5261 }
5262
5263 #[serial(cooldown_state)]
5264 #[tokio::test]
5265 async fn emit_degraded_agents_clears_alert_when_below_threshold() {
5266 cooldown::reset_global_state().await;
5267
5268 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5269
5270 let mut config = RouterConfig::default();
5271 config
5272 .model_map
5273 .entry("medium".to_string())
5274 .or_default()
5275 .insert(
5276 "test-agent-2a".to_string(),
5277 vec!["test-model-a".to_string()],
5278 );
5279 config
5280 .model_map
5281 .entry("medium".to_string())
5282 .or_default()
5283 .insert(
5284 "test-agent-2b".to_string(),
5285 vec!["test-model-b".to_string()],
5286 );
5287 let mut router = Router::new(config);
5288 router.available_agents = vec!["test-agent-2a".to_string(), "test-agent-2b".to_string()];
5289
5290 set_agent_cooldown("test-agent-2a", 3600).await;
5292 set_agent_cooldown("test-agent-2b", 3600).await;
5293
5294 emit_degraded_agents_if_needed(&router.available_agents, &router.config, Some(&store))
5295 .await;
5296
5297 let count_val = store
5299 .kv_get("metrics:orch.agents_degraded.count")
5300 .await
5301 .unwrap();
5302 assert_eq!(count_val.as_deref(), Some("2"));
5303
5304 let alert_val = store
5306 .kv_get("metrics:orch.agents_degraded.alert")
5307 .await
5308 .unwrap();
5309 assert_eq!(alert_val.as_deref(), Some("0"));
5310
5311 let stale_pool_alert = store
5312 .kv_get("metrics:orch.model_pool_stale_persistent.alert")
5313 .await
5314 .unwrap();
5315 assert_eq!(stale_pool_alert.as_deref(), Some("0"));
5316 }
5317
5318 #[serial(cooldown_state)]
5319 #[tokio::test]
5320 async fn emit_degraded_agents_sets_stale_pool_alert_for_persistent_model_failures() {
5321 cooldown::reset_global_state().await;
5322
5323 let store = Arc::new(TaskStore::open_memory().await.unwrap());
5324
5325 let mut config = RouterConfig::default();
5326 config
5327 .model_map
5328 .entry("medium".to_string())
5329 .or_default()
5330 .insert(
5331 "test-agent-stale".to_string(),
5332 vec!["model-a".to_string(), "model-b".to_string()],
5333 );
5334 let mut router = Router::new(config);
5335 router.available_agents = vec!["test-agent-stale".to_string()];
5336
5337 record_persistent_model_failure("test-agent-stale", "model-a").await;
5339
5340 emit_degraded_agents_if_needed(&router.available_agents, &router.config, Some(&store))
5341 .await;
5342
5343 let stale_pool_alert = store
5344 .kv_get("metrics:orch.model_pool_stale_persistent.alert")
5345 .await
5346 .unwrap();
5347 assert_eq!(stale_pool_alert.as_deref(), Some("1"));
5348 }
5349
5350 #[serial(cooldown_state)]
5353 #[test]
5354 fn classify_skip_already_processed() {
5355 let action = classify_comment(
5356 "@orch /retry",
5357 Some("https://api.github.com/repos/o/r/issues/1"),
5358 "@bot",
5359 true,
5360 );
5361 assert_eq!(action, CommentAction::Skip);
5362 }
5363
5364 #[serial(cooldown_state)]
5365 #[test]
5366 fn classify_skip_irrelevant_comment() {
5367 let action = classify_comment("just a regular comment", None, "@bot", false);
5368 assert_eq!(action, CommentAction::Skip);
5369 }
5370
5371 #[serial(cooldown_state)]
5372 #[test]
5373 fn classify_command_without_mention() {
5374 let action = classify_comment(
5375 "/retry",
5376 Some("https://api.github.com/repos/o/r/issues/42"),
5377 "@bot",
5378 false,
5379 );
5380 assert_eq!(
5381 action,
5382 CommentAction::ExecuteCommand {
5383 command: OwnerCommand::Retry,
5384 issue_num: "42".to_string(),
5385 }
5386 );
5387 }
5388
5389 #[serial(cooldown_state)]
5390 #[test]
5391 fn classify_command_with_mention() {
5392 let action = classify_comment(
5393 "@orch /close",
5394 Some("https://api.github.com/repos/o/r/issues/7"),
5395 "@bot",
5396 false,
5397 );
5398 assert_eq!(
5399 action,
5400 CommentAction::ExecuteCommandForMention {
5401 command: OwnerCommand::Close,
5402 issue_num: "7".to_string(),
5403 }
5404 );
5405 }
5406
5407 #[serial(cooldown_state)]
5408 #[test]
5409 fn classify_mention_without_command() {
5410 let action = classify_comment(
5411 "@orch can you look at this?",
5412 Some("https://api.github.com/repos/o/r/issues/99"),
5413 "@bot",
5414 false,
5415 );
5416 assert_eq!(
5417 action,
5418 CommentAction::CreateMentionTask {
5419 issue_num: Some("99".to_string()),
5420 }
5421 );
5422 }
5423
5424 #[serial(cooldown_state)]
5425 #[test]
5426 fn classify_mention_without_issue_url() {
5427 let action = classify_comment("@orch help me", None, "@bot", false);
5428 assert_eq!(action, CommentAction::CreateMentionTask { issue_num: None });
5429 }
5430
5431 #[serial(cooldown_state)]
5432 #[test]
5433 fn classify_command_no_mention_no_url_skips() {
5434 let action = classify_comment("/retry", None, "@bot", false);
5435 assert_eq!(action, CommentAction::Skip);
5436 }
5437
5438 #[serial(cooldown_state)]
5439 #[test]
5440 fn classify_mention_via_current_user() {
5441 let action = classify_comment(
5442 "@bot please help",
5443 Some("https://api.github.com/repos/o/r/issues/5"),
5444 "@bot",
5445 false,
5446 );
5447 assert_eq!(
5448 action,
5449 CommentAction::CreateMentionTask {
5450 issue_num: Some("5".to_string()),
5451 }
5452 );
5453 }
5454
5455 #[serial(cooldown_state)]
5456 #[test]
5457 fn classify_command_with_mention_no_url_creates_task() {
5458 let action = classify_comment("@orch /retry", None, "@bot", false);
5460 assert_eq!(action, CommentAction::CreateMentionTask { issue_num: None });
5461 }
5462
5463 #[serial(cooldown_state)]
5464 #[test]
5465 fn mention_cursor_stops_advancing_after_gap() {
5466 let mut cursor = MentionCursor::default();
5467 cursor.advance("2026-01-01T00:00:00Z");
5468 cursor.block_on_gap();
5469 cursor.advance("2026-01-02T00:00:00Z");
5470
5471 assert_eq!(
5472 cursor.into_last_safe_ts().as_deref(),
5473 Some("2026-01-01T00:00:00Z")
5474 );
5475 }
5476
5477 #[serial(cooldown_state)]
5478 #[test]
5479 fn mention_cursor_tracks_max_timestamp_until_gap() {
5480 let mut cursor = MentionCursor::default();
5481 cursor.advance("2026-01-01T00:00:00Z");
5482 cursor.advance("2026-01-01T00:05:00Z");
5483 cursor.block_on_gap();
5484 cursor.advance("2026-01-01T00:10:00Z");
5485
5486 assert_eq!(
5487 cursor.into_last_safe_ts().as_deref(),
5488 Some("2026-01-01T00:05:00Z")
5489 );
5490 }
5491
5492 struct AckTrackingBackend {
5496 ack_calls: Mutex<Vec<String>>,
5497 posted: Mutex<Vec<String>>,
5498 ack_failure: bool,
5499 inner: NoopBackend,
5500 }
5501
5502 impl AckTrackingBackend {
5503 fn new(ack_failure: bool) -> Arc<Self> {
5504 Arc::new(Self {
5505 ack_calls: Mutex::new(Vec::new()),
5506 posted: Mutex::new(Vec::new()),
5507 ack_failure,
5508 inner: NoopBackend,
5509 })
5510 }
5511
5512 fn fail_on_ack(&self) -> bool {
5513 self.ack_failure
5514 }
5515 }
5516
5517 #[async_trait]
5518 impl ExternalBackend for AckTrackingBackend {
5519 fn name(&self) -> &'static str {
5520 "ack-tracking"
5521 }
5522
5523 async fn acknowledge_mention(&self, comment_id: &str) -> anyhow::Result<()> {
5524 self.ack_calls.lock().await.push(comment_id.to_string());
5525 if self.fail_on_ack() {
5526 anyhow::bail!("network error: temporary failure posting reaction");
5528 }
5529 Ok(())
5530 }
5531
5532 async fn get_authenticated_user(&self) -> anyhow::Result<Option<String>> {
5534 Ok(Some("bot".into()))
5535 }
5536
5537 async fn get_mentions(&self, _since: &str) -> anyhow::Result<Vec<Mention>> {
5539 Ok(vec![])
5540 }
5541
5542 async fn create_task(&self, t: &str, b: &str, l: &[String]) -> anyhow::Result<ExternalId> {
5544 self.inner.create_task(t, b, l).await
5545 }
5546 async fn get_task(&self, id: &ExternalId) -> anyhow::Result<ExternalTask> {
5547 self.inner.get_task(id).await
5548 }
5549 async fn list_by_status(&self, s: Status) -> anyhow::Result<Vec<ExternalTask>> {
5550 self.inner.list_by_status(s).await
5551 }
5552 async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
5553 self.inner.list_routable().await
5554 }
5555 async fn post_comment(&self, id: &ExternalId, b: &str) -> anyhow::Result<()> {
5556 self.posted.lock().await.push(b.to_string());
5557 self.inner.post_comment(id, b).await
5558 }
5559 async fn set_labels(&self, id: &ExternalId, l: &[String]) -> anyhow::Result<()> {
5560 self.inner.set_labels(id, l).await
5561 }
5562 async fn remove_label(&self, id: &ExternalId, l: &str) -> anyhow::Result<()> {
5563 self.inner.remove_label(id, l).await
5564 }
5565 async fn get_sub_issues(&self, id: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
5566 self.inner.get_sub_issues(id).await
5567 }
5568 async fn create_sub_task(
5569 &self,
5570 p: &ExternalId,
5571 t: &str,
5572 b: &str,
5573 l: &[String],
5574 ) -> anyhow::Result<ExternalId> {
5575 self.inner.create_sub_task(p, t, b, l).await
5576 }
5577 async fn ensure_status_label(&self, l: &str) -> anyhow::Result<()> {
5578 self.inner.ensure_status_label(l).await
5579 }
5580 async fn has_open_issue_with_title(&self, t: &str, l: &str) -> anyhow::Result<bool> {
5581 self.inner.has_open_issue_with_title(t, l).await
5582 }
5583 async fn health_check(&self) -> anyhow::Result<()> {
5584 self.inner.health_check().await
5585 }
5586 async fn is_pr_merged(&self, b: &str) -> anyhow::Result<bool> {
5587 self.inner.is_pr_merged(b).await
5588 }
5589 async fn update_status(&self, id: &ExternalId, s: Status) -> anyhow::Result<()> {
5590 self.inner.update_status(id, s).await
5591 }
5592 }
5593
5594 #[serial(cooldown_state)]
5597 #[tokio::test]
5598 async fn create_mention_task_skips_on_ack_failure() {
5599 cooldown::reset_global_state().await;
5600 let backend: Arc<AckTrackingBackend> = AckTrackingBackend::new(true);
5601 let store: Arc<TaskStore> = Arc::new(TaskStore::open_memory().await.unwrap());
5602
5603 let initial_ts = "2026-01-01T00:00:00Z";
5605 store
5606 .kv_set("mentions_last_checked", initial_ts)
5607 .await
5608 .unwrap();
5609
5610 let mention = Mention {
5611 id: "mention-abc".into(),
5612 body: "@bot can you help?".into(),
5613 author: "alice".into(),
5614 created_at: "2026-01-02T00:00:00Z".into(),
5615 issue_url: Some("https://api.github.com/repos/owner/repo/issues/42".into()),
5616 };
5617
5618 let backend_trait: Arc<dyn ExternalBackend> = Arc::<AckTrackingBackend>::clone(&backend);
5619 let task_manager = Arc::new(TaskManager::new(Arc::clone(&backend_trait)));
5620
5621 scan_comments(
5622 &backend_trait,
5623 Some(&store),
5624 "owner/repo",
5625 &task_manager,
5626 Some(&[mention]),
5627 )
5628 .await
5629 .unwrap();
5630
5631 let calls = backend.ack_calls.lock().await;
5633 assert_eq!(
5634 &**calls,
5635 &["mention-abc"],
5636 "acknowledge_mention must be called"
5637 );
5638
5639 let all = store.list_all("owner/repo").await.unwrap();
5641 assert!(
5642 all.is_empty(),
5643 "no task should be created when acknowledge_mention fails"
5644 );
5645
5646 let cursor = store.kv_get("mentions_last_checked").await.unwrap();
5648 assert_eq!(
5649 cursor.as_deref(),
5650 Some(initial_ts),
5651 "cursor must not advance when acknowledge_mention fails"
5652 );
5653 }
5654
5655 #[serial(cooldown_state)]
5658 #[tokio::test]
5659 async fn execute_command_for_mention_skips_on_ack_failure() {
5660 cooldown::reset_global_state().await;
5661 let backend: Arc<AckTrackingBackend> = AckTrackingBackend::new(true);
5662 let store: Arc<TaskStore> = Arc::new(TaskStore::open_memory().await.unwrap());
5663
5664 let initial_ts = "2026-01-01T00:00:00Z";
5665 store
5666 .kv_set("mentions_last_checked", initial_ts)
5667 .await
5668 .unwrap();
5669
5670 let mention = Mention {
5672 id: "mention-def".into(),
5673 body: "@bot /close".into(),
5674 author: "alice".into(),
5675 created_at: "2026-01-02T00:00:00Z".into(),
5676 issue_url: Some("https://api.github.com/repos/owner/repo/issues/42".into()),
5677 };
5678
5679 let backend_trait: Arc<dyn ExternalBackend> = Arc::<AckTrackingBackend>::clone(&backend);
5680 let task_manager = Arc::new(TaskManager::new(Arc::clone(&backend_trait)));
5681
5682 scan_comments(
5683 &backend_trait,
5684 Some(&store),
5685 "owner/repo",
5686 &task_manager,
5687 Some(&[mention]),
5688 )
5689 .await
5690 .unwrap();
5691
5692 let calls = backend.ack_calls.lock().await;
5694 assert_eq!(&**calls, &["mention-def"]);
5695
5696 let all = store.list_all("owner/repo").await.unwrap();
5698 assert!(
5699 all.is_empty(),
5700 "no task should be created when acknowledge_mention fails for ExecuteCommandForMention"
5701 );
5702
5703 let cursor = store.kv_get("mentions_last_checked").await.unwrap();
5705 assert_eq!(
5706 cursor.as_deref(),
5707 Some(initial_ts),
5708 "cursor must not advance when acknowledge_mention fails for ExecuteCommandForMention"
5709 );
5710 }
5711
5712 #[derive(Default)]
5716 struct FlakyGh {
5717 get_issue_calls: AtomicUsize,
5718 }
5719
5720 #[async_trait]
5721 impl CommandGh for FlakyGh {
5722 async fn get_issue(&self, _repo: &str, number: &str) -> anyhow::Result<GitHubIssue> {
5723 if self.get_issue_calls.fetch_add(1, Ordering::SeqCst) == 0 {
5724 anyhow::bail!("network error: temporary failure fetching issue");
5725 }
5726 Ok(GitHubIssue {
5727 number: number.parse().unwrap_or(0),
5728 id: 1,
5729 title: "t".into(),
5730 body: None,
5731 state: "open".into(),
5732 labels: Vec::new(),
5733 user: GitHubUser {
5734 login: "alice".into(),
5735 },
5736 created_at: String::new(),
5737 updated_at: String::new(),
5738 html_url: String::new(),
5739 node_id: None,
5740 pull_request: None,
5741 author_association: Some("OWNER".into()),
5742 state_reason: None,
5743 })
5744 }
5745
5746 async fn is_collaborator(&self, _repo: &str, _username: &str) -> anyhow::Result<bool> {
5747 Ok(true)
5748 }
5749
5750 async fn close_issue(&self, _repo: &str, _number: &str) -> anyhow::Result<()> {
5751 Ok(())
5752 }
5753 }
5754
5755 #[serial(cooldown_state)]
5758 #[tokio::test]
5759 async fn slash_command_retries_after_transient_fetch_failure() {
5760 cooldown::reset_global_state().await;
5761 let backend: Arc<AckTrackingBackend> = AckTrackingBackend::new(false);
5762 let store: Arc<TaskStore> = Arc::new(TaskStore::open_memory().await.unwrap());
5763 let initial_ts = "2026-01-01T00:00:00Z";
5764
5765 let mention = Mention {
5766 id: "mention-ghi".into(),
5767 body: "@bot /unblock".into(),
5768 author: "alice".into(),
5769 created_at: "2026-01-02T00:00:00Z".into(),
5770 issue_url: Some("https://api.github.com/repos/owner/repo/issues/42".into()),
5771 };
5772
5773 let backend_trait: Arc<dyn ExternalBackend> = Arc::<AckTrackingBackend>::clone(&backend);
5774 let task_manager = Arc::new(TaskManager::new(Arc::clone(&backend_trait)));
5775 let gh = FlakyGh::default();
5776 let mut cursor = MentionCursor::default();
5777
5778 let advanced_safely = handle_slash_command(
5780 &backend_trait,
5781 Some(&store),
5782 "owner/repo",
5783 &task_manager,
5784 &gh,
5785 &mention,
5786 &OwnerCommand::Unblock,
5787 &mut cursor,
5788 )
5789 .await
5790 .unwrap();
5791
5792 assert!(
5793 !advanced_safely,
5794 "a transient fetch failure must stop cursor advancement"
5795 );
5796 assert!(
5797 cursor.last_safe_ts.is_none(),
5798 "cursor must stay before the comment so the next tick retries it"
5799 );
5800 assert!(
5801 store.list_all("owner/repo").await.unwrap().is_empty(),
5802 "no mention task on a transient failure"
5803 );
5804 assert!(
5805 backend.posted.lock().await.is_empty(),
5806 "nothing is posted to GitHub on a transient failure"
5807 );
5808 assert_eq!(
5809 filter_mentions_by_since(slice::from_ref(&mention), initial_ts).len(),
5810 1,
5811 "the comment is still in range for the next scan"
5812 );
5813
5814 let mut cursor = MentionCursor::default();
5816 let advanced_safely = handle_slash_command(
5817 &backend_trait,
5818 Some(&store),
5819 "owner/repo",
5820 &task_manager,
5821 &gh,
5822 &mention,
5823 &OwnerCommand::Unblock,
5824 &mut cursor,
5825 )
5826 .await
5827 .unwrap();
5828
5829 assert!(advanced_safely);
5830 assert_eq!(
5831 cursor.into_last_safe_ts().as_deref(),
5832 Some("2026-01-02T00:00:00Z"),
5833 "cursor advances once the command actually runs"
5834 );
5835 assert_eq!(
5836 backend.posted.lock().await.len(),
5837 1,
5838 "the command runs once, posting a single confirmation"
5839 );
5840 assert_eq!(
5841 store.list_all("owner/repo").await.unwrap().len(),
5842 1,
5843 "one sentinel mention task for the executed command"
5844 );
5845 }
5846
5847 #[serial(cooldown_state)]
5850 #[tokio::test]
5851 async fn execute_command_keeps_cursor_on_fetch_failure() {
5852 cooldown::reset_global_state().await;
5853 let backend: Arc<AckTrackingBackend> = AckTrackingBackend::new(false);
5854 let store: Arc<TaskStore> = Arc::new(TaskStore::open_memory().await.unwrap());
5855
5856 let initial_ts = "2026-01-01T00:00:00Z";
5857 store
5858 .kv_set("mentions_last_checked", initial_ts)
5859 .await
5860 .unwrap();
5861
5862 let comment = Mention {
5864 id: "comment-jkl".into(),
5865 body: "/unblock".into(),
5866 author: "alice".into(),
5867 created_at: "2026-01-02T00:00:00Z".into(),
5868 issue_url: Some("https://api.github.com/repos/owner/repo/issues/42".into()),
5869 };
5870
5871 let backend_trait: Arc<dyn ExternalBackend> = Arc::<AckTrackingBackend>::clone(&backend);
5872 let task_manager = Arc::new(TaskManager::new(Arc::clone(&backend_trait)));
5873
5874 scan_comments(
5875 &backend_trait,
5876 Some(&store),
5877 "owner/repo",
5878 &task_manager,
5879 Some(&[comment]),
5880 )
5881 .await
5882 .unwrap();
5883
5884 assert!(
5885 backend.posted.lock().await.is_empty(),
5886 "nothing is posted when the issue fetch fails"
5887 );
5888 assert_eq!(
5889 store
5890 .kv_get("mentions_last_checked")
5891 .await
5892 .unwrap()
5893 .as_deref(),
5894 Some(initial_ts),
5895 "cursor must not advance past a command that never ran"
5896 );
5897 }
5898
5899 fn make_task(block_reason: Option<String>) -> Task {
5902 Task {
5903 id: 1,
5904 external_id: None,
5905 repo: "owner/repo".to_string(),
5906 origin: String::new(),
5907 title: String::new(),
5908 body: String::new(),
5909 status: TaskStatus::Blocked,
5910 source: String::new(),
5911 source_id: String::new(),
5912 author: String::new(),
5913 url: String::new(),
5914 labels: vec![],
5915 agent: None,
5916 model: None,
5917 complexity: String::new(),
5918 estimate: 0,
5919 route_reason: String::new(),
5920 agent_profile: String::new(),
5921 route_attempts: 0,
5922 attempts: 0,
5923 branch: String::new(),
5924 worktree: String::new(),
5925 worktree_cleaned: false,
5926 owner_node: None,
5927 summary: String::new(),
5928 last_error: String::new(),
5929 parent_id: None,
5930 block_reason,
5931 pr_number: Some(42),
5932 pr_review_context: String::new(),
5933 last_review_ts: String::new(),
5934 review_ts_map: String::new(),
5935 last_comment_review_ts: String::new(),
5936 merge_conflict_retries: 0,
5937 ci_merge_failures: 0,
5938 pr_create_failures: 0,
5939 push_failures: 0,
5940 network_retries: 0,
5941 timeout_retries: 0,
5942 review_agent_failures: 0,
5943 review_resets: 0,
5944 review_cycles: 0,
5945 review_invocations: 0,
5946 needs_review_refires: 0,
5947 input_tokens: 0,
5948 output_tokens: 0,
5949 total_cost_usd: 0.0,
5950 model_reroute_chain: String::new(),
5951 limit_reroute_chain: String::new(),
5952 memory: vec![],
5953 delegations: vec![],
5954 auto_unblock_count: 0,
5955 auto_unblock_last_at: String::new(),
5956 auto_unblock_last_reason: String::new(),
5957 ci_recovery_count: 0,
5958 no_code_reroutes: 0,
5959 no_code_last_agent: String::new(),
5960 created_at: String::new(),
5961 updated_at: String::new(),
5962 }
5963 }
5964
5965 #[serial(cooldown_state)]
5966 #[test]
5967 fn is_ci_failure_block_matches_ci_failure_limit() {
5968 let task = make_task(Some(
5969 "CI failure limit (3) reached during auto-merge".to_string(),
5970 ));
5971 assert!(is_ci_failure_block(&task));
5972 }
5973
5974 #[serial(cooldown_state)]
5975 #[test]
5976 fn is_ci_failure_block_matches_ci_checks_timeout() {
5977 let task = make_task(Some(
5978 "CI checks timed out after 3 auto-merge failures".to_string(),
5979 ));
5980 assert!(is_ci_failure_block(&task));
5981 }
5982
5983 #[serial(cooldown_state)]
5984 #[test]
5985 fn is_ci_failure_block_false_for_regular_block() {
5986 let task = make_task(Some("waiting on input".to_string()));
5987 assert!(!is_ci_failure_block(&task));
5988 }
5989
5990 #[serial(cooldown_state)]
5991 #[test]
5992 fn is_ci_failure_block_false_for_none_reason() {
5993 let task = make_task(None);
5994 assert!(!is_ci_failure_block(&task));
5995 }
5996
5997 #[serial(cooldown_state)]
6000 #[test]
6001 fn ci_failure_cooldown_zero_count_immediate() {
6002 let task = make_task(None);
6004 assert!(ci_failure_unblock_cooldown_elapsed(&task));
6005 }
6006
6007 #[serial(cooldown_state)]
6008 #[test]
6009 fn ci_failure_cooldown_empty_last_at_immediate() {
6010 let mut task = make_task(None);
6012 task.auto_unblock_count = 1;
6013 assert!(ci_failure_unblock_cooldown_elapsed(&task));
6014 }
6015
6016 #[serial(cooldown_state)]
6017 #[test]
6018 fn ci_failure_cooldown_invalid_timestamp_immediate() {
6019 let mut task = make_task(None);
6021 task.auto_unblock_count = 1;
6022 task.auto_unblock_last_at = "not-a-timestamp".to_string();
6023 assert!(ci_failure_unblock_cooldown_elapsed(&task));
6024 }
6025
6026 #[serial(cooldown_state)]
6027 #[test]
6028 fn ci_failure_cooldown_high_count_still_eventually_elapses() {
6029 let long_ago = (chrono::Utc::now() - chrono::Duration::days(30)).to_rfc3339();
6032 for count in [3i32, 5, 99] {
6033 let mut task = make_task(None);
6034 task.auto_unblock_count = count;
6035 task.auto_unblock_last_at = long_ago.clone();
6036 assert!(
6037 ci_failure_unblock_cooldown_elapsed(&task),
6038 "count={count} should eventually elapse"
6039 );
6040 }
6041 }
6042
6043 #[serial(cooldown_state)]
6044 #[test]
6045 fn ci_failure_cooldown_recent_timestamp_still_blocks() {
6046 let just_now = chrono::Utc::now().to_rfc3339();
6048 for count in [1i32, 2, 3, 99] {
6049 let mut task = make_task(None);
6050 task.auto_unblock_count = count;
6051 task.auto_unblock_last_at = just_now.clone();
6052 assert!(
6053 !ci_failure_unblock_cooldown_elapsed(&task),
6054 "count={count} should still be cooling down"
6055 );
6056 }
6057 }
6058
6059 #[serial(cooldown_state)]
6062 #[tokio::test]
6063 async fn global_ci_failure_sweep_returns_ok_with_no_blocked_tasks() {
6064 cooldown::reset_global_state().await;
6065 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6066 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6067 let task_manager = Arc::new(TaskManager::with_store(
6068 backend,
6069 Arc::clone(&store),
6070 "owner/repo".to_string(),
6071 ));
6072 auto_unblock_ci_failure_blocked_tasks_global(
6073 &task_manager,
6074 &store,
6075 &Arc::new(DashSet::new()),
6076 )
6077 .await
6078 .unwrap();
6079 }
6080
6081 #[serial(cooldown_state)]
6082 #[tokio::test]
6083 async fn global_ci_failure_sweep_skips_tasks_in_cooldown() {
6084 cooldown::reset_global_state().await;
6087 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6088 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6089 let task_manager = Arc::new(TaskManager::with_store(
6090 backend,
6091 Arc::clone(&store),
6092 "active/repo".to_string(),
6093 ));
6094
6095 let id = store
6097 .upsert_external(&crate::store::UpsertExternal {
6098 repo: "inactive/repo",
6099 ext_id: "10",
6100 title: "Old blocked task",
6101 body: "",
6102 author: "",
6103 url: "",
6104 labels: &[],
6105 origin: "github",
6106 })
6107 .await
6108 .unwrap();
6109 store.update_status(id, TaskStatus::Blocked).await.unwrap();
6110 let just_now = chrono::Utc::now().to_rfc3339();
6111 store
6112 .set_fields(
6113 id,
6114 &[
6115 (
6116 "block_reason",
6117 serde_json::json!("CI failure limit reached during auto-merge"),
6118 ),
6119 ("pr_number", serde_json::json!(99)),
6120 ("auto_unblock_count", serde_json::json!(1)),
6121 ("auto_unblock_last_at", serde_json::json!(just_now)),
6122 ],
6123 )
6124 .await
6125 .unwrap();
6126
6127 auto_unblock_ci_failure_blocked_tasks_global(
6128 &task_manager,
6129 &store,
6130 &Arc::new(DashSet::new()),
6131 )
6132 .await
6133 .unwrap();
6134
6135 let task = store.get(id).await.unwrap();
6137 assert_eq!(
6138 task.auto_unblock_count, 1,
6139 "task in cooldown must not be processed"
6140 );
6141 }
6142
6143 #[serial(cooldown_state)]
6144 #[tokio::test]
6145 async fn global_ci_failure_sweep_ignores_non_ci_failure_blocks() {
6146 cooldown::reset_global_state().await;
6147 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6148 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6149 let task_manager = Arc::new(TaskManager::with_store(
6150 backend,
6151 Arc::clone(&store),
6152 "active/repo".to_string(),
6153 ));
6154
6155 let id = store
6157 .upsert_external(&crate::store::UpsertExternal {
6158 repo: "inactive/repo",
6159 ext_id: "20",
6160 title: "Manual block",
6161 body: "",
6162 author: "",
6163 url: "",
6164 labels: &[],
6165 origin: "github",
6166 })
6167 .await
6168 .unwrap();
6169 store.update_status(id, TaskStatus::Blocked).await.unwrap();
6170 store
6171 .set_fields(
6172 id,
6173 &[("block_reason", serde_json::json!("waiting for human input"))],
6174 )
6175 .await
6176 .unwrap();
6177
6178 auto_unblock_ci_failure_blocked_tasks_global(
6179 &task_manager,
6180 &store,
6181 &Arc::new(DashSet::new()),
6182 )
6183 .await
6184 .unwrap();
6185
6186 let task = store.get(id).await.unwrap();
6188 assert_eq!(task.status, TaskStatus::Blocked);
6189 }
6190
6191 #[serial(cooldown_state)]
6194 #[tokio::test]
6195 async fn global_rebroadcast_sweep_returns_ok_with_no_blocked_tasks() {
6196 cooldown::reset_global_state().await;
6197 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6198 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6199 let task_manager = Arc::new(TaskManager::with_store(
6200 backend,
6201 Arc::clone(&store),
6202 "owner/repo".to_string(),
6203 ));
6204 let router = router_with_healthy_review_agent();
6205 auto_recover_rebroadcast_blocked_tasks_global(
6206 &router,
6207 &task_manager,
6208 &store,
6209 &sweep_repos(),
6210 )
6211 .await
6212 .unwrap();
6213 }
6214
6215 #[serial(cooldown_state)]
6216 #[tokio::test]
6217 async fn global_rebroadcast_sweep_ignores_non_rebroadcast_blocks() {
6218 cooldown::reset_global_state().await;
6219 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6220 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6221 let task_manager = Arc::new(TaskManager::with_store(
6222 backend,
6223 Arc::clone(&store),
6224 "active/repo".to_string(),
6225 ));
6226 let router = router_with_healthy_review_agent();
6227
6228 let id = store
6229 .upsert_external(&crate::store::UpsertExternal {
6230 repo: "inactive/repo",
6231 ext_id: "30",
6232 title: "Manual block",
6233 body: "",
6234 author: "",
6235 url: "",
6236 labels: &[],
6237 origin: "github",
6238 })
6239 .await
6240 .unwrap();
6241 store.update_status(id, TaskStatus::Blocked).await.unwrap();
6242 store
6243 .set_fields(
6244 id,
6245 &[("block_reason", serde_json::json!("waiting for human input"))],
6246 )
6247 .await
6248 .unwrap();
6249 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
6250 .bind(id)
6251 .execute(store.pool())
6252 .await
6253 .unwrap();
6254
6255 auto_recover_rebroadcast_blocked_tasks_global(
6256 &router,
6257 &task_manager,
6258 &store,
6259 &sweep_repos(),
6260 )
6261 .await
6262 .unwrap();
6263
6264 let task = store.get(id).await.unwrap();
6265 assert_eq!(task.status, TaskStatus::Blocked);
6266 assert_eq!(
6267 task.block_reason.as_deref(),
6268 Some("waiting for human input")
6269 );
6270 }
6271
6272 #[serial(cooldown_state)]
6273 #[tokio::test]
6274 async fn global_rebroadcast_sweep_skips_freshly_blocked_tasks() {
6275 cooldown::reset_global_state().await;
6276 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6277 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6278 let task_manager = Arc::new(TaskManager::with_store(
6279 backend,
6280 Arc::clone(&store),
6281 "active/repo".to_string(),
6282 ));
6283 let router = router_with_healthy_review_agent();
6284
6285 let id = store
6286 .upsert_external(&crate::store::UpsertExternal {
6287 repo: "inactive/repo",
6288 ext_id: "40",
6289 title: "Just escalated",
6290 body: "",
6291 author: "",
6292 url: "",
6293 labels: &[],
6294 origin: "github",
6295 })
6296 .await
6297 .unwrap();
6298 store.update_status(id, TaskStatus::Blocked).await.unwrap();
6299 store
6300 .set_fields(
6301 id,
6302 &[("block_reason", serde_json::json!(REBROADCAST_BLOCK_REASON))],
6303 )
6304 .await
6305 .unwrap();
6306 auto_recover_rebroadcast_blocked_tasks_global(
6310 &router,
6311 &task_manager,
6312 &store,
6313 &sweep_repos(),
6314 )
6315 .await
6316 .unwrap();
6317
6318 let task = store.get(id).await.unwrap();
6319 assert_eq!(task.status, TaskStatus::Blocked);
6320 }
6321
6322 #[serial(cooldown_state)]
6327 #[tokio::test]
6328 async fn global_rebroadcast_sweep_recovers_task_from_inactive_repo() {
6329 cooldown::reset_global_state().await;
6330 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6331 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6332 let task_manager = Arc::new(TaskManager::with_store(
6333 backend,
6334 Arc::clone(&store),
6335 "active/repo".to_string(),
6336 ));
6337 let router = router_with_healthy_review_agent();
6338
6339 let id = store
6340 .upsert_external(&crate::store::UpsertExternal {
6341 repo: "inactive/repo",
6342 ext_id: "458",
6343 title: "Rebroadcast blocked from inactive repo",
6344 body: "",
6345 author: "",
6346 url: "",
6347 labels: &[],
6348 origin: "github",
6349 })
6350 .await
6351 .unwrap();
6352 store.update_status(id, TaskStatus::Blocked).await.unwrap();
6353 store
6354 .set_fields(
6355 id,
6356 &[
6357 ("block_reason", serde_json::json!(REBROADCAST_BLOCK_REASON)),
6358 ("needs_review_refires", serde_json::json!(6)),
6359 ],
6360 )
6361 .await
6362 .unwrap();
6363 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
6364 .bind(id)
6365 .execute(store.pool())
6366 .await
6367 .unwrap();
6368
6369 auto_recover_rebroadcast_blocked_tasks_global(
6370 &router,
6371 &task_manager,
6372 &store,
6373 &sweep_repos(),
6374 )
6375 .await
6376 .unwrap();
6377
6378 let task = store.get(id).await.unwrap();
6379 assert_eq!(
6380 task.status,
6381 TaskStatus::NeedsReview,
6382 "cross-repo rebroadcast-blocked task must be recovered to needs_review"
6383 );
6384 assert!(
6385 task.block_reason.is_none(),
6386 "block_reason must be cleared on recovery"
6387 );
6388 assert_eq!(
6389 task.needs_review_refires, 0,
6390 "needs_review_refires must be reset to 0 on recovery"
6391 );
6392 }
6393
6394 #[serial(cooldown_state)]
6397 #[tokio::test]
6398 async fn global_rebroadcast_sweep_skips_reset_without_routable_review_agent() {
6399 cooldown::reset_global_state().await;
6400 let store = Arc::new(TaskStore::open_memory().await.unwrap());
6401 let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6402 let task_manager = Arc::new(TaskManager::with_store(
6403 backend,
6404 Arc::clone(&store),
6405 "active/repo".to_string(),
6406 ));
6407 let router = Router::new_for_test(RouterConfig::default(), vec!["claude".to_string()]);
6410
6411 let id = store
6412 .upsert_external(&crate::store::UpsertExternal {
6413 repo: "inactive/repo",
6414 ext_id: "459",
6415 title: "Rebroadcast blocked, no routable agents",
6416 body: "",
6417 author: "",
6418 url: "",
6419 labels: &[],
6420 origin: "github",
6421 })
6422 .await
6423 .unwrap();
6424 store.update_status(id, TaskStatus::Blocked).await.unwrap();
6425 store
6426 .set_fields(
6427 id,
6428 &[("block_reason", serde_json::json!(REBROADCAST_BLOCK_REASON))],
6429 )
6430 .await
6431 .unwrap();
6432 sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
6433 .bind(id)
6434 .execute(store.pool())
6435 .await
6436 .unwrap();
6437
6438 auto_recover_rebroadcast_blocked_tasks_global(
6439 &router,
6440 &task_manager,
6441 &store,
6442 &sweep_repos(),
6443 )
6444 .await
6445 .unwrap();
6446
6447 let task = store.get(id).await.unwrap();
6448 assert_eq!(task.status, TaskStatus::Blocked);
6449 assert_eq!(task.block_reason.as_deref(), Some(REBROADCAST_BLOCK_REASON));
6450 }
6451}