Skip to main content

orch/engine/
sync.rs

1//! Sync tick — periodic operations that run less frequently than the core tick.
2//!
3//! The sync tick runs every ~45 seconds and handles:
4//! - Worktree cleanup for done tasks
5//! - Merged-PR detection
6//! - @mention scanning and internal task creation
7//! - PR review processing and follow-up dispatch
8//! - Stale InReview detection
9//! - Owner /slash command scanning
10
11use super::EngineConfig;
12use super::ci_bootstrap;
13use super::cleanup::{check_merged_prs, cleanup_done_worktrees, prune_old_run_output};
14use super::commands::{CommandOutcome, CommandStoreOps, parse_command, validate_and_run_command};
15use super::review_poll::review_open_prs;
16use crate::backends::ExternalId;
17use crate::backends::ExternalTask;
18use crate::backends::{ExternalBackend, Mention, Status};
19use crate::config;
20use crate::engine;
21use crate::engine::auto_merge::{is_pr_behind, required_checks_state};
22use crate::engine::cleanup::cleanup_task_worktree;
23use crate::engine::commands::CommandGh;
24use crate::engine::commands::OwnerCommand;
25use crate::engine::cooldown;
26use crate::engine::cooldown::{
27    cooldown_reason, github_circuit_remaining_secs, is_agent_in_cooldown, is_github_circuit_open,
28    is_model_in_cooldown, refresh_degraded_agents,
29};
30use crate::engine::dependencies;
31use crate::engine::router::Router;
32use crate::engine::router::RouterConfig;
33use crate::engine::router::config as router_config;
34use crate::engine::runner::agents::patterns;
35use crate::engine::tasks;
36use crate::engine::tasks::{TaskManager, status_to_task_status};
37use crate::github::http::GhHttp;
38use crate::github::types::extract_issue_number_from_url;
39use crate::store;
40use crate::store::Task;
41use crate::store::TaskRun;
42use crate::store::TaskStatus;
43use crate::store::TaskStore;
44use crate::tmux::TmuxManager;
45use dashmap::{DashMap, DashSet};
46use futures::future;
47use std::collections::HashMap;
48use std::collections::HashSet;
49use std::sync::Arc;
50use std::time::Duration;
51use std::time::Instant;
52use tokio::sync::OnceCell;
53use tokio::sync::RwLock;
54use tokio::time;
55
56/// Read a KV value from the store.
57///
58/// Returns `None` if the store is unavailable, the key doesn't exist, OR if
59/// the store query fails. Use [`try_kv_get_prefer_store`] if you need to
60/// distinguish between "key not found" and actual errors.
61async fn kv_get_prefer_store(store: Option<&Arc<TaskStore>>, key: &str) -> Option<String> {
62    match try_kv_get_prefer_store(store, key).await {
63        Ok(opt) => opt,
64        Err(e) => {
65            tracing::error!(key, err = %e, "kv_get failed");
66            None
67        }
68    }
69}
70
71/// Try to read a KV value from the store, returning errors instead of
72/// silently discarding them.
73///
74/// Returns `Ok(None)` if the store is unavailable or the key doesn't exist.
75/// Returns `Err` if the store query fails.
76async fn try_kv_get_prefer_store(
77    store: Option<&Arc<TaskStore>>,
78    key: &str,
79) -> anyhow::Result<Option<String>> {
80    if let Some(s) = store {
81        Ok(s.kv_get(key).await?)
82    } else {
83        Ok(None)
84    }
85}
86
87/// Write a KV value to the store.
88///
89/// Logs warnings on failure but provides no way for callers to know if the
90/// write succeeded. Use [`try_kv_set_prefer_store`] if you need to handle
91/// errors or guarantee persistence (e.g., for circuit breaker state).
92async fn kv_set_prefer_store(store: Option<&Arc<TaskStore>>, key: &str, value: &str) {
93    if let Err(e) = try_kv_set_prefer_store(store, key, value).await {
94        tracing::error!(key, err = %e, "kv_set failed");
95    }
96}
97
98/// Try to write a KV value to the store, returning errors instead of
99/// silently logging them.
100///
101/// Returns `Ok(())` if the store is unavailable (best-effort) or if the
102/// write succeeds. Returns `Err` if the store query fails.
103async fn try_kv_set_prefer_store(
104    store: Option<&Arc<TaskStore>>,
105    key: &str,
106    value: &str,
107) -> anyhow::Result<()> {
108    if let Some(s) = store {
109        s.kv_set(key, value).await?;
110    }
111    Ok(())
112}
113
114#[derive(Debug, Clone)]
115pub(crate) struct ReviewTaskSnapshot {
116    pub external: ExternalTask,
117    pub stored: Task,
118}
119
120async fn prefetch_review_tasks(
121    store: &Arc<TaskStore>,
122    repo: &str,
123) -> anyhow::Result<(
124    Vec<ReviewTaskSnapshot>,
125    Vec<ReviewTaskSnapshot>,
126    Vec<ReviewTaskSnapshot>,
127)> {
128    // Approved tasks wait on the same merge path as a review in progress
129    let mut in_review = store.list_by_status(repo, TaskStatus::InReview).await?;
130    in_review.extend(store.list_by_status(repo, TaskStatus::Approved).await?);
131    let in_review = in_review
132        .into_iter()
133        .map(|stored| ReviewTaskSnapshot {
134            external: tasks::store_task_to_external(&stored),
135            stored,
136        })
137        .collect();
138    let needs_review = store
139        .list_by_status(repo, TaskStatus::NeedsReview)
140        .await?
141        .into_iter()
142        .map(|stored| ReviewTaskSnapshot {
143            external: tasks::store_task_to_external(&stored),
144            stored,
145        })
146        .collect();
147
148    // Also prefetch blocked tasks that have a branch or PR number recorded.
149    // These can be merged externally (or manually) but previously were invisible
150    // to the merged-PR checker, leaving tasks stuck in Blocked after merges.
151    let blocked_candidates = store
152        .list_by_status(repo, TaskStatus::Blocked)
153        .await?
154        .into_iter()
155        .filter(|stored| stored.pr_number.is_some() || !stored.branch.is_empty())
156        .map(|stored| ReviewTaskSnapshot {
157            external: tasks::store_task_to_external(&stored),
158            stored,
159        })
160        .collect();
161
162    Ok((in_review, needs_review, blocked_candidates))
163}
164
165async fn fetch_comments_since(
166    backend: &Arc<dyn ExternalBackend>,
167    since: &str,
168) -> anyhow::Result<Vec<Mention>> {
169    backend.get_mentions(since).await
170}
171
172fn filter_mentions_by_since(comments: &[Mention], since: &str) -> Vec<Mention> {
173    comments
174        .iter()
175        .filter(|comment| comment.created_at.as_str() > since)
176        .cloned()
177        .collect()
178}
179
180#[derive(Debug, Clone, Copy, PartialEq, Eq)]
181enum FailureCategory {
182    FalseFailure,
183    RateLimit,
184    CreditExhausted,
185    SilentExit0,
186    Timeout,
187    ConnectionError,
188    CliFlagError,
189    AllAgentsExhausted,
190    ParseError,
191    MaxAttempts,
192    PushFailed,
193    PrCreateFailed,
194    ModelUnavailable,
195    Unknown,
196}
197
198impl FailureCategory {
199    /// Whether auto-unblock may put the task back in the queue on its own.
200    ///
201    /// `MaxAttempts` is deliberately absent: the task already ran the configured
202    /// number of times, so an automatic retry just restarts the loop that produced
203    /// the block. It stays blocked until `orch task unblock` or `orch task retry`
204    /// (#3783)
205    fn is_recoverable(self) -> bool {
206        matches!(
207            self,
208            FailureCategory::FalseFailure
209                | FailureCategory::RateLimit
210                | FailureCategory::CreditExhausted
211                | FailureCategory::SilentExit0
212                | FailureCategory::Timeout
213                | FailureCategory::ConnectionError
214                | FailureCategory::CliFlagError
215                | FailureCategory::AllAgentsExhausted
216                | FailureCategory::ParseError
217                | FailureCategory::ModelUnavailable
218        )
219    }
220
221    fn as_str(self) -> &'static str {
222        match self {
223            FailureCategory::FalseFailure => "FalseFailure",
224            FailureCategory::RateLimit => "RateLimit",
225            FailureCategory::CreditExhausted => "CreditExhausted",
226            FailureCategory::SilentExit0 => "SilentExit0",
227            FailureCategory::Timeout => "Timeout",
228            FailureCategory::ConnectionError => "ConnectionError",
229            FailureCategory::CliFlagError => "CliFlagError",
230            FailureCategory::AllAgentsExhausted => "AllAgentsExhausted",
231            FailureCategory::ParseError => "ParseError",
232            FailureCategory::MaxAttempts => "MaxAttempts",
233            FailureCategory::PushFailed => "PushFailed",
234            FailureCategory::PrCreateFailed => "PrCreateFailed",
235            FailureCategory::ModelUnavailable => "ModelUnavailable",
236            FailureCategory::Unknown => "Unknown",
237        }
238    }
239}
240
241fn classify_failure(error: &str, outcome: &str) -> FailureCategory {
242    let lower = error.to_lowercase();
243
244    if outcome == "timeout" || lower.contains("timeout") {
245        return FailureCategory::Timeout;
246    }
247
248    // Credit exhaustion is a transient condition (credits replenish, billing cycles
249    // reset). Classifying it as recoverable allows auto-unblock to retry once the
250    // credit-related cooldown expires, instead of permanently blocking the task.
251    //
252    // Checked before the generic rate-limit branch below: billing-cycle exhaustion
253    // messages (e.g. Kimi's "usage limit for this billing cycle") also contain
254    // "usage limit"/"rate limit" substrings, so checking outcome/content-based
255    // rate-limit patterns first would misclassify them as plain RateLimit (issue
256    // #3529). `outcome` also carries the specific string from `classify_run_outcome`
257    // when the task_run already recorded it.
258    if outcome == "billing_cycle_exhausted"
259        || outcome == "credit_exhausted"
260        || outcome == "org_level_disabled"
261        || cooldown::detect_credit_exhaustion(&lower).is_some()
262    {
263        return FailureCategory::CreditExhausted;
264    }
265
266    if outcome == "rate_limit" || lower.contains("rate limit") || lower.contains("usage limit") {
267        return FailureCategory::RateLimit;
268    }
269
270    if error.trim().is_empty() {
271        return FailureCategory::FalseFailure;
272    }
273
274    if lower.contains("all agents exhausted") {
275        return FailureCategory::AllAgentsExhausted;
276    }
277
278    if lower.contains("max attempts") || lower.contains("exceeded max attempts") {
279        return FailureCategory::MaxAttempts;
280    }
281
282    if lower.contains("output-format=stream-json") && lower.contains("requires --verbose") {
283        return FailureCategory::CliFlagError;
284    }
285
286    if lower.contains("model unavailable")
287        || lower.contains("model not found")
288        || (lower.contains("does not exist") && lower.contains("model"))
289    {
290        return FailureCategory::ModelUnavailable;
291    }
292
293    if lower.contains("push failed") {
294        return FailureCategory::PushFailed;
295    }
296
297    if lower.contains("create pr failed")
298        || lower.contains("failed to create pull request")
299        || lower.contains("pull request creation failed")
300    {
301        return FailureCategory::PrCreateFailed;
302    }
303
304    if lower.contains("exit 0") {
305        return FailureCategory::SilentExit0;
306    }
307
308    if outcome == "parse_error"
309        || outcome == "truncated"
310        || lower.contains("invalid response")
311        || lower.contains("parse error")
312        || lower.contains("truncated")
313    {
314        return FailureCategory::ParseError;
315    }
316
317    if patterns::detect_network_error(&lower).is_some() {
318        return FailureCategory::ConnectionError;
319    }
320
321    FailureCategory::Unknown
322}
323
324fn classify_failure_from_run(run: &TaskRun) -> Option<FailureCategory> {
325    // Completed successfully — no failure to classify.
326    if run.outcome == "success" && run.error.trim().is_empty() {
327        return None;
328    }
329
330    // Skip runs with no meaningful data. Some runs are created then never
331    // populated (aborted/stale work). Treat these as non-failures so they
332    // don't block auto-unblock logic which considers recent failures.
333    // Historically an all-NULL/empty run would be promoted to Unknown which
334    // prevented auto-unblock. Ensure we early-return None for truly empty
335    // runs (no outcome, no error, no exit_code, no output).
336    if run.outcome.trim().is_empty()
337        && run.error.trim().is_empty()
338        && run.exit_code.is_none()
339        && run.stdout.trim().is_empty()
340        && run.stderr.trim().is_empty()
341    {
342        return None;
343    }
344
345    let has_output = !run.stdout.trim().is_empty() || !run.stderr.trim().is_empty();
346    if run.exit_code == Some(0) && !has_output && run.outcome != "success" {
347        return Some(FailureCategory::SilentExit0);
348    }
349
350    let mut category = classify_failure(&run.error, &run.outcome);
351
352    // Promote a bare "false failure" to Unknown only when we have a
353    // concrete non-zero exit code or other signs the run truly failed.
354    // Do NOT promote when exit_code is None (incomplete run) — those are
355    // handled above by skipping. This prevents NULL/aborted runs from
356    // turning recoverable FalseFailure into non-recoverable Unknown.
357    // Only promote when we have explicit evidence: a non-zero exit code
358    // or non-empty stdout/stderr. If exit_code is None we keep FalseFailure
359    // so the run remains recoverable.
360    if category == FailureCategory::FalseFailure {
361        let has_nonzero_exit = run.exit_code.is_some() && run.exit_code != Some(0);
362        let has_output = !run.stdout.trim().is_empty() || !run.stderr.trim().is_empty();
363        if has_nonzero_exit || has_output {
364            category = FailureCategory::Unknown;
365        }
366    }
367
368    Some(category)
369}
370
371/// Resolve the parent task ID for a mention.
372///
373/// - If the mention is on a PR, look up the task that owns that PR via `pr_number`.
374/// - If the mention is on an issue, look up the task by external_id (issue number).
375///
376/// Returns:
377/// - `Ok(Some(id))` — parent found
378/// - `Ok(None)` — no matching parent (genuine miss, e.g. unknown issue/PR)
379/// - `Err(_)` — lookup failed (DB error); caller should defer/retry
380async fn resolve_mention_parent_id(
381    store: &TaskStore,
382    repo: &str,
383    issue_num: &str,
384    is_pr: bool,
385) -> anyhow::Result<Option<i64>> {
386    if is_pr {
387        if let Ok(pr_num) = issue_num.parse::<i32>() {
388            store.resolve_id_by_pr_number(repo, pr_num).await
389        } else {
390            Ok(None)
391        }
392    } else {
393        store.resolve_id_by_external(repo, issue_num).await
394    }
395}
396
397/// Create a sentinel mention task in the store and advance the cursor.
398async fn record_mention_task(
399    store: &TaskStore,
400    repo: &str,
401    mention: &Mention,
402    title: &str,
403    body: &str,
404    parent_id: Option<i64>,
405    cursor: &mut MentionCursor,
406) -> bool {
407    match store
408        .create_internal(repo, title, body, "mention", &mention.id, parent_id)
409        .await
410    {
411        Ok(task_id) => {
412            tracing::info!(task_id, mention_id = %mention.id, "created mention task");
413            cursor.advance(&mention.created_at);
414            true
415        }
416        Err(e) => {
417            tracing::warn!(mention_id = %mention.id, err = %e, "failed to create mention task");
418            false
419        }
420    }
421}
422
423/// Advance the mention cursor if the timestamp is newer.
424fn advance_cursor(last_safe_ts: &mut Option<String>, ts: &str) {
425    if last_safe_ts.as_deref() < Some(ts) {
426        *last_safe_ts = Some(ts.to_string());
427    }
428}
429
430#[derive(Debug, Default)]
431struct MentionCursor {
432    last_safe_ts: Option<String>,
433    blocked_by_gap: bool,
434}
435
436impl MentionCursor {
437    fn advance(&mut self, ts: &str) {
438        if self.blocked_by_gap {
439            return;
440        }
441        advance_cursor(&mut self.last_safe_ts, ts);
442    }
443
444    fn block_on_gap(&mut self) {
445        self.blocked_by_gap = true;
446    }
447
448    fn into_last_safe_ts(self) -> Option<String> {
449        self.last_safe_ts
450    }
451}
452
453/// What to do with a comment in the unified scan loop.
454#[derive(Debug, PartialEq)]
455pub(crate) enum CommentAction {
456    /// Already processed or not relevant — skip.
457    Skip,
458    /// Slash command without @mention — execute the command.
459    ExecuteCommand {
460        command: OwnerCommand,
461        issue_num: String,
462    },
463    /// @mention with a slash command — execute and record sentinel task.
464    ExecuteCommandForMention {
465        command: OwnerCommand,
466        issue_num: String,
467    },
468    /// @mention without command — create internal task for agent to respond.
469    CreateMentionTask { issue_num: Option<String> },
470}
471
472/// Classify a comment into the appropriate action.
473///
474/// Pure function — no I/O, no async. Determines what the scan loop should do
475/// with each comment based on its content and whether it was already processed.
476pub(crate) fn classify_comment(
477    body: &str,
478    issue_url: Option<&str>,
479    current_user: &str,
480    already_processed: bool,
481) -> CommentAction {
482    if already_processed {
483        return CommentAction::Skip;
484    }
485
486    let is_mention =
487        body.contains(current_user) || body.contains("@orch") || body.contains("@orchestrator");
488    let command = parse_command(body);
489    let issue_num = issue_url.and_then(extract_issue_number_from_url);
490
491    match (command, is_mention, issue_num) {
492        (Some(cmd), true, Some(num)) => CommentAction::ExecuteCommandForMention {
493            command: cmd,
494            issue_num: num,
495        },
496        (Some(cmd), false, Some(num)) => CommentAction::ExecuteCommand {
497            command: cmd,
498            issue_num: num,
499        },
500        // Command but no issue URL — if it's a mention, create task; otherwise skip
501        (Some(_), true, None) => CommentAction::CreateMentionTask { issue_num: None },
502        // Mention without command — create task for agent
503        (None, true, issue_num) => CommentAction::CreateMentionTask { issue_num },
504        // Neither mention nor command — skip
505        (Some(_), false, None) | (None, false, _) => CommentAction::Skip,
506    }
507}
508
509/// Handle a slash command from a mention. Records a sentinel task and advances
510/// the cursor on success; logs a warning and returns early if the mention has
511/// no valid issue URL.
512///
513/// Execute a slash command mentioned by `@orch` on a comment.
514///
515/// Returns:
516/// - `Ok(true)` — command processed and cursor may continue advancing
517/// - `Ok(false)` — mention task insert failed, or validation hit a transient
518///   GitHub error; caller must stop cursor advancement so the next tick retries
519/// - `Err(_)` — DB lookup failed for parent_id; caller must NOT advance cursor
520///   so this mention is retried on the next sync
521#[expect(
522    clippy::too_many_arguments,
523    reason = "the sync tick hands over its shared state piece by piece"
524)]
525async fn handle_slash_command(
526    backend: &Arc<dyn ExternalBackend>,
527    store: Option<&Arc<TaskStore>>,
528    repo: &str,
529    task_manager: &Arc<TaskManager>,
530    gh: &dyn CommandGh,
531    mention: &Mention,
532    command: &OwnerCommand,
533    cursor: &mut MentionCursor,
534) -> anyhow::Result<bool> {
535    let Some(issue_num) = mention
536        .issue_url
537        .as_deref()
538        .and_then(extract_issue_number_from_url)
539    else {
540        tracing::warn!(comment_id = %mention.id, "slash command without valid issue number");
541        return Ok(true);
542    };
543
544    let store_opt: Option<Arc<dyn CommandStoreOps>> =
545        store.map(|s| Arc::clone(s) as Arc<dyn CommandStoreOps>);
546    let outcome = validate_and_run_command(
547        backend,
548        gh,
549        repo,
550        &issue_num,
551        command,
552        &mention.author,
553        &store_opt,
554        task_manager,
555    )
556    .await;
557
558    // Extract is_pr from the outcome (avoids a second get_issue call)
559    let is_pr = match &outcome {
560        CommandOutcome::Executed { is_pr }
561        | CommandOutcome::NotOpen { is_pr }
562        | CommandOutcome::NotCollaborator { is_pr } => *is_pr,
563        CommandOutcome::FetchFailed | CommandOutcome::CollaboratorCheckFailed => {
564            // Nothing was executed and nothing was posted to GitHub. Leave the cursor
565            // before this comment so the next tick retries the command.
566            tracing::warn!(
567                comment_id = %mention.id,
568                command = %command,
569                "deferring @orch slash command after a transient GitHub error"
570            );
571            return Ok(false);
572        }
573    };
574
575    // Record a sentinel task for the mention (with correct parent_id).
576    // On DB lookup failure, defer so the mention can be retried on next sync.
577    if let Some(s) = store {
578        let parent_id = match resolve_mention_parent_id(s, repo, &issue_num, is_pr).await {
579            Ok(parent_id) => parent_id,
580            Err(e) => {
581                tracing::warn!(
582                    comment_id = %mention.id,
583                    err = %e,
584                    "deferring mention task due to parent lookup failure"
585                );
586                return Err(e);
587            }
588        };
589
590        let (title, body) = match &outcome {
591            CommandOutcome::NotOpen { .. } => (
592                format!(
593                    "Skipped @orch command on closed #{issue_num} from @{}",
594                    mention.author
595                ),
596                format!(
597                    "Mention by @{}:\n\n{}\n\n**Skipped:** target is closed",
598                    mention.author, mention.body
599                ),
600            ),
601            CommandOutcome::NotCollaborator { .. } => (
602                format!(
603                    "Skipped @orch command on #{issue_num} from non-collaborator @{}",
604                    mention.author
605                ),
606                format!(
607                    "Mention by @{}:\n\n{}\n\n**Skipped:** author is not a collaborator",
608                    mention.author, mention.body
609                ),
610            ),
611            CommandOutcome::Executed { .. } => (
612                format!(
613                    "Respond to mention by @{} on task #{issue_num} — command executed",
614                    mention.author
615                ),
616                format!(
617                    "Mention by @{} on #{issue_num}:\n\n{}\n\n**Status:** Command executed",
618                    mention.author, mention.body
619                ),
620            ),
621            CommandOutcome::FetchFailed | CommandOutcome::CollaboratorCheckFailed => {
622                tracing::debug!("skipping mention task record due to earlier command failure");
623                return Ok(true);
624            }
625        };
626
627        let created = record_mention_task(s, repo, mention, &title, &body, parent_id, cursor).await;
628        if !created {
629            return Ok(false);
630        }
631    }
632    Ok(true)
633}
634
635fn auto_unblock_cooldown_elapsed(count: i32, last_at: &str) -> bool {
636    if count == 0 {
637        return true;
638    }
639
640    let required = match count {
641        1 => chrono::Duration::minutes(30),
642        2 => chrono::Duration::hours(2),
643        _ => return false,
644    };
645
646    if last_at.trim().is_empty() {
647        return true;
648    }
649
650    match chrono::DateTime::parse_from_rfc3339(last_at) {
651        Ok(ts) => chrono::Utc::now() - ts.with_timezone(&chrono::Utc) >= required,
652        Err(_) => true,
653    }
654}
655
656fn is_ci_failure_block(task: &Task) -> bool {
657    let reason = task.block_reason.as_deref().unwrap_or("");
658    reason.contains("CI failure limit") || reason.contains("CI checks timed out")
659}
660
661fn ci_failure_unblock_cooldown_elapsed(task: &Task) -> bool {
662    if task.auto_unblock_count == 0 {
663        return true;
664    }
665
666    // Short cooldowns — the PR may close quickly after the first check.
667    // We never permanently block (no `_ => return false`): eventually the PR
668    // will close or a human will intervene.
669    let required = match task.auto_unblock_count {
670        1 => chrono::Duration::minutes(10),
671        2 => chrono::Duration::hours(1),
672        3 => chrono::Duration::hours(6),
673        _ => chrono::Duration::hours(24),
674    };
675
676    let last_at = &task.auto_unblock_last_at;
677    if last_at.trim().is_empty() {
678        return true;
679    }
680
681    match chrono::DateTime::parse_from_rfc3339(last_at) {
682        Ok(ts) => chrono::Utc::now() - ts.with_timezone(&chrono::Utc) >= required,
683        Err(_) => true,
684    }
685}
686
687async fn try_unblock_ci_failure_task(
688    gh: &GhHttp,
689    repo: &str,
690    task_manager: &Arc<TaskManager>,
691    store: &Arc<TaskStore>,
692    task: &Task,
693    auto_merge_in_flight: &Arc<DashSet<String>>,
694) -> anyhow::Result<bool> {
695    let Some(pr_number) = task.pr_number.and_then(|n| u64::try_from(n).ok()) else {
696        tracing::debug!(
697            task_id = task.id,
698            "CI failure blocked task has no PR number"
699        );
700        return Ok(false);
701    };
702
703    let pr = match gh.get_pr(repo, pr_number).await {
704        Ok(p) => p,
705        Err(e) => {
706            tracing::warn!(task_id = task.id, pr_number, err = %e, "failed to get PR status for CI failure block");
707            return Ok(false);
708        }
709    };
710
711    tracing::debug!(
712        task_id = task.id,
713        pr_number,
714        merged = pr.merged,
715        state = ?pr.state,
716        "verifying PR state for CI failure blocked task"
717    );
718
719    if pr.merged.unwrap_or(false) || pr.state.eq_ignore_ascii_case("closed") {
720        tracing::info!(
721            task_id = task.id,
722            pr_number,
723            merged = pr.merged,
724            state = %pr.state,
725            "PR merged/closed — unblocking CI failure blocked task"
726        );
727        let ext_id = task
728            .external_id
729            .clone()
730            .unwrap_or_else(|| format!("internal:{}", task.id));
731        // Update the store directly by the already-resolved numeric id instead of going
732        // through `task_manager.update_task_status`, which resolves by `self.repo` — this
733        // sweep runs cross-repo, so `task.repo` may not match the caller's own repo scope.
734        task_manager
735            .mark_task_done_by_store_id(task.id, &task.repo, &ExternalId(ext_id))
736            .await?;
737        let fields = vec![
738            ("block_reason", serde_json::Value::Null),
739            ("auto_unblock_count", serde_json::json!(0)),
740            ("auto_unblock_last_at", serde_json::json!("")),
741            ("auto_unblock_last_reason", serde_json::json!("")),
742        ];
743        store.set_fields(task.id, &fields).await?;
744        return Ok(true);
745    }
746
747    // Skip if a recovery merge is already in flight for this task. The poll loop
748    // can take up to 10 minutes, and the same task may be seen by both the per-repo
749    // sync_tick and the global sweep in the same tick window.
750    let task_id_str = task
751        .external_id
752        .clone()
753        .unwrap_or_else(|| format!("internal:{}", task.id));
754    if auto_merge_in_flight.contains(&task_id_str) {
755        tracing::debug!(
756            task_id = task.id,
757            pr_number,
758            "CI-failure recovery merge already in flight — skipping"
759        );
760        return Ok(false);
761    }
762
763    // Give the PR an active chance to recover: merge the base branch into the PR
764    // branch to re-trigger CI, then re-enable GitHub auto-merge. GitHub's queue
765    // can silently stall even when checks are green and the branch is up to date,
766    // so after arming auto-merge we poll for completion and fall back to a direct
767    // merge once the PR is mergeable and CI is passing.
768    let branch_update_result = gh.update_pr_branch(repo, pr_number).await;
769    match branch_update_result {
770        Ok(()) => {
771            tracing::info!(
772                task_id = task.id,
773                pr_number,
774                "updated CI-failure-blocked PR branch against base to retrigger CI"
775            );
776        }
777        Err(ref e) => {
778            // warn, not debug, so a silently-broken corrective retry doesn't look like a working one (#3561)
779            tracing::warn!(
780                task_id = task.id,
781                pr_number,
782                err = %e,
783                outcome = "branch_update_failed",
784                "failed to update CI-failure-blocked PR branch (may already be up to date)"
785            );
786        }
787    }
788
789    if let Err(e) = gh.enable_auto_merge(repo, pr_number).await {
790        tracing::warn!(task_id = task.id, pr_number, err = %e, "failed to re-enable auto-merge after branch update");
791    }
792
793    // Record this attempt so the cooldown advances even though the actual
794    // poll/merge now runs in the background.
795    let pr_state = pr.state.as_str();
796    let new_reason = if let Err(ref e) = branch_update_result {
797        format!(
798            "CI failure limit reached during auto-merge (PR #{pr_number} still open, state: {pr_state}, branch update failed: {e})"
799        )
800    } else {
801        format!(
802            "CI failure limit reached during auto-merge (PR #{pr_number} still open, state: {pr_state})"
803        )
804    };
805    let now = chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string();
806    // Increment auto_unblock_count so the cooldown backoff (24h, 3d, never) kicks in
807    // and we don't hammer the GitHub API every tick for a PR that stays open.
808    let new_count = task.auto_unblock_count.saturating_add(1);
809    let fields = vec![
810        ("block_reason", serde_json::json!(&new_reason)),
811        ("auto_unblock_count", serde_json::json!(new_count)),
812        ("auto_unblock_last_at", serde_json::json!(&now)),
813    ];
814    // Use set_fields_silent so the probe write does not bump updated_at on stale
815    // blocked tasks — triage queries keyed off updated_at must only see real
816    // status/result changes, not periodic re-checks of an open PR.
817    store.set_fields_silent(task.id, &fields).await?;
818
819    // Spawn the poll and direct-merge fallback in the background so the
820    // recovery sweep does not block the main tick loop (or sync_tick) while
821    // CI runs. The in-flight guard prevents double-spawns across ticks.
822    auto_merge_in_flight.insert(task_id_str.clone());
823    let gh_clone = gh.clone();
824    let repo_string = repo.to_string();
825    let task_manager_clone = Arc::clone(task_manager);
826    let store_clone = Arc::clone(store);
827    let task_clone = task.clone();
828    let in_flight_clone = Arc::clone(auto_merge_in_flight);
829
830    tokio::spawn(async move {
831        let _guard = scopeguard::guard((), |()| {
832            in_flight_clone.remove(&task_id_str);
833        });
834
835        match poll_and_merge_recovered_pr(&gh_clone, &repo_string, pr_number, task_clone.id).await {
836            Ok(true) => {
837                tracing::info!(
838                    task_id = task_clone.id,
839                    pr_number,
840                    "CI-failure-blocked PR merged during recovery sweep"
841                );
842                let ext_id = task_clone
843                    .external_id
844                    .clone()
845                    .unwrap_or_else(|| format!("internal:{}", task_clone.id));
846                let fields = [
847                    ("block_reason", serde_json::Value::Null),
848                    ("auto_unblock_count", serde_json::json!(0)),
849                    ("auto_unblock_last_at", serde_json::json!("")),
850                    ("auto_unblock_last_reason", serde_json::json!("")),
851                ];
852                if let Err(e) = task_manager_clone
853                    .update_task_status_and_result_by_store_id(
854                        task_clone.id,
855                        &task_clone.repo,
856                        &ExternalId(ext_id),
857                        Status::Done,
858                        &fields,
859                    )
860                    .await
861                {
862                    tracing::warn!(
863                        task_id = task_clone.id,
864                        pr_number,
865                        err = %e,
866                        "failed to mark CI-failure-blocked task done after recovery merge"
867                    );
868                }
869                if let Err(e) =
870                    cleanup_task_worktree(&format!("{}", task_clone.id), &repo_string, &store_clone)
871                        .await
872                {
873                    tracing::warn!(
874                        task_id = task_clone.id,
875                        pr_number,
876                        err = %e,
877                        "cleanup after CI-failure recovery merge failed"
878                    );
879                }
880            }
881            Ok(false) => {
882                tracing::debug!(
883                    task_id = task_clone.id,
884                    pr_number,
885                    "CI-failure-blocked PR not merged this recovery tick — poll completed without merge"
886                );
887            }
888            Err(e) => {
889                tracing::warn!(
890                    task_id = task_clone.id,
891                    pr_number,
892                    err = %e,
893                    "error while polling CI-failure-blocked PR for recovery"
894                );
895            }
896        }
897    });
898
899    Ok(false)
900}
901
902/// Poll a recovered CI-failure PR until it merges or its checks settle, then
903/// attempt a direct merge if GitHub's auto-merge queue has not completed it.
904///
905/// GitHub's `enablePullRequestAutoMerge` hands the PR off to GitHub's queue and
906/// returns immediately; the queue can stall silently. This helper mirrors the
907/// primary `auto_merge_pr` polling pattern but is scoped to the recovery sweep:
908/// it waits for required checks to pass, verifies the PR is not behind the base
909/// branch, and calls `gh.merge_pr()` directly when the PR is ready but still
910/// open. Returns `Ok(true)` when the PR is merged/closed, `Ok(false)` when the
911/// PR needs to wait for a future tick.
912async fn poll_and_merge_recovered_pr(
913    gh: &GhHttp,
914    repo: &str,
915    pr_number: u64,
916    task_id: i64,
917) -> anyhow::Result<bool> {
918    // The branch can fall behind while CI runs. Retry the update-branch call a
919    // few times inside the poll loop instead of exiting and waiting for the next
920    // 24-hour recovery cooldown.
921    const MAX_BEHIND_RETRIES: u32 = 3;
922
923    let max_wait_secs: u64 = config::get("workflow.ci_poll_max_wait_secs")
924        .ok()
925        .and_then(|s| s.parse().ok())
926        .unwrap_or(600);
927    let base_interval_secs: u64 = config::get("workflow.ci_poll_interval_secs")
928        .ok()
929        .and_then(|s| s.parse().ok())
930        .unwrap_or(15);
931    let max_wait = Duration::from_secs(max_wait_secs);
932    let start = Instant::now();
933    let mut poll_count: u32 = 0;
934    let mut behind_retries: u32 = 0;
935
936    let initial_pr = gh.get_pr(repo, pr_number).await?;
937    let base_branch = initial_pr.base.ref_.clone();
938    let required_contexts = gh
939        .get_required_status_check_contexts(repo, &base_branch)
940        .await?;
941    let repo_has_workflows = if required_contexts.is_empty() {
942        gh.has_workflows(repo).await?
943    } else {
944        true
945    };
946
947    loop {
948        let pr = gh.get_pr(repo, pr_number).await?;
949        if pr.merged.unwrap_or(false) || pr.state.eq_ignore_ascii_case("closed") {
950            tracing::info!(
951                task_id = task_id,
952                pr_number,
953                "PR merged/closed during recovery poll"
954            );
955            return Ok(true);
956        }
957
958        if pr.mergeable == Some(false) {
959            tracing::info!(
960                task_id = task_id,
961                pr_number,
962                outcome = "poll_exited_without_merge",
963                reason = "merge_conflicts",
964                "PR has merge conflicts after recovery update — leaving for next recovery cycle"
965            );
966            return Ok(false);
967        }
968
969        if is_pr_behind(&pr) {
970            if behind_retries >= MAX_BEHIND_RETRIES {
971                tracing::info!(
972                    task_id = task_id,
973                    pr_number,
974                    behind_retries,
975                    max_behind_retries = MAX_BEHIND_RETRIES,
976                    outcome = "poll_exited_without_merge",
977                    reason = "behind_retry_exhausted",
978                    "PR still behind base after recovery retries — leaving for next recovery cycle"
979                );
980                return Ok(false);
981            }
982            behind_retries += 1;
983            match gh.update_pr_branch(repo, pr_number).await {
984                Ok(()) => {
985                    tracing::info!(
986                        task_id = task_id,
987                        pr_number,
988                        behind_retries,
989                        "PR was behind base — updated branch and continuing recovery poll"
990                    );
991                    continue;
992                }
993                Err(e) => {
994                    tracing::warn!(
995                        task_id = task_id,
996                        pr_number,
997                        behind_retries,
998                        err = %e,
999                        outcome = "branch_update_failed",
1000                        "failed to update behind PR branch during recovery poll"
1001                    );
1002                    return Ok(false);
1003                }
1004            }
1005        }
1006
1007        let head_sha = pr.head.sha.clone();
1008        let (state, total, passing, failing, pending) = if required_contexts.is_empty() {
1009            gh.get_combined_status(
1010                repo,
1011                &head_sha,
1012                repo_has_workflows,
1013                pr.mergeable_state.as_deref(),
1014            )
1015            .await?
1016        } else {
1017            let check_runs = gh.get_check_runs(repo, &head_sha).await?;
1018            let statuses = gh.get_commit_status_contexts(repo, &head_sha).await?;
1019            required_checks_state(&required_contexts, &check_runs, &statuses)
1020        };
1021
1022        tracing::info!(
1023            task_id = task_id,
1024            pr_number,
1025            state = %state,
1026            total,
1027            passing,
1028            failing,
1029            pending,
1030            "CI recovery status check"
1031        );
1032
1033        match state.as_str() {
1034            "success" => {
1035                tracing::info!(
1036                    task_id = task_id,
1037                    pr_number,
1038                    "CI green after recovery — attempting direct merge"
1039                );
1040                match gh.merge_pr(repo, pr_number, true, &head_sha).await {
1041                    Ok(()) => {
1042                        tracing::info!(
1043                            task_id = task_id,
1044                            pr_number,
1045                            "direct merge succeeded after recovery"
1046                        );
1047                        return Ok(true);
1048                    }
1049                    Err(e) => {
1050                        let err = e.to_string().to_ascii_lowercase();
1051                        tracing::warn!(
1052                            task_id = task_id,
1053                            pr_number,
1054                            err = %e,
1055                            "direct merge failed after recovery"
1056                        );
1057                        // If the PR became merged between the get_pr call and the
1058                        // merge attempt, treat it as success rather than a failed tick.
1059                        if err.contains("already merged")
1060                            || err.contains("no merge commit")
1061                            || err.contains("pull request already merged")
1062                        {
1063                            return Ok(true);
1064                        }
1065                        // The head moved after we read it, and updating the branch would skip CI on the new commit
1066                        if err.contains("head branch was modified")
1067                            || err.contains("sha does not match")
1068                        {
1069                            tracing::info!(
1070                                task_id = task_id,
1071                                pr_number,
1072                                outcome = "poll_exited_without_merge",
1073                                reason = "head_moved",
1074                                "merge rejected because the head sha changed"
1075                            );
1076                            return Ok(false);
1077                        }
1078                        // GitHub may reject the merge because the branch fell behind
1079                        // again while CI was finishing. Retry the branch update inside
1080                        // the poll window instead of waiting for the next sweep.
1081                        if (err.contains("head branch is out of date")
1082                            || err.contains("behind")
1083                            || err.contains("409"))
1084                            && behind_retries < MAX_BEHIND_RETRIES
1085                        {
1086                            behind_retries += 1;
1087                            match gh.update_pr_branch(repo, pr_number).await {
1088                                Ok(()) => {
1089                                    tracing::info!(
1090                                        task_id = task_id,
1091                                        pr_number,
1092                                        behind_retries,
1093                                        "merge rejected due to out-of-date branch — updated and continuing recovery poll"
1094                                    );
1095                                    continue;
1096                                }
1097                                Err(update_err) => {
1098                                    tracing::warn!(
1099                                        task_id = task_id,
1100                                        pr_number,
1101                                        behind_retries,
1102                                        err = %update_err,
1103                                        outcome = "branch_update_failed",
1104                                        "failed to update out-of-date PR branch during recovery poll"
1105                                    );
1106                                }
1107                            }
1108                        }
1109                        tracing::info!(
1110                            task_id = task_id,
1111                            pr_number,
1112                            outcome = "poll_exited_without_merge",
1113                            reason = "direct_merge_failed",
1114                            "direct merge did not succeed — leaving for next recovery cycle"
1115                        );
1116                        return Ok(false);
1117                    }
1118                }
1119            }
1120            "failure" => {
1121                tracing::info!(
1122                    task_id = task_id,
1123                    pr_number,
1124                    outcome = "poll_exited_without_merge",
1125                    reason = "ci_failure",
1126                    "CI still failing after recovery — will retry later"
1127                );
1128                return Ok(false);
1129            }
1130            _ => {}
1131        }
1132
1133        if start.elapsed() >= max_wait {
1134            tracing::info!(
1135                task_id = task_id,
1136                pr_number,
1137                outcome = "poll_exited_without_merge",
1138                reason = "poll_timeout",
1139                "CI still pending after recovery poll timeout — will retry later"
1140            );
1141            return Ok(false);
1142        }
1143
1144        poll_count += 1;
1145        let multiplier = (2_u64).pow(poll_count.saturating_sub(1)).min(4);
1146        let sleep_secs = base_interval_secs.saturating_mul(multiplier);
1147        time::sleep(Duration::from_secs(sleep_secs)).await;
1148    }
1149}
1150
1151/// Exact `block_reason` written when a task's NeedsReview refire counter exhausts
1152/// during a rebroadcast escalation (`refire_and_escalate_stale_needs_review_global`).
1153/// Shared with `auto_recover_rebroadcast_blocked_tasks_global` so escalation and
1154/// recovery never drift apart on the literal string
1155const REBROADCAST_BLOCK_REASON: &str = "review agent rebroadcast escalated after repeated retries";
1156
1157/// Global sweep: auto-recover rebroadcast-blocked tasks across all repos.
1158///
1159/// When a NeedsReview refire counter exhausts, the task is blocked with
1160/// `REBROADCAST_BLOCK_REASON`. Once any review agent is routable again, this
1161/// sweep resets matching tasks to `NeedsReview` so the subscriber can pick them up.
1162/// Writes through the store via `*_by_store_id` helpers, not a repo-scoped
1163/// `TaskManager`. Called once per main tick from `tick_global_sweeps`.
1164///
1165/// Before resetting, the attached PR's live state is checked (mirroring
1166/// `try_unblock_ci_failure_task`): merged means the task is marked done, closed
1167/// unmerged means it stays blocked with an annotated reason, and only an open PR
1168/// (or no PR) falls through to `NeedsReview`. A 5-minute block age prevents
1169/// same-tick escalate/recover flapping. Reset-to-NeedsReview still requires at
1170/// least one routable review agent
1171pub(crate) async fn auto_recover_rebroadcast_blocked_tasks_global(
1172    router: &Router,
1173    task_manager: &Arc<TaskManager>,
1174    store: &Arc<TaskStore>,
1175    active_repos: &HashSet<String>,
1176) -> anyhow::Result<()> {
1177    const MIN_BLOCK_AGE_MINUTES: i64 = 5;
1178
1179    let all_blocked = match store.list_all_by_status_global(TaskStatus::Blocked).await {
1180        Ok(tasks) => tasks,
1181        Err(e) => {
1182            tracing::warn!(err = %e, "global rebroadcast-blocked sweep: failed to list blocked tasks");
1183            return Ok(());
1184        }
1185    };
1186
1187    let now = chrono::Utc::now();
1188    let candidates: Vec<_> = all_blocked
1189        .into_iter()
1190        .filter(|t| {
1191            if t.block_reason.as_deref() != Some(REBROADCAST_BLOCK_REASON) {
1192                return false;
1193            }
1194            // A repo outside the active set has no review subscriber, so a reset would loop
1195            if !active_repos.contains(&t.repo) {
1196                return false;
1197            }
1198            // Only recover tasks blocked long enough to rule out same-tick escalations.
1199            match chrono::DateTime::parse_from_rfc3339(&t.updated_at) {
1200                Ok(updated_at) => {
1201                    let age = now - updated_at.with_timezone(&chrono::Utc);
1202                    age.num_minutes() >= MIN_BLOCK_AGE_MINUTES
1203                }
1204                Err(_) => true,
1205            }
1206        })
1207        .collect();
1208
1209    if candidates.is_empty() {
1210        return Ok(());
1211    }
1212
1213    let gh = match GhHttp::new() {
1214        Ok(g) => Some(g),
1215        Err(e) => {
1216            tracing::warn!(err = %e, "global rebroadcast-blocked sweep: failed to create GhHttp — skipping PR-state checks this tick");
1217            None
1218        }
1219    };
1220
1221    // Skip reset-to-NeedsReview when no review agent is routable, re-firing
1222    // while cooled would immediately re-exhaust the counter
1223    let any_routable =
1224        !router.available_agents.is_empty() && router.healthy_agent_count("review") > 0;
1225
1226    for task in candidates {
1227        let ext_id = task
1228            .external_id
1229            .clone()
1230            .unwrap_or_else(|| format!("internal:{}", task.id));
1231
1232        if let (Some(gh), Some(pr_number)) =
1233            (&gh, task.pr_number.and_then(|n| u64::try_from(n).ok()))
1234        {
1235            match gh.get_pr(&task.repo, pr_number).await {
1236                Ok(pr) => {
1237                    if pr.merged.unwrap_or(false) {
1238                        tracing::info!(
1239                            task_id = task.id,
1240                            repo = %task.repo,
1241                            pr_number,
1242                            "global rebroadcast-blocked sweep: PR already merged — marking task done"
1243                        );
1244                        if let Err(e) = task_manager
1245                            .mark_task_done_by_store_id(
1246                                task.id,
1247                                &task.repo,
1248                                &ExternalId(ext_id.clone()),
1249                            )
1250                            .await
1251                        {
1252                            tracing::warn!(task_id = task.id, repo = %task.repo, err = %e, "global rebroadcast-blocked sweep: failed to mark merged task done");
1253                            continue;
1254                        }
1255                        let fields: &[(&str, serde_json::Value)] = &[
1256                            ("block_reason", serde_json::Value::Null),
1257                            ("needs_review_refires", serde_json::json!(0)),
1258                        ];
1259                        if let Err(e) = store.set_fields(task.id, fields).await {
1260                            tracing::warn!(task_id = task.id, err = %e, "global rebroadcast-blocked sweep: failed to clear fields after marking done");
1261                        }
1262                        continue;
1263                    }
1264                    if pr.state.eq_ignore_ascii_case("closed") {
1265                        tracing::info!(
1266                            task_id = task.id,
1267                            repo = %task.repo,
1268                            pr_number,
1269                            "global rebroadcast-blocked sweep: PR closed unmerged — leaving task blocked, not re-dispatching"
1270                        );
1271                        let new_reason = format!(
1272                            "{REBROADCAST_BLOCK_REASON} (PR #{pr_number} closed unmerged — no further auto-recovery)"
1273                        );
1274                        let fields: &[(&str, serde_json::Value)] =
1275                            &[("block_reason", serde_json::json!(new_reason))];
1276                        // set_fields_silent: this is a terminal annotation, not a real
1277                        // status change — don't bump updated_at so triage queries keyed
1278                        // off it aren't confused by a task that stays blocked forever.
1279                        if let Err(e) = store.set_fields_silent(task.id, fields).await {
1280                            tracing::warn!(task_id = task.id, err = %e, "global rebroadcast-blocked sweep: failed to annotate closed-unmerged block reason");
1281                        }
1282                        continue;
1283                    }
1284                    // PR still open — fall through to the reset-to-NeedsReview path.
1285                }
1286                Err(e) => {
1287                    tracing::warn!(task_id = task.id, repo = %task.repo, pr_number, err = %e, "global rebroadcast-blocked sweep: failed to get PR status — skipping this tick");
1288                    continue;
1289                }
1290            }
1291        }
1292
1293        if !any_routable {
1294            continue;
1295        }
1296
1297        if let Err(e) = task_manager
1298            .update_task_status_by_store_id(
1299                task.id,
1300                &task.repo,
1301                &ExternalId(ext_id.clone()),
1302                Status::NeedsReview,
1303            )
1304            .await
1305        {
1306            tracing::warn!(task_id = task.id, repo = %task.repo, err = %e, "global rebroadcast-blocked sweep: failed to reset task to needs_review — skipping");
1307            continue;
1308        }
1309
1310        let fields: &[(&str, serde_json::Value)] = &[
1311            ("block_reason", serde_json::Value::Null),
1312            ("needs_review_refires", serde_json::json!(0)),
1313        ];
1314        if let Err(e) = store.set_fields(task.id, fields).await {
1315            tracing::warn!(task_id = task.id, err = %e, "global rebroadcast-blocked sweep: failed to clear block_reason/needs_review_refires after status reset");
1316        }
1317
1318        tracing::info!(
1319            task_id = task.id,
1320            repo = %task.repo,
1321            ext_id = %ext_id,
1322            "global rebroadcast-blocked sweep: auto-recovered rebroadcast-blocked task — review agents now available"
1323        );
1324    }
1325
1326    Ok(())
1327}
1328
1329async fn auto_unblock_blocked_tasks(
1330    repo: &str,
1331    task_manager: &Arc<TaskManager>,
1332    store: &Arc<TaskStore>,
1333    dispatching: &Arc<DashMap<String, String>>,
1334    auto_merge_in_flight: &Arc<DashSet<String>>,
1335) -> anyhow::Result<()> {
1336    let blocked = match store.list_by_status(repo, TaskStatus::Blocked).await {
1337        Ok(tasks) => tasks,
1338        Err(e) => {
1339            tracing::warn!(err = %e, "failed to list blocked tasks — skipping auto-unblock this tick");
1340            return Ok(());
1341        }
1342    };
1343
1344    if blocked.is_empty() {
1345        return Ok(());
1346    }
1347
1348    // Batch-load all runs for blocked tasks in a single query instead of N individual queries.
1349    let task_ids: Vec<i64> = blocked.iter().map(|t| t.id).collect();
1350    let mut runs_by_task = match store.get_runs_batch(&task_ids).await {
1351        Ok(map) => map,
1352        Err(e) => {
1353            tracing::warn!(err = %e, "get_runs_batch failed — skipping auto-unblock this tick");
1354            return Ok(());
1355        }
1356    };
1357
1358    // Separate CI-failure blocked tasks from other blocked tasks.
1359    let (ci_failure_tasks, other_blocked): (Vec<_>, Vec<_>) =
1360        blocked.into_iter().partition(is_ci_failure_block);
1361
1362    // Handle CI-failure blocked tasks by verifying PR state.
1363    if !ci_failure_tasks.is_empty() {
1364        let gh = match GhHttp::new() {
1365            Ok(g) => Some(g),
1366            Err(e) => {
1367                tracing::warn!(err = %e, "failed to create GhHttp — skipping CI failure unblock this tick");
1368                None
1369            }
1370        };
1371        for task in ci_failure_tasks {
1372            if ci_failure_unblock_cooldown_elapsed(&task)
1373                && let Some(ref gh) = gh
1374                && let Err(e) = try_unblock_ci_failure_task(
1375                    gh,
1376                    repo,
1377                    task_manager,
1378                    store,
1379                    &task,
1380                    auto_merge_in_flight,
1381                )
1382                .await
1383            {
1384                tracing::warn!(task_id = task.id, err = %e, "CI failure unblock check failed");
1385            }
1386        }
1387    }
1388
1389    // Process regular blocked tasks (those without a block_reason).
1390    for task in other_blocked {
1391        if task.block_reason.is_some() {
1392            continue;
1393        }
1394
1395        let task_id_str = task
1396            .external_id
1397            .clone()
1398            .unwrap_or_else(|| format!("internal:{}", task.id));
1399        let dispatch_key = format!("{repo}/{task_id_str}");
1400        if dispatching.contains_key(&dispatch_key) {
1401            continue;
1402        }
1403
1404        let runs = runs_by_task.remove(&task.id).unwrap_or_default();
1405        let relevant_runs: Vec<_> = runs
1406            .into_iter()
1407            .filter(|r| r.run_type == "agent" || r.run_type == "review")
1408            .collect();
1409        let mut failures = Vec::new();
1410        let mut has_review_failure = false;
1411        for run in relevant_runs.iter().rev() {
1412            if let Some(category) = classify_failure_from_run(run) {
1413                if run.run_type == "review" {
1414                    has_review_failure = true;
1415                }
1416                failures.push(category);
1417            }
1418            if failures.len() >= 3 {
1419                break;
1420            }
1421        }
1422
1423        if failures.is_empty() || !failures.iter().all(|f| f.is_recoverable()) {
1424            continue;
1425        }
1426
1427        // Determine the most recent failure category as the "reason key" for this unblock.
1428        let reason_key = failures.first().map_or("", |f| f.as_str());
1429
1430        // Detect reason changes to reset exponential backoff.
1431        // New reasons bypass old cooldown windows and start fresh attempt counters.
1432        let current_reason = task.auto_unblock_last_reason.clone();
1433        let reason_changed = reason_key != current_reason;
1434
1435        // Compute effective counter: when the reason changes, start from 0;
1436        // otherwise use the accumulated count for exponential backoff.
1437        let cooldown_count = if reason_changed {
1438            0
1439        } else {
1440            task.auto_unblock_count
1441        };
1442
1443        // Gate on attempt count using the effective count — if the reason changed, the
1444        // counter was reset above so cooldown_count is 0 and a new reason always gets a
1445        // fresh attempt regardless of how many times the old reason fired.
1446        if cooldown_count >= 3 {
1447            continue;
1448        }
1449
1450        if !auto_unblock_cooldown_elapsed(cooldown_count, &task.auto_unblock_last_at) {
1451            continue;
1452        }
1453
1454        // Compute the new counter value: when reason changed, cooldown_count is 0,
1455        // so incrementing gives 1 (first attempt for the new reason). When the reason is the
1456        // same, advance from current count for exponential backoff.
1457        let new_count = cooldown_count + 1;
1458        let now = chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string();
1459
1460        let ext_id = task
1461            .external_id
1462            .clone()
1463            .unwrap_or_else(|| format!("internal:{}", task.id));
1464        let new_status = if has_review_failure {
1465            Status::NeedsReview
1466        } else {
1467            Status::New
1468        };
1469        let new_status_str = if has_review_failure {
1470            "needs_review"
1471        } else {
1472            "new"
1473        };
1474
1475        // First update the status; only increment the counter if this succeeds.
1476        // This prevents transient GitHub API errors from exhausting the retry quota.
1477        if let Err(e) = task_manager
1478            .update_task_status(&ExternalId(ext_id), new_status)
1479            .await
1480        {
1481            tracing::warn!(
1482                task_id = task.id,
1483                err = %e,
1484                "auto-unblock failed to update status — counter not incremented"
1485            );
1486            continue;
1487        }
1488
1489        // Status update succeeded; now record the counter increment and clear related fields.
1490        let mut fields: Vec<(&str, serde_json::Value)> = vec![
1491            ("auto_unblock_count", serde_json::json!(new_count)),
1492            ("auto_unblock_last_at", serde_json::json!(now)),
1493            ("auto_unblock_last_reason", serde_json::json!(reason_key)),
1494            ("agent", serde_json::Value::Null),
1495            ("model", serde_json::Value::Null),
1496        ];
1497        fields.push(("no_code_reroutes", serde_json::json!(0)));
1498        fields.push(("no_code_last_agent", serde_json::json!("")));
1499        if has_review_failure {
1500            fields.push(("review_agent_failures", serde_json::json!(0)));
1501            fields.push(("review_invocations", serde_json::json!(0)));
1502            fields.push(("review_resets", serde_json::json!(0)));
1503        }
1504        if let Err(e) = store.set_fields(task.id, &fields).await {
1505            tracing::warn!(task_id = task.id, err = %e, "failed to set auto_unblock fields after successful status update");
1506            // Task is unblocked (correct behavior) but counter hasn't advanced.
1507            // Slightly over-eager retries are better than permanent task stalls.
1508        }
1509
1510        if let Err(e) = store
1511            .append_activity(
1512                task.id,
1513                "auto_unblock",
1514                Some("blocked"),
1515                Some(new_status_str),
1516                None,
1517                None,
1518                Some(&serde_json::json!({
1519                    "reason": reason_key,
1520                    "count": if reason_changed { 1 } else { task.auto_unblock_count + 1 },
1521                    "failures": failures.iter().map(|f| format!("{f:?}")).collect::<Vec<_>>(),
1522                })),
1523            )
1524            .await
1525        {
1526            tracing::warn!(task_id = task.id, err = %e, "failed to log auto_unblock activity");
1527        }
1528
1529        tracing::info!(
1530            task_id = task.id,
1531            failures = ?failures,
1532            review_failure = has_review_failure,
1533            "auto-unblocked task with recoverable failures"
1534        );
1535    }
1536
1537    Ok(())
1538}
1539
1540/// Global sweep for CI-failure blocked tasks from inactive or removed projects.
1541///
1542/// `auto_unblock_blocked_tasks` is called from `sync_tick` and only runs for the
1543/// currently-active repo. Tasks that were blocked for a project that is later removed
1544/// from orch config never get processed and accumulate in the DB indefinitely.
1545///
1546/// This function queries all blocked tasks across all repos and handles the CI-failure
1547/// cases (block_reason LIKE '%CI failure%' or '%CI checks timed out%') so they are
1548/// eventually cleaned up regardless of project activity. Called once per main tick from
1549/// `tick_global_sweeps`.
1550pub(crate) async fn auto_unblock_ci_failure_blocked_tasks_global(
1551    task_manager: &Arc<TaskManager>,
1552    store: &Arc<TaskStore>,
1553    auto_merge_in_flight: &Arc<DashSet<String>>,
1554) -> anyhow::Result<()> {
1555    let all_blocked = match store.list_all_by_status_global(TaskStatus::Blocked).await {
1556        Ok(tasks) => tasks,
1557        Err(e) => {
1558            tracing::warn!(err = %e, "global CI-failure blocked sweep: failed to list blocked tasks");
1559            return Ok(());
1560        }
1561    };
1562
1563    let ci_failure_tasks: Vec<_> = all_blocked
1564        .into_iter()
1565        .filter(is_ci_failure_block)
1566        .collect();
1567    if ci_failure_tasks.is_empty() {
1568        return Ok(());
1569    }
1570
1571    let gh = match GhHttp::new() {
1572        Ok(g) => g,
1573        Err(e) => {
1574            tracing::warn!(err = %e, "global CI-failure blocked sweep: failed to create GhHttp");
1575            return Ok(());
1576        }
1577    };
1578
1579    for task in ci_failure_tasks {
1580        if !ci_failure_unblock_cooldown_elapsed(&task) {
1581            continue;
1582        }
1583        if let Err(e) = try_unblock_ci_failure_task(
1584            &gh,
1585            &task.repo,
1586            task_manager,
1587            store,
1588            &task,
1589            auto_merge_in_flight,
1590        )
1591        .await
1592        {
1593            tracing::warn!(
1594                task_id = task.id,
1595                repo = %task.repo,
1596                err = %e,
1597                "global CI-failure blocked sweep: unblock check failed"
1598            );
1599        }
1600    }
1601
1602    Ok(())
1603}
1604
1605/// Global sweep: re-fire stale NeedsReview tasks and escalate persistently-stuck ones
1606/// to Blocked, across all repos.
1607///
1608/// Two cases where the event-driven subscriber in `subscribers/review.rs` can miss
1609/// a NeedsReview task: all review slots are busy when the event fires, or the
1610/// broadcast receiver lagged and dropped it. Re-publish the NeedsReview status
1611/// after a short idle window so the subscriber can pick it up when a slot is free.
1612/// Using a status re-fire rather than spawning the review agent directly keeps this
1613/// path stateless. The subscriber is still the sole spawner and uses the InReview
1614/// transition as its atomic guard (issue #857)
1615///
1616/// Queries across all repos and writes through the store via `*_by_store_id`
1617/// TaskManager helpers. Called once per main tick from `tick_global_sweeps`
1618pub(crate) async fn refire_and_escalate_stale_needs_review_global(
1619    task_manager: &Arc<TaskManager>,
1620    store: &Arc<TaskStore>,
1621    router: &Router,
1622    dispatching: &Arc<DashMap<String, String>>,
1623    active_repos: &HashSet<String>,
1624) -> anyhow::Result<()> {
1625    const MIN_STALE_NEEDS_REVIEW_MINUTES: i64 = 1;
1626    const MAX_NEEDS_REVIEW_REFIRE_ATTEMPTS: u64 = 5;
1627
1628    let needs_review_tasks = match store
1629        .list_all_by_status_global(TaskStatus::NeedsReview)
1630        .await
1631    {
1632        Ok(tasks) => tasks,
1633        Err(e) => {
1634            tracing::warn!(err = %e, "global NeedsReview refire sweep: failed to list needs_review tasks");
1635            return Ok(());
1636        }
1637    };
1638
1639    if needs_review_tasks.is_empty() {
1640        return Ok(());
1641    }
1642
1643    // Re-firing while every review agent is cooled wastes the counter and can
1644    // escalate a task to Blocked while the subscriber would only bail again
1645    let all_review_agents_cooled = router.healthy_agent_count("review") == 0;
1646    if all_review_agents_cooled {
1647        tracing::info!(
1648            count = needs_review_tasks.len(),
1649            "global NeedsReview refire sweep: no review agent available — skipping this pass"
1650        );
1651        return Ok(());
1652    }
1653
1654    for stored in needs_review_tasks {
1655        // No review subscriber listens for a repo outside the active set
1656        if !active_repos.contains(&stored.repo) {
1657            continue;
1658        }
1659
1660        let external = tasks::store_task_to_external(&stored);
1661        let ext_id = external.id.clone();
1662        let repo = stored.repo.clone();
1663
1664        let age_minutes = match chrono::DateTime::parse_from_rfc3339(&external.updated_at) {
1665            Ok(updated_at) => {
1666                let age = chrono::Utc::now() - updated_at.with_timezone(&chrono::Utc);
1667                if age.num_minutes() < MIN_STALE_NEEDS_REVIEW_MINUTES {
1668                    continue;
1669                }
1670                Some(age.num_minutes())
1671            }
1672            Err(_) => None,
1673        };
1674
1675        // Skip tasks actively being dispatched by their owning repo's engine loop.
1676        let dispatch_key = format!("{}/{}", repo, ext_id.0);
1677        if dispatching.contains_key(&dispatch_key) {
1678            continue;
1679        }
1680
1681        let current_refires = u64::try_from(stored.needs_review_refires).unwrap_or(0);
1682        let new_refires = current_refires + 1;
1683
1684        if new_refires > MAX_NEEDS_REVIEW_REFIRE_ATTEMPTS {
1685            tracing::warn!(
1686                task_id = stored.id,
1687                repo = %repo,
1688                new_refires,
1689                "global NeedsReview refire sweep: escalating to Blocked after repeated refires"
1690            );
1691            let fields = [
1692                ("block_reason", serde_json::json!(REBROADCAST_BLOCK_REASON)),
1693                (
1694                    "last_error",
1695                    serde_json::json!(format!("escalated after {} retries", new_refires)),
1696                ),
1697            ];
1698            if let Err(e) = task_manager
1699                .update_task_status_and_result_by_store_id(
1700                    stored.id,
1701                    &repo,
1702                    &ext_id,
1703                    Status::Blocked,
1704                    &fields,
1705                )
1706                .await
1707            {
1708                tracing::error!(task_id = stored.id, repo = %repo, err = %e, "global NeedsReview refire sweep: escalation to Blocked failed — skipping to avoid silent auto-unblock loop");
1709                continue;
1710            }
1711            if let Err(e) = store::store_increment_by_id(
1712                &Some(Arc::clone(store)),
1713                stored.id,
1714                "needs_review_refires",
1715            )
1716            .await
1717            {
1718                tracing::warn!(
1719                    task_id = stored.id,
1720                    repo = %repo,
1721                    err = %e,
1722                    "global NeedsReview refire sweep: failed to increment needs_review_refires after escalation"
1723                );
1724            }
1725            continue;
1726        }
1727
1728        // Exponential backoff: MIN * 2^(current_refires)
1729        let required_minutes =
1730            MIN_STALE_NEEDS_REVIEW_MINUTES * (1i64 << u32::try_from(current_refires).unwrap_or(0));
1731
1732        let should_fire = match age_minutes {
1733            Some(age) => age >= required_minutes,
1734            None => true,
1735        };
1736
1737        if !should_fire {
1738            continue;
1739        }
1740
1741        if let Err(e) = task_manager
1742            .update_task_status_by_store_id(stored.id, &repo, &ext_id, Status::NeedsReview)
1743            .await
1744        {
1745            tracing::warn!(
1746                task_id = stored.id,
1747                repo = %repo,
1748                err = %e,
1749                "global NeedsReview refire sweep: failed to re-fire NeedsReview event — not incrementing counter"
1750            );
1751            continue;
1752        }
1753
1754        let fired_refires = match store::store_increment_by_id(
1755            &Some(Arc::clone(store)),
1756            stored.id,
1757            "needs_review_refires",
1758        )
1759        .await
1760        {
1761            Ok(v) => v,
1762            Err(e) => {
1763                tracing::warn!(
1764                    task_id = stored.id,
1765                    repo = %repo,
1766                    err = %e,
1767                    "global NeedsReview refire sweep: failed to increment needs_review_refires after re-fire"
1768                );
1769                current_refires + 1
1770            }
1771        };
1772
1773        tracing::info!(
1774            task_id = stored.id,
1775            repo = %repo,
1776            age_minutes,
1777            refires = fired_refires,
1778            required_minutes,
1779            "global NeedsReview refire sweep: re-firing NeedsReview event for stale task"
1780        );
1781    }
1782
1783    Ok(())
1784}
1785
1786/// Sync tick — runs every 45s.
1787///
1788/// Handles less-frequent operations:
1789/// - Cleanup finished worktrees
1790/// - Check for merged PRs → mark tasks done
1791/// - Scan for @mentions
1792#[expect(
1793    clippy::too_many_arguments,
1794    reason = "the engine passes its shared handles one by one"
1795)]
1796pub(crate) async fn sync_tick(
1797    backend: &Arc<dyn ExternalBackend>,
1798    tmux: &Arc<TmuxManager>,
1799    repo: &str,
1800    config: &EngineConfig,
1801    router: &Arc<RwLock<Router>>,
1802    task_manager: &Arc<TaskManager>,
1803    store: &Arc<TaskStore>,
1804    dispatching: &Arc<DashMap<String, String>>,
1805    auto_merge_in_flight: &Arc<DashSet<String>>,
1806) -> anyhow::Result<()> {
1807    // One node polls GitHub and decides for the whole cluster
1808    if !store.is_leader() {
1809        return Ok(());
1810    }
1811
1812    tracing::debug!("sync tick");
1813
1814    let mut phases: Vec<(&str, u128)> = Vec::new();
1815    let mut at = Instant::now();
1816
1817    // Global GitHub 5xx circuit breaker — skip non-critical background work
1818    // during sustained GitHub outages to reduce churn and wasted API calls.
1819    let gh_circuit_open = is_github_circuit_open();
1820
1821    if gh_circuit_open {
1822        let remaining = github_circuit_remaining_secs();
1823        tracing::debug!(
1824            remaining_secs = remaining,
1825            "GitHub 5xx circuit breaker open — skipping non-critical sync work"
1826        );
1827        // Persist circuit breaker state as a metric for operators.
1828        // This is critical for avoiding thundering herd after restart during outages.
1829        if let Err(e) =
1830            try_kv_set_prefer_store(Some(store), "metrics:orch.github_5xx_circuit.open", "1").await
1831        {
1832            tracing::error!(err = %e, "failed to persist circuit breaker state — may cause thundering herd after restart");
1833        }
1834    } else if let Err(e) =
1835        try_kv_set_prefer_store(Some(store), "metrics:orch.github_5xx_circuit.open", "0").await
1836    {
1837        tracing::error!(err = %e, "failed to clear circuit breaker state");
1838    }
1839
1840    // 0. Ingest all active external tasks into the unified store.
1841    //    This ensures the store has data for tasks created before dual-write was added.
1842    if !gh_circuit_open {
1843        if let Err(e) = ingest_external_tasks(backend, repo, store).await {
1844            tracing::debug!(err = %e, "external task ingest failed");
1845        }
1846        phases.push(("ingest", at.elapsed().as_millis()));
1847        at = Instant::now();
1848
1849        // Reconcile existing board members too: labels can change without a
1850        // local task transition, and a human/automation may move a card. The
1851        // backend only writes drifted fields through the shared GitHub client.
1852        if let Err(e) = backend.reconcile_project_statuses().await {
1853            tracing::warn!(err = %e, "project status reconciliation failed");
1854        }
1855        phases.push(("board_reconcile", at.elapsed().as_millis()));
1856    }
1857    at = Instant::now();
1858
1859    // 1. Cleanup worktrees for done tasks (background — must not block routing/dispatch)
1860    if !gh_circuit_open {
1861        let backend = Arc::clone(backend);
1862        let repo = repo.to_string();
1863        let task_manager = Arc::clone(task_manager);
1864        let store = Arc::clone(store);
1865        tokio::spawn(async move {
1866            if let Err(e) = cleanup_done_worktrees(&backend, &repo, &task_manager, &store).await {
1867                tracing::warn!(err = %e, "worktree cleanup failed");
1868            }
1869            if let Err(e) = prune_old_run_output(&store).await {
1870                tracing::warn!(err = %e, "task run pruning failed");
1871            }
1872        });
1873    }
1874
1875    let (in_review_tasks, needs_review_tasks, blocked_review_candidates) =
1876        match prefetch_review_tasks(store, repo).await {
1877            Ok(tasks) => tasks,
1878            Err(e) => {
1879                tracing::warn!(err = %e, "failed to prefetch review tasks");
1880                (vec![], vec![], vec![])
1881            }
1882        };
1883    phases.push(("prefetch_review", at.elapsed().as_millis()));
1884    at = Instant::now();
1885
1886    if !gh_circuit_open {
1887        let mention_fallback = chrono::Utc::now() - chrono::Duration::hours(24);
1888        let mention_since = kv_get_prefer_store(Some(store), "mentions_last_checked")
1889            .await
1890            .unwrap_or_else(|| mention_fallback.format("%Y-%m-%dT%H:%M:%SZ").to_string());
1891        let shared_since = mention_since.clone();
1892
1893        let cached_router_config = router.read().await.config.clone();
1894        // Include blocked tasks that have branch/pr info when checking for merged PRs.
1895        let mut combined_needs = needs_review_tasks.clone();
1896        combined_needs.extend(blocked_review_candidates.clone());
1897
1898        let (merge_result, comments_result, review_result) = tokio::join!(
1899            check_merged_prs(
1900                backend,
1901                repo,
1902                store,
1903                task_manager,
1904                &in_review_tasks,
1905                &combined_needs,
1906            ),
1907            fetch_comments_since(backend, &shared_since),
1908            async {
1909                let gh = match GhHttp::new() {
1910                    Ok(g) => g,
1911                    Err(e) => {
1912                        tracing::warn!(err = %e, "failed to create GitHub HTTP client for review poll");
1913                        return Ok(());
1914                    }
1915                };
1916                Box::pin(review_open_prs(
1917                    backend,
1918                    repo,
1919                    config,
1920                    task_manager,
1921                    store,
1922                    dispatching,
1923                    auto_merge_in_flight,
1924                    &in_review_tasks,
1925                    &gh,
1926                    &cached_router_config,
1927                ))
1928                .await
1929            }
1930        );
1931
1932        if let Err(e) = merge_result {
1933            tracing::warn!(err = %e, "PR merge check failed");
1934        }
1935
1936        match comments_result {
1937            Ok(comments) => {
1938                let filtered = filter_mentions_by_since(&comments, &mention_since);
1939
1940                if let Err(e) =
1941                    scan_comments(backend, Some(store), repo, task_manager, Some(&filtered)).await
1942                {
1943                    tracing::warn!(err = %e, "comment scan failed");
1944                }
1945            }
1946            Err(e) => {
1947                tracing::warn!(err = %e, "comment fetch failed");
1948            }
1949        }
1950
1951        if let Err(e) = review_result {
1952            tracing::warn!(err = %e, "PR review failed");
1953        }
1954
1955        if let Err(e) = ci_bootstrap::apply_pending(store, task_manager, repo).await {
1956            tracing::warn!(err = %e, "CI bootstrap ruleset apply failed");
1957        }
1958
1959        phases.push(("merge_comments_review", at.elapsed().as_millis()));
1960    }
1961    at = Instant::now();
1962
1963    // 5. Stale InReview recovery lives in `tick_recover_stuck_tasks`, the single owner of an
1964    // in_review task whose review session is gone or hung. It ran here too, after one minute,
1965    // which beat a finished review's result to the engine and dropped it as stale
1966
1967    // 5c. Detect stale InProgress tasks and recover them.
1968    //
1969    // Similar to stale InReview detection, we check for InProgress tasks whose
1970    // agent tmux session has died (crash, OOM, etc.) and reset them to Routed
1971    // so they can be re-dispatched.
1972    let in_progress_tasks: Vec<_> = match store.list_by_status(repo, TaskStatus::InProgress).await {
1973        Ok(tasks) => tasks
1974            .into_iter()
1975            .map(|stored| {
1976                let external = tasks::store_task_to_external(&stored);
1977                (stored, external)
1978            })
1979            .collect(),
1980        Err(e) => {
1981            tracing::warn!(err = %e, "failed to list in_progress tasks — skipping stale InProgress check");
1982            vec![]
1983        }
1984    };
1985
1986    if !in_progress_tasks.is_empty() {
1987        // Fetch all live tmux sessions once instead of one subprocess call per task.
1988        let live_sessions: HashSet<String> = tmux
1989            .list_sessions()
1990            .await
1991            .unwrap_or_default()
1992            .into_iter()
1993            .map(|s| s.name)
1994            .collect();
1995
1996        for (_stored, external) in &in_progress_tasks {
1997            // A task is only considered stale if it has been in InProgress for > 5 minutes.
1998            const MIN_STALE_MINUTES: i64 = 5;
1999
2000            let task_id = &external.id.0;
2001
2002            // Skip tasks currently being processed by the main tick (dispatch + agent flow).
2003            let dispatch_key = format!("{repo}/{task_id}");
2004            {
2005                if dispatching.contains_key(&dispatch_key) {
2006                    tracing::debug!(
2007                        task_id = %task_id,
2008                        "task locked by dispatch flow, skipping stale check"
2009                    );
2010                    continue;
2011                }
2012            }
2013
2014            // Skip tasks that just transitioned to InProgress — allow time for the
2015            // agent to start its tmux session before treating it as stale.
2016            match chrono::DateTime::parse_from_rfc3339(&external.updated_at) {
2017                Ok(updated_at) => {
2018                    let age = chrono::Utc::now() - updated_at.with_timezone(&chrono::Utc);
2019                    if age.num_minutes() < MIN_STALE_MINUTES {
2020                        tracing::debug!(
2021                            task_id = %task_id,
2022                            age_seconds = age.num_seconds(),
2023                            "InProgress task is too young to be considered stale, skipping"
2024                        );
2025                        continue;
2026                    }
2027                }
2028                Err(e) => {
2029                    tracing::warn!(
2030                        task_id = %task_id,
2031                        ts = %external.updated_at,
2032                        err = %e,
2033                        "invalid updated_at timestamp — treating task as potentially stale"
2034                    );
2035                }
2036            }
2037
2038            let agent_session = tmux.session_name(repo, task_id);
2039            if live_sessions.contains(&agent_session) {
2040                // Agent is still alive — skip.
2041                continue;
2042            }
2043
2044            // No agent session exists. The agent is dead regardless of
2045            // session tracking — hard restart, crash, or flag not persisted.
2046            // Reset to Routed so the dispatcher can re-spawn it.
2047            {
2048                tracing::warn!(
2049                    task_id = %task_id,
2050                    session = %agent_session,
2051                    "InProgress task has no active agent session — resetting to Routed"
2052                );
2053                match task_manager
2054                    .update_task_status_if(&external.id, Status::Routed, Status::InProgress)
2055                    .await
2056                {
2057                    Err(e) => {
2058                        tracing::error!(task_id = %task_id, err = %e, "failed to reset stale InProgress task — task may be stuck in InProgress indefinitely");
2059                    }
2060                    Ok(false) => {
2061                        tracing::debug!(
2062                            task_id = %task_id,
2063                            "stale InProgress reset skipped — task already transitioned (concurrent Done/Blocked/NeedsReview)"
2064                        );
2065                    }
2066                    Ok(true) => {
2067                        // Successfully reset to Routed
2068                    }
2069                }
2070            }
2071        }
2072    }
2073
2074    phases.push(("stale_in_progress", at.elapsed().as_millis()));
2075    at = Instant::now();
2076    // 5c. Auto-unblock tasks with recoverable failures.
2077    if let Err(e) =
2078        auto_unblock_blocked_tasks(repo, task_manager, store, dispatching, auto_merge_in_flight)
2079            .await
2080    {
2081        tracing::warn!(err = %e, "auto-unblock failed");
2082    }
2083
2084    phases.push(("auto_unblock", at.elapsed().as_millis()));
2085
2086    engine::log_phase_timings("sync", repo, &phases);
2087    Ok(())
2088}
2089
2090/**
2091 * Agent health is the same whichever project asks, so it runs once per sync pass rather than
2092 * once per project. Called after every project's `sync_tick`
2093 **/
2094pub(crate) async fn sync_shared_tail(router: &Arc<RwLock<Router>>, store: Option<&Arc<TaskStore>>) {
2095    let Some(store) = store else {
2096        return;
2097    };
2098    if !store.is_leader() {
2099        return;
2100    }
2101
2102    // Pre-emptive health check: refresh degraded-agent flags from the rate_limits table.
2103    // Clone what is needed out of the router first, so the write lock is not starved for the
2104    // duration of the DB query
2105    let (available_agents, config) = {
2106        let r = router.read().await;
2107        (r.available_agents.clone(), r.config.clone())
2108    };
2109    let model_checker = |agent: &str| -> bool {
2110        for comp in &["simple", "medium", "complex", "review"] {
2111            if config.has_available_model_for_complexity(agent, comp) {
2112                return true;
2113            }
2114        }
2115        false
2116    };
2117    refresh_degraded_agents(
2118        store,
2119        &available_agents,
2120        &model_checker,
2121        router_config::health_check_window_hours(),
2122        router_config::degraded_rate_limit_threshold(),
2123    )
2124    .await;
2125
2126    emit_degraded_agents_if_needed(&available_agents, &config, Some(store)).await;
2127}
2128
2129/// Scan for @mentions and handle them.
2130///
2131/// Checks recent issue comments for @orch (and legacy @orchestrator) mentions.
2132/// Unified comment scanner — handles both @mentions and slash commands in one pass.
2133///
2134/// For each comment, classifies it via [`classify_comment`] and dispatches:
2135/// - Slash commands → validate + execute (with sentinel task if it was an @mention)
2136/// - @mentions without commands → create internal task for agent to respond
2137/// - Everything else → skip
2138async fn scan_comments(
2139    backend: &Arc<dyn ExternalBackend>,
2140    store: Option<&Arc<TaskStore>>,
2141    repo: &str,
2142    task_manager: &Arc<TaskManager>,
2143    prefetched_comments: Option<&[Mention]>,
2144) -> anyhow::Result<()> {
2145    let current_user = match backend.get_authenticated_user().await {
2146        Ok(Some(u)) => format!("@{u}"),
2147        Ok(None) => {
2148            tracing::debug!("backend does not support user identity, skipping comment scan");
2149            return Ok(());
2150        }
2151        Err(e) => {
2152            tracing::warn!(err = %e, "failed to get current user, skipping comment scan");
2153            return Ok(());
2154        }
2155    };
2156
2157    let comments = if let Some(c) = prefetched_comments {
2158        c.to_vec()
2159    } else {
2160        let fallback = chrono::Utc::now() - chrono::Duration::hours(24);
2161        let since_str = match kv_get_prefer_store(store, "mentions_last_checked").await {
2162            Some(ts) => ts,
2163            None => fallback.format("%Y-%m-%dT%H:%M:%SZ").to_string(),
2164        };
2165        match backend.get_mentions(&since_str).await {
2166            Ok(m) => m,
2167            Err(e) => {
2168                tracing::warn!(err = %e, "failed to get comments");
2169                return Ok(());
2170            }
2171        }
2172    };
2173
2174    // Dedup set: mention tasks already created (by source_id).
2175    let existing_mentions: HashSet<String> = if let Some(s) = store {
2176        match s.list_source_ids_by_source(repo, "mention", 30).await {
2177            Ok(ids) => ids.into_iter().collect(),
2178            Err(e) => {
2179                tracing::warn!(err = %e, "failed to load mention source IDs — skipping to prevent duplicates");
2180                return Ok(());
2181            }
2182        }
2183    } else {
2184        HashSet::new()
2185    };
2186
2187    // Lazily create GhHttp — only allocate when we actually need to make GitHub
2188    // API calls.  This avoids network initialization in test environments where
2189    // the code path never reaches an API call.
2190    let gh_cell: OnceCell<GhHttp> = OnceCell::new();
2191    let get_gh = || async { gh_cell.get_or_try_init(|| async { GhHttp::new() }).await };
2192
2193    // Pre-classify all comments up front so we can batch the is_pull_request
2194    // checks for CreateMentionTask entries before entering the main loop.
2195    let actions: Vec<CommentAction> = comments
2196        .iter()
2197        .map(|c| {
2198            classify_comment(
2199                &c.body,
2200                c.issue_url.as_deref(),
2201                &current_user,
2202                existing_mentions.contains(&c.id),
2203            )
2204        })
2205        .collect();
2206
2207    // Collect (comment_index, issue_num) for all CreateMentionTask entries that
2208    // need an is_pull_request check, then resolve them all concurrently with
2209    // join_all to avoid N × RTT serial API calls inside the loop.
2210    let pr_check_inputs: Vec<(usize, String)> = actions
2211        .iter()
2212        .enumerate()
2213        .filter_map(|(i, action)| {
2214            if let CommentAction::CreateMentionTask {
2215                issue_num: Some(num),
2216            } = action
2217            {
2218                Some((i, num.clone()))
2219            } else {
2220                None
2221            }
2222        })
2223        .collect();
2224
2225    // Only create GhHttp if we actually have PR checks to perform.
2226    let is_pr_values: Vec<bool> = if pr_check_inputs.is_empty() {
2227        Vec::new()
2228    } else {
2229        let gh = get_gh().await?;
2230        future::join_all(
2231            pr_check_inputs
2232                .iter()
2233                .map(|(_, num)| async move { gh.is_pull_request(repo, num).await }),
2234        )
2235        .await
2236    };
2237
2238    // Build a lookup map: comment_index → is_pr.
2239    let is_pr_map: HashMap<usize, bool> = pr_check_inputs
2240        .iter()
2241        .map(|(i, _)| *i)
2242        .zip(is_pr_values)
2243        .collect();
2244
2245    let mut cursor = MentionCursor::default();
2246
2247    for (comment_idx, (comment, action)) in comments.iter().zip(actions).enumerate() {
2248        match action {
2249            CommentAction::Skip => {
2250                cursor.advance(&comment.created_at);
2251            }
2252
2253            CommentAction::ExecuteCommand { command, issue_num } => {
2254                let store_opt: Option<Arc<dyn CommandStoreOps>> =
2255                    store.map(|s| Arc::clone(s) as Arc<dyn CommandStoreOps>);
2256                let gh = get_gh().await?;
2257                let outcome = validate_and_run_command(
2258                    backend,
2259                    gh,
2260                    repo,
2261                    &issue_num,
2262                    &command,
2263                    &comment.author,
2264                    &store_opt,
2265                    task_manager,
2266                )
2267                .await;
2268                match outcome {
2269                    // Validation hit a transient GitHub error: nothing ran and nothing
2270                    // was posted. Stop the cursor so the next tick retries this comment.
2271                    CommandOutcome::FetchFailed | CommandOutcome::CollaboratorCheckFailed => {
2272                        tracing::warn!(
2273                            comment_id = %comment.id,
2274                            command = %command,
2275                            "deferring slash command after a transient GitHub error"
2276                        );
2277                        cursor.block_on_gap();
2278                        break;
2279                    }
2280                    // The outcome comment (success or error) was already posted to
2281                    // GitHub, so re-processing would create duplicates.
2282                    CommandOutcome::Executed { .. }
2283                    | CommandOutcome::NotOpen { .. }
2284                    | CommandOutcome::NotCollaborator { .. } => cursor.advance(&comment.created_at),
2285                }
2286            }
2287
2288            CommentAction::ExecuteCommandForMention { command, .. } => {
2289                if let Err(e) = backend.acknowledge_mention(&comment.id).await {
2290                    tracing::debug!(err = %e, mention_id = %comment.id, "failed to acknowledge mention");
2291                    // Do NOT advance the cursor or call handle_slash_command — the GitHub
2292                    // reaction was never posted. Retry on next tick once the network
2293                    // glitch has resolved.
2294                } else {
2295                    // Parent lookup failure propagates via `?` — scan_comments returns early
2296                    // without advancing the cursor so the mention is retried on next sync.
2297                    let gh = get_gh().await?;
2298                    let advanced_safely = handle_slash_command(
2299                        backend,
2300                        store,
2301                        repo,
2302                        task_manager,
2303                        gh,
2304                        comment,
2305                        &command,
2306                        &mut cursor,
2307                    )
2308                    .await?;
2309                    if !advanced_safely {
2310                        cursor.block_on_gap();
2311                        break;
2312                    }
2313                }
2314            }
2315
2316            CommentAction::CreateMentionTask { issue_num } => {
2317                if let Err(e) = backend.acknowledge_mention(&comment.id).await {
2318                    tracing::debug!(err = %e, mention_id = %comment.id, "failed to acknowledge mention");
2319                    // Do NOT record the mention task or advance the cursor — the GitHub
2320                    // reaction was never posted. Retry on next tick once the network
2321                    // glitch has resolved.
2322                } else {
2323                    // Use the pre-fetched value — no serial API call here.
2324                    // issue_num may be None (e.g. bot mentioned with no issue URL), in
2325                    // which case is_pr_map has no entry; default to false (the None arm
2326                    // of the downstream match ignores is_pr entirely).
2327                    let is_pr = is_pr_map.get(&comment_idx).copied().unwrap_or(false);
2328
2329                    let (title, task_body) = match (&issue_num, is_pr) {
2330                        (Some(num), false) => (
2331                            format!("Respond to mention by @{} on task #{num}", comment.author),
2332                            format!(
2333                                "Mention by @{} on GitHub issue #{num}:\n\n{}\n\n---\nThis mention was posted on issue #{num}. Review the full issue to understand the context before responding.",
2334                                comment.author, comment.body
2335                            ),
2336                        ),
2337                        (Some(num), true) => (
2338                            format!("Respond to mention by @{} on PR #{num}", comment.author),
2339                            format!(
2340                                "Mention by @{} on GitHub PR #{num}:\n\n{}\n\n---\nThis mention was posted on PR #{num}. Review the full PR (diff, description, and comments) to understand the context before responding.",
2341                                comment.author, comment.body
2342                            ),
2343                        ),
2344                        (None, _) => (
2345                            format!("Respond to mention by @{}", comment.author),
2346                            format!("Mention by @{}:\n\n{}", comment.author, comment.body),
2347                        ),
2348                    };
2349
2350                    if let Some(s) = store {
2351                        // On DB lookup failure, do NOT record the mention task or advance
2352                        // the cursor — the mention will be retried on next sync.
2353                        let parent_id = match issue_num {
2354                            Some(ref num) => {
2355                                match resolve_mention_parent_id(s, repo, num, is_pr).await {
2356                                    Ok(id) => id,
2357                                    Err(e) => {
2358                                        tracing::warn!(
2359                                            mention_id = %comment.id,
2360                                            err = %e,
2361                                            "deferring mention task due to parent lookup failure"
2362                                        );
2363                                        return Err(anyhow::anyhow!(
2364                                            "deferring mention {} due to parent lookup failure",
2365                                            comment.id
2366                                        ));
2367                                    }
2368                                }
2369                            }
2370                            None => None,
2371                        };
2372                        if !record_mention_task(
2373                            s,
2374                            repo,
2375                            comment,
2376                            &title,
2377                            &task_body,
2378                            parent_id,
2379                            &mut cursor,
2380                        )
2381                        .await
2382                        {
2383                            cursor.block_on_gap();
2384                            break;
2385                        }
2386                    }
2387                }
2388            }
2389        }
2390    }
2391
2392    if let Some(ts) = cursor.into_last_safe_ts() {
2393        kv_set_prefer_store(store, "mentions_last_checked", &ts).await;
2394    }
2395
2396    Ok(())
2397}
2398
2399/// Per-agent degradation detail collected during the alerting check.
2400struct DegradedAgentDetail {
2401    /// Agent name (e.g. "claude", "codex").
2402    agent: String,
2403    /// Reason the agent is degraded: the stored cooldown reason when the agent
2404    /// itself is in cooldown, or `"no_available_model"` when all model pools
2405    /// are individually cooled.
2406    reason: String,
2407    /// Models that are individually in cooldown for this agent, deduped across
2408    /// all complexity tiers.
2409    cooled_models: Vec<String>,
2410}
2411
2412/// Per-agent pool health used for stale model pool detection.
2413struct AgentPoolHealth {
2414    agent: String,
2415    cooled_models: Vec<String>,
2416    persistent_failed_models: Vec<String>,
2417    total_configured_models: usize,
2418}
2419
2420/// Inspect router & cooldown state and emit metrics every tick.
2421///
2422/// - `metrics:orch.agents_degraded.count` is written on every call with the
2423///   current number of degraded agents (0 when healthy).
2424/// - When `count >= 3` a WARN log is emitted with full dimensions (agent
2425///   names, cooled models, cooldown reasons) and a dedicated alert metric
2426///   `metrics:orch.agents_degraded.alert` is set to `"1"`. The alert is
2427///   cleared to `"0"` when the count drops below the threshold.
2428pub(crate) async fn emit_degraded_agents_if_needed(
2429    available_agents: &[String],
2430    config: &RouterConfig,
2431    store: Option<&Arc<TaskStore>>,
2432) {
2433    // Edge-triggered: only log when the affected-agents set changes, so a
2434    // persistent condition doesn't flood the log every sync tick.
2435    const STALE_AGENTS_KV: &str = "metrics:orch.model_pool_stale_persistent.agents";
2436
2437    // Complexity tiers to check for configured models.
2438    const COMPLEXITIES: &[&str] = &["simple", "medium", "complex", "review"];
2439    const HEAVILY_COOLED_RATIO_THRESHOLD: f64 = 0.5;
2440
2441    let mut details: Vec<DegradedAgentDetail> = Vec::new();
2442    let mut pool_health: Vec<AgentPoolHealth> = Vec::new();
2443
2444    for agent in available_agents {
2445        let agent_in_cd = is_agent_in_cooldown(agent);
2446
2447        // Collect individually cooled models across all complexity tiers (deduped).
2448        let mut cooled_models: Vec<String> = Vec::new();
2449        let mut persistent_failed_models: Vec<String> = Vec::new();
2450        let mut seen_models = HashSet::new();
2451        for comp in COMPLEXITIES {
2452            if let Some(pool) = config.model_pool_for_complexity(agent, comp) {
2453                for model in pool {
2454                    if seen_models.insert(model.clone()) && is_model_in_cooldown(agent, &model) {
2455                        let key = format!("{agent}:{model}");
2456                        if matches!(
2457                            cooldown_reason(&key).as_deref(),
2458                            Some("persistent_model_error" | "model_not_found")
2459                        ) {
2460                            persistent_failed_models.push(model.clone());
2461                        }
2462                        cooled_models.push(model);
2463                    }
2464                }
2465            }
2466        }
2467        let total_configured_models = seen_models.len();
2468        pool_health.push(AgentPoolHealth {
2469            agent: agent.clone(),
2470            cooled_models: cooled_models.clone(),
2471            persistent_failed_models: persistent_failed_models.clone(),
2472            total_configured_models,
2473        });
2474
2475        // An agent is degraded if it is in agent-level cooldown OR has no
2476        // available (non-cooled) model across any complexity tier.
2477        let has_model = COMPLEXITIES
2478            .iter()
2479            .any(|comp| config.has_available_model_for_complexity(agent, comp));
2480
2481        if agent_in_cd || !has_model {
2482            let reason = if agent_in_cd {
2483                cooldown_reason(agent).unwrap_or_else(|| "agent_cooldown".to_string())
2484            } else {
2485                "no_available_model".to_string()
2486            };
2487            details.push(DegradedAgentDetail {
2488                agent: agent.clone(),
2489                reason,
2490                cooled_models,
2491            });
2492        }
2493    }
2494
2495    let count = details.len();
2496
2497    // Always persist the count metric so operators can scrape it every tick,
2498    // including when the count is 0.
2499    if let Err(e) = try_kv_set_prefer_store(
2500        store,
2501        "metrics:orch.agents_degraded.count",
2502        &count.to_string(),
2503    )
2504    .await
2505    {
2506        tracing::warn!(err = %e, "failed to persist degraded agents count metric");
2507    }
2508
2509    if count >= 3 {
2510        // Build structured dimension strings for the log.
2511        let degraded_agents: Vec<&str> = details.iter().map(|d| d.agent.as_str()).collect();
2512        let cooled_models_dim: Vec<String> = details
2513            .iter()
2514            .filter(|d| !d.cooled_models.is_empty())
2515            .map(|d| format!("{}:[{}]", d.agent, d.cooled_models.join(",")))
2516            .collect();
2517        let reasons_dim: Vec<String> = details
2518            .iter()
2519            .map(|d| format!("{}={}", d.agent, d.reason))
2520            .collect();
2521
2522        tracing::warn!(
2523            degraded_count = count,
2524            degraded_agents = ?degraded_agents,
2525            cooled_models = %cooled_models_dim.join("; "),
2526            cooldown_reasons = %reasons_dim.join("; "),
2527            "multi-agent degradation detected"
2528        );
2529
2530        // Dedicated alert metric: "1" while the threshold is exceeded.
2531        if let Err(e) =
2532            try_kv_set_prefer_store(store, "metrics:orch.agents_degraded.alert", "1").await
2533        {
2534            tracing::warn!(err = %e, "failed to persist degraded agents alert metric");
2535        }
2536    } else if let Err(e) =
2537        try_kv_set_prefer_store(store, "metrics:orch.agents_degraded.alert", "0").await
2538    {
2539        // Clear the alert metric when healthy.
2540        tracing::warn!(err = %e, "failed to clear degraded agents alert metric");
2541    }
2542
2543    // Additional operator signal for dead/stale model pools:
2544    // alert when any agent has at least 50% of configured models cooled and at
2545    // least one persistent model failure marker in cooldown reason.
2546    let heavily_cooled_agents: Vec<String> = pool_health
2547        .iter()
2548        .filter_map(|d| {
2549            if d.total_configured_models == 0 || d.persistent_failed_models.is_empty() {
2550                return None;
2551            }
2552            #[expect(
2553                clippy::cast_precision_loss,
2554                reason = "model counts are far below 2^52"
2555            )]
2556            let ratio = d.cooled_models.len() as f64 / d.total_configured_models as f64;
2557            (ratio >= HEAVILY_COOLED_RATIO_THRESHOLD).then(|| {
2558                format!(
2559                    "{}:{}/{}:{}",
2560                    d.agent,
2561                    d.cooled_models.len(),
2562                    d.total_configured_models,
2563                    d.persistent_failed_models.join(",")
2564                )
2565            })
2566        })
2567        .collect();
2568
2569    let current_dim = heavily_cooled_agents.join(",");
2570    let prev_dim = kv_get_prefer_store(store, STALE_AGENTS_KV)
2571        .await
2572        .unwrap_or_default();
2573
2574    if heavily_cooled_agents.is_empty() {
2575        if !prev_dim.is_empty() {
2576            tracing::info!("agent model pool stale condition cleared");
2577        }
2578        if let Err(e) = try_kv_set_prefer_store(store, STALE_AGENTS_KV, "").await {
2579            tracing::warn!(err = %e, "failed to clear stale model pool agents state");
2580        }
2581        if let Err(e) =
2582            try_kv_set_prefer_store(store, "metrics:orch.model_pool_stale_persistent.alert", "0")
2583                .await
2584        {
2585            tracing::warn!(err = %e, "failed to clear stale model pool alert metric");
2586        }
2587    } else {
2588        if current_dim != prev_dim {
2589            tracing::warn!(
2590                affected_agents = ?heavily_cooled_agents,
2591                "agent model pool appears stale: persistent model failures in heavily cooled pool"
2592            );
2593        }
2594        if let Err(e) = try_kv_set_prefer_store(store, STALE_AGENTS_KV, &current_dim).await {
2595            tracing::warn!(err = %e, "failed to persist stale model pool agents state");
2596        }
2597        if let Err(e) =
2598            try_kv_set_prefer_store(store, "metrics:orch.model_pool_stale_persistent.alert", "1")
2599                .await
2600        {
2601            tracing::warn!(err = %e, "failed to persist stale model pool alert metric");
2602        }
2603    }
2604}
2605
2606/// Ingest all active external tasks into the unified SQLite store.
2607///
2608/// Upserts each task so the store stays in sync with the backend.
2609/// Fetches all open issues (including unlabeled ones) so newly created
2610/// issues appear in the store immediately, not only after they get a
2611/// `status:*` label.  This is best-effort — individual task failures
2612/// are logged and skipped.
2613pub(crate) async fn ingest_external_tasks(
2614    backend: &Arc<dyn ExternalBackend>,
2615    repo: &str,
2616    store: &Arc<TaskStore>,
2617) -> anyhow::Result<()> {
2618    #[derive(Clone)]
2619    struct DuplicateTarget {
2620        id: String,
2621        status_label: String,
2622    }
2623
2624    let mut existing_by_title: HashMap<String, DuplicateTarget> = HashMap::new();
2625    let mut accepted_by_title: HashMap<String, DuplicateTarget> = HashMap::new();
2626
2627    let cutoff = (chrono::Utc::now() - chrono::Duration::hours(24))
2628        .format("%Y-%m-%dT%H:%M:%SZ")
2629        .to_string();
2630    match store.list_for_doctor(repo, &cutoff).await {
2631        Ok(recent) => {
2632            let mut insert_target = |task: &Task| {
2633                if task.origin == "internal" {
2634                    return;
2635                }
2636                let Some(ext_id) = task.external_id.as_ref() else {
2637                    return;
2638                };
2639                let status_label = task
2640                    .labels
2641                    .iter()
2642                    .find(|l| l.starts_with("status:"))
2643                    .cloned()
2644                    .unwrap_or_default();
2645                existing_by_title
2646                    .entry(task.title.clone())
2647                    .or_insert(DuplicateTarget {
2648                        id: ext_id.clone(),
2649                        status_label,
2650                    });
2651            };
2652
2653            // Prefer active tasks over recently done ones when selecting a canonical issue.
2654            for task in recent.iter().filter(|t| t.status != TaskStatus::Done) {
2655                insert_target(task);
2656            }
2657            for task in recent.iter().filter(|t| t.status == TaskStatus::Done) {
2658                insert_target(task);
2659            }
2660        }
2661        Err(e) => {
2662            tracing::warn!(repo, err = %e, "ingest: failed to load recent tasks for dedup");
2663        }
2664    }
2665
2666    // Fetch every open issue, with no `since` filter. The GitHub backend overrides
2667    // list_active_open_issues() to use one list_all_open_issues() request.
2668    //
2669    // A `since` cursor keys on updated_at, so an open issue with no task was skipped
2670    // forever once the cursor passed its updated_at, and the cursor had to be reset,
2671    // sentinelled and periodically ignored to recover (#3545, #3787, #3792). An
2672    // unfiltered listing has no such state and is cheaper too: its URL repeats, so
2673    // `get_all_pages` sends it conditionally and GitHub answers 304 while nothing
2674    // changed. A 304 does not count against the REST quota, a `since` fetch does.
2675    let all_tasks: Vec<(ExternalTask, Option<Status>)> =
2676        match backend.list_active_open_issues().await {
2677            Ok(tasks) => tasks,
2678            Err(e) => {
2679                tracing::warn!(
2680                    repo,
2681                    err = %e,
2682                    "ingest: failed to list open issues — no new issue is ingested this cycle"
2683                );
2684                return Ok(());
2685            }
2686        };
2687
2688    // Upsert into the store, collecting store IDs for batch status check
2689    // Also acknowledge newly detected issues (eyes reaction).
2690    let mut id_status_pairs: Vec<(i64, Status)> = Vec::new();
2691    // Collect (store_id, node_id) for newly ingested tasks so we can batch-fetch
2692    // estimate values from GitHub Projects after the loop.
2693    let mut new_task_node_ids: Vec<(i64, String)> = Vec::new();
2694    // Pre-load existing external IDs for this repo to avoid N+1 queries
2695    // (each get_by_external_id call is an individual SQL query). If the
2696    // lookup fails, fall back to an empty set so we conservatively treat
2697    // tasks as new (they will be upserted below).
2698    let existing_ext_ids: HashSet<String> = match store.existing_external_ids(repo).await {
2699        Ok(s) => s,
2700        Err(e) => {
2701            tracing::warn!(repo = repo, err = %e, "ingest: failed to load existing external ids — proceeding");
2702            HashSet::new()
2703        }
2704    };
2705    for (task, status) in &all_tasks {
2706        // Check if this task already exists in the store before upserting.
2707        // Use the pre-loaded HashSet to avoid issuing N individual queries.
2708        let is_new = !existing_ext_ids.contains(&task.id.0);
2709
2710        if is_new {
2711            let duplicate_target = existing_by_title
2712                .get(&task.title)
2713                .or_else(|| accepted_by_title.get(&task.title));
2714            if let Some(target) = duplicate_target
2715                && target.id != task.id.0
2716            {
2717                let label = target.status_label.as_str();
2718                let should_dedupe =
2719                    match backend.has_open_issue_with_title(&task.title, label).await {
2720                        Ok(true) => true,
2721                        Ok(false) => false,
2722                        Err(e) => {
2723                            tracing::warn!(
2724                                task_id = task.id.0,
2725                                err = %e,
2726                                "ingest: dedup check failed — defaulting to keep issue"
2727                            );
2728                            false
2729                        }
2730                    };
2731
2732                if should_dedupe {
2733                    let comment = format!(
2734                        "Duplicate of #{id} — closing to avoid duplicate work.{footer}",
2735                        id = target.id,
2736                        footer = engine::orch_footer()
2737                    );
2738                    if let Err(e) = backend.post_comment(&task.id, &comment).await {
2739                        tracing::warn!(
2740                            task_id = task.id.0,
2741                            err = %e,
2742                            "ingest: failed to comment on duplicate issue"
2743                        );
2744                    }
2745                    match backend.update_status(&task.id, Status::Done).await {
2746                        Ok(()) => {
2747                            tracing::info!(
2748                                task_id = task.id.0,
2749                                duplicate_of = target.id,
2750                                "ingest: closed duplicate issue"
2751                            );
2752                            continue;
2753                        }
2754                        Err(e) => {
2755                            tracing::warn!(
2756                                task_id = task.id.0,
2757                                err = %e,
2758                                "ingest: failed to close duplicate issue — falling through to ingest normally"
2759                            );
2760                        }
2761                    }
2762                }
2763            }
2764        }
2765
2766        match store.ensure_external_task(repo, task).await {
2767            Ok(store_id) => {
2768                if let Ok(number) = task.id.0.parse::<i64>()
2769                    && let Err(err) = dependencies::sync_issue_dependencies(
2770                        backend.as_ref(),
2771                        store,
2772                        store_id,
2773                        repo,
2774                        Some(number),
2775                        &task.body,
2776                    )
2777                    .await
2778                {
2779                    tracing::warn!(
2780                        task_id = task.id.0,
2781                        err = %err,
2782                        "ingest: dependency sync failed"
2783                    );
2784                }
2785                if let Some(s) = status {
2786                    id_status_pairs.push((store_id, *s));
2787                }
2788                // Acknowledge newly detected issues with an eyes reaction.
2789                if is_new {
2790                    let status_label = task
2791                        .labels
2792                        .iter()
2793                        .find(|l| l.starts_with("status:"))
2794                        .cloned()
2795                        .unwrap_or_default();
2796                    accepted_by_title
2797                        .entry(task.title.clone())
2798                        .or_insert(DuplicateTarget {
2799                            id: task.id.0.clone(),
2800                            status_label,
2801                        });
2802                    if let Err(e) = backend.acknowledge_issue(&task.id).await {
2803                        tracing::debug!(
2804                            task_id = task.id.0,
2805                            err = %e,
2806                            "ingest: acknowledgment failed"
2807                        );
2808                    }
2809                    // Sync to project board only for newly ingested tasks.
2810                    // Collect the returned node_id so we can batch-fetch estimates.
2811                    let task_status = status.unwrap_or(Status::New);
2812                    match backend.sync_to_project(&task.id, task_status).await {
2813                        Ok(Some(node_id)) => {
2814                            new_task_node_ids.push((store_id, node_id));
2815                        }
2816                        Ok(None) => {
2817                            // Project sync not configured or failed — no node_id to collect.
2818                        }
2819                        Err(e) => {
2820                            tracing::debug!(
2821                                task_id = task.id.0,
2822                                err = %e,
2823                                "ingest: project board sync failed"
2824                            );
2825                        }
2826                    }
2827                }
2828            }
2829            Err(e) => {
2830                tracing::debug!(task_id = task.id.0, ?e, "ingest: upsert failed");
2831            }
2832        }
2833    }
2834
2835    // Batch-fetch all upserted tasks in a single query, then sync status for NEW tasks.
2836    // Only sync status from backend → store for NEW tasks (first ingest).
2837    // Once a task exists in the store, its status is authoritative —
2838    // re-ingestion must not overwrite store-first status changes
2839    // (e.g., store has Routed but GitHub still shows New labels).
2840    if !id_status_pairs.is_empty() {
2841        let all_ids: Vec<i64> = id_status_pairs.iter().map(|(id, _)| *id).collect();
2842        let existing_map = match store.get_batch(&all_ids).await {
2843            Ok(map) => map,
2844            Err(e) => {
2845                tracing::warn!(err = %e, task_count = all_ids.len(), "ingest: get_batch failed — skipping initial status sync for newly ingested tasks this tick");
2846                return Ok(());
2847            }
2848        };
2849        for (store_id, status) in id_status_pairs {
2850            if let Some(existing) = existing_map.get(&store_id)
2851                && existing.status == TaskStatus::New
2852            {
2853                let db_status = status_to_task_status(status);
2854                if db_status != TaskStatus::New
2855                    && let Err(e) = store.update_status(store_id, db_status).await
2856                {
2857                    tracing::debug!(?e, "ingest: status sync failed");
2858                }
2859            }
2860        }
2861    }
2862
2863    // Batch-fetch estimate values from GitHub Projects for newly ingested tasks.
2864    // For each task where `tasks.estimate == 0`, populate it from the project board.
2865    if !new_task_node_ids.is_empty() {
2866        let node_ids: Vec<String> = new_task_node_ids.iter().map(|(_, n)| n.clone()).collect();
2867        match backend.get_project_item_estimates(&node_ids).await {
2868            Ok(estimates) => {
2869                for (store_id, node_id) in new_task_node_ids {
2870                    if let Some(&estimate) = estimates.get(&node_id)
2871                        && estimate > 0
2872                        && let Err(e) = store.set_estimate_if_zero(store_id, estimate).await
2873                    {
2874                        tracing::debug!(
2875                            store_id,
2876                            estimate,
2877                            err = %e,
2878                            "ingest: failed to set estimate from project"
2879                        );
2880                    }
2881                }
2882            }
2883            Err(e) => {
2884                tracing::debug!(
2885                    err = %e,
2886                    "ingest: project estimate fetch failed — skipping estimate sync this tick"
2887                );
2888            }
2889        }
2890    }
2891
2892    Ok(())
2893}
2894
2895#[cfg(test)]
2896mod tests {
2897    use super::*;
2898    use super::{CommentAction, MentionCursor, classify_comment};
2899    use crate::backends::test_helpers::NoopBackend;
2900    use crate::backends::{ExternalId, ExternalTask, Mention, Status};
2901    use crate::engine::commands::OwnerCommand;
2902    use crate::engine::cooldown::record_persistent_model_failure;
2903    use crate::engine::cooldown::set_agent_cooldown;
2904    use crate::engine::dispatch_guard::DispatchGuard;
2905    use crate::engine::router::Router;
2906    use crate::engine::router::RouterConfig;
2907    use crate::github::types::GitHubIssue;
2908    use crate::github::types::GitHubUser;
2909    use crate::store::NewTask;
2910    use crate::store::{RunTokenUsage, TaskStore};
2911    use async_trait::async_trait;
2912    use serial_test::serial;
2913    use std::slice;
2914    use std::sync::Arc;
2915    use std::sync::atomic::AtomicUsize;
2916    use std::sync::atomic::Ordering;
2917    use tokio::sync::Mutex;
2918    use tokio::sync::oneshot;
2919
2920    // ── ingest_external_tasks tests ─────────────────────────────────────────
2921
2922    /// Mock backend that returns configurable tasks per status.
2923    struct IngestMockBackend {
2924        /// Stored as (status_label, tasks) pairs since Status doesn't impl Hash.
2925        tasks: Vec<(String, Vec<ExternalTask>)>,
2926        dedup_result: bool,
2927        comments: Mutex<Vec<(String, String)>>,
2928        status_updates: Mutex<Vec<(String, Status)>>,
2929        project_syncs: Mutex<Vec<(String, Status)>>,
2930        project_reconciliations: AtomicUsize,
2931        /// Counts `list_active_open_issues` calls.
2932        list_calls: AtomicUsize,
2933    }
2934
2935    impl IngestMockBackend {
2936        fn with_tasks(tasks: Vec<(Status, ExternalTask)>) -> Arc<Self> {
2937            Self::with_tasks_and_dedup(tasks, false)
2938        }
2939
2940        fn with_tasks_and_dedup(
2941            tasks: Vec<(Status, ExternalTask)>,
2942            dedup_result: bool,
2943        ) -> Arc<Self> {
2944            let mut grouped: Vec<(String, Vec<ExternalTask>)> = Vec::new();
2945            for (status, task) in tasks {
2946                let label = status.as_label().to_string();
2947                if let Some(entry) = grouped.iter_mut().find(|(l, _)| l == &label) {
2948                    entry.1.push(task);
2949                } else {
2950                    grouped.push((label, vec![task]));
2951                }
2952            }
2953            Arc::new(Self {
2954                tasks: grouped,
2955                dedup_result,
2956                comments: Mutex::new(Vec::new()),
2957                status_updates: Mutex::new(Vec::new()),
2958                project_syncs: Mutex::new(Vec::new()),
2959                project_reconciliations: AtomicUsize::new(0),
2960                list_calls: AtomicUsize::new(0),
2961            })
2962        }
2963    }
2964
2965    #[async_trait]
2966    impl ExternalBackend for IngestMockBackend {
2967        fn name(&self) -> &'static str {
2968            "ingest-mock"
2969        }
2970        async fn create_task(&self, _: &str, _: &str, _: &[String]) -> anyhow::Result<ExternalId> {
2971            Ok(ExternalId("new".into()))
2972        }
2973        async fn get_task(&self, _: &ExternalId) -> anyhow::Result<ExternalTask> {
2974            anyhow::bail!("not implemented")
2975        }
2976        async fn list_by_status(&self, status: Status) -> anyhow::Result<Vec<ExternalTask>> {
2977            let label = status.as_label().to_string();
2978            Ok(self
2979                .tasks
2980                .iter()
2981                .find(|(l, _)| l == &label)
2982                .map(|(_, t)| t.clone())
2983                .unwrap_or_default())
2984        }
2985        async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
2986            Ok(vec![])
2987        }
2988        async fn list_active_open_issues(
2989            &self,
2990        ) -> anyhow::Result<Vec<(ExternalTask, Option<Status>)>> {
2991            self.list_calls.fetch_add(1, Ordering::Relaxed);
2992            let mut result = Vec::new();
2993            for (label, tasks) in &self.tasks {
2994                let status = match label.as_str() {
2995                    "status:routed" => Some(Status::Routed),
2996                    "status:in_progress" => Some(Status::InProgress),
2997                    "status:needs_review" => Some(Status::NeedsReview),
2998                    "status:in_review" => Some(Status::InReview),
2999                    "status:approved" => Some(Status::Approved),
3000                    "status:blocked" => Some(Status::Blocked),
3001                    _ => None,
3002                };
3003                for task in tasks {
3004                    result.push((task.clone(), status));
3005                }
3006            }
3007            Ok(result)
3008        }
3009        async fn post_comment(&self, id: &ExternalId, body: &str) -> anyhow::Result<()> {
3010            self.comments
3011                .lock()
3012                .await
3013                .push((id.0.clone(), body.to_string()));
3014            Ok(())
3015        }
3016        async fn set_labels(&self, _: &ExternalId, _: &[String]) -> anyhow::Result<()> {
3017            Ok(())
3018        }
3019        async fn remove_label(&self, _: &ExternalId, _: &str) -> anyhow::Result<()> {
3020            Ok(())
3021        }
3022        async fn get_sub_issues(&self, _: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
3023            Ok(vec![])
3024        }
3025        async fn health_check(&self) -> anyhow::Result<()> {
3026            Ok(())
3027        }
3028        async fn get_mentions(&self, _: &str) -> anyhow::Result<Vec<Mention>> {
3029            Ok(vec![])
3030        }
3031        async fn update_status(&self, id: &ExternalId, status: Status) -> anyhow::Result<()> {
3032            self.status_updates
3033                .lock()
3034                .await
3035                .push((id.0.clone(), status));
3036            Ok(())
3037        }
3038
3039        async fn has_open_issue_with_title(&self, _: &str, _: &str) -> anyhow::Result<bool> {
3040            Ok(self.dedup_result)
3041        }
3042
3043        async fn sync_to_project(
3044            &self,
3045            id: &ExternalId,
3046            status: Status,
3047        ) -> anyhow::Result<Option<String>> {
3048            self.project_syncs.lock().await.push((id.0.clone(), status));
3049            Ok(None) // Mock doesn't have real node IDs.
3050        }
3051
3052        async fn reconcile_project_statuses(&self) -> anyhow::Result<()> {
3053            self.project_reconciliations.fetch_add(1, Ordering::Relaxed);
3054            Ok(())
3055        }
3056    }
3057
3058    fn make_ext_task(id: &str, title: &str) -> ExternalTask {
3059        ExternalTask {
3060            id: ExternalId(id.to_string()),
3061            title: title.to_string(),
3062            body: String::new(),
3063            state: "open".to_string(),
3064            labels: vec![],
3065            author: "user".to_string(),
3066            created_at: "2026-01-01T00:00:00Z".to_string(),
3067            updated_at: "2026-01-01T00:00:00Z".to_string(),
3068            url: format!("https://github.com/owner/repo/issues/{id}"),
3069        }
3070    }
3071
3072    #[serial(cooldown_state)]
3073    #[tokio::test]
3074    async fn ingest_upserts_tasks_into_store() {
3075        cooldown::reset_global_state().await;
3076        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![
3077            (Status::New, make_ext_task("1", "First issue")),
3078            (Status::InProgress, make_ext_task("2", "Second issue")),
3079        ]);
3080        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3081
3082        ingest_external_tasks(&backend, "owner/repo", &store)
3083            .await
3084            .unwrap();
3085
3086        // Both tasks should be in the store
3087        let all = store.list_all("owner/repo").await.unwrap();
3088        assert_eq!(all.len(), 2);
3089
3090        let task1 = store
3091            .get_by_external_id("owner/repo", "1")
3092            .await
3093            .unwrap()
3094            .unwrap();
3095        assert_eq!(task1.title, "First issue");
3096        assert_eq!(task1.status, TaskStatus::New);
3097
3098        let task2 = store
3099            .get_by_external_id("owner/repo", "2")
3100            .await
3101            .unwrap()
3102            .unwrap();
3103        assert_eq!(task2.title, "Second issue");
3104        assert_eq!(task2.status, TaskStatus::InProgress);
3105    }
3106
3107    #[serial(cooldown_state)]
3108    #[tokio::test]
3109    async fn ingest_takes_every_open_issue_whatever_its_age_or_the_last_ingest() {
3110        // An issue opened after a restart reaches the store on the first sync, and so does
3111        // one whose updated_at predates any window a `since` cursor would have used. The
3112        // ingest keeps no cursor, so a stale KV value from an older build changes nothing.
3113        cooldown::reset_global_state().await;
3114        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3115        store
3116            .kv_set("issues_last_ingested:owner/repo", "2026-10-26T00:00:00Z")
3117            .await
3118            .unwrap();
3119
3120        let mut old = make_ext_task("73", "Filed weeks ago, never ingested");
3121        old.updated_at = "2026-01-01T00:00:00Z".to_string();
3122        let backend = IngestMockBackend::with_tasks(vec![(Status::New, old)]);
3123        let backend_dyn: Arc<dyn ExternalBackend> = Arc::<IngestMockBackend>::clone(&backend);
3124        ingest_external_tasks(&backend_dyn, "owner/repo", &store)
3125            .await
3126            .unwrap();
3127        assert!(
3128            store
3129                .get_by_external_id("owner/repo", "73")
3130                .await
3131                .unwrap()
3132                .is_some()
3133        );
3134
3135        // A second sync, standing in for the one right after a restart, sees a new issue.
3136        let backend = IngestMockBackend::with_tasks(vec![(
3137            Status::New,
3138            make_ext_task("78", "Opened after the restart"),
3139        )]);
3140        let backend_dyn: Arc<dyn ExternalBackend> = Arc::<IngestMockBackend>::clone(&backend);
3141        ingest_external_tasks(&backend_dyn, "owner/repo", &store)
3142            .await
3143            .unwrap();
3144        assert_eq!(
3145            backend.list_calls.load(Ordering::Relaxed),
3146            1,
3147            "one sync is one listing"
3148        );
3149        assert!(
3150            store
3151                .get_by_external_id("owner/repo", "78")
3152                .await
3153                .unwrap()
3154                .is_some()
3155        );
3156        assert_eq!(
3157            store
3158                .kv_get("issues_last_ingested:owner/repo")
3159                .await
3160                .unwrap()
3161                .as_deref(),
3162            Some("2026-10-26T00:00:00Z"),
3163            "the ingest must keep no cursor, so a stale one is never read or advanced"
3164        );
3165    }
3166
3167    #[serial(cooldown_state)]
3168    #[tokio::test]
3169    async fn ingest_updates_existing_tasks() {
3170        cooldown::reset_global_state().await;
3171        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![(
3172            Status::Routed,
3173            make_ext_task("42", "Updated title"),
3174        )]);
3175        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3176
3177        // Pre-create the task
3178        store
3179            .create(&NewTask {
3180                external_id: Some("42".to_string()),
3181                repo: "owner/repo".to_string(),
3182                origin: "github".to_string(),
3183                title: "Original title".to_string(),
3184                ..Default::default()
3185            })
3186            .await
3187            .unwrap();
3188
3189        ingest_external_tasks(&backend, "owner/repo", &store)
3190            .await
3191            .unwrap();
3192
3193        // Should have updated the title
3194        let task = store
3195            .get_by_external_id("owner/repo", "42")
3196            .await
3197            .unwrap()
3198            .unwrap();
3199        assert_eq!(task.title, "Updated title");
3200        assert_eq!(task.status, TaskStatus::Routed);
3201    }
3202
3203    #[serial(cooldown_state)]
3204    #[tokio::test]
3205    async fn ingest_closes_duplicate_titles() {
3206        cooldown::reset_global_state().await;
3207
3208        let backend = IngestMockBackend::with_tasks_and_dedup(
3209            vec![(Status::New, make_ext_task("2", "Duplicate issue"))],
3210            true,
3211        );
3212        let backend_dyn: Arc<dyn ExternalBackend> = Arc::<IngestMockBackend>::clone(&backend);
3213        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3214
3215        store
3216            .create(&NewTask {
3217                external_id: Some("1".to_string()),
3218                repo: "owner/repo".to_string(),
3219                origin: "github".to_string(),
3220                title: "Duplicate issue".to_string(),
3221                labels: vec!["status:new".to_string()],
3222                ..Default::default()
3223            })
3224            .await
3225            .unwrap();
3226
3227        ingest_external_tasks(&backend_dyn, "owner/repo", &store)
3228            .await
3229            .unwrap();
3230
3231        let all = store.list_all("owner/repo").await.unwrap();
3232        assert_eq!(all.len(), 1);
3233
3234        let comments = backend.comments.lock().await;
3235        assert_eq!(comments.len(), 1);
3236        assert!(comments[0].1.contains("Duplicate of #1"));
3237
3238        let status_updates = backend.status_updates.lock().await;
3239        assert_eq!(status_updates.len(), 1);
3240        assert_eq!(status_updates[0].0, "2");
3241        assert_eq!(status_updates[0].1, Status::Done);
3242    }
3243
3244    #[serial(cooldown_state)]
3245    #[tokio::test]
3246    async fn ingest_handles_empty_backend() {
3247        cooldown::reset_global_state().await;
3248        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
3249        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3250
3251        // Should not error on empty backend
3252        let result = ingest_external_tasks(&backend, "owner/repo", &store).await;
3253        assert!(result.is_ok());
3254
3255        let all = store.list_all("owner/repo").await.unwrap();
3256        assert!(all.is_empty());
3257    }
3258
3259    #[serial(cooldown_state)]
3260    #[tokio::test]
3261    async fn ingest_syncs_status_correctly_across_statuses() {
3262        cooldown::reset_global_state().await;
3263        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![
3264            (Status::New, make_ext_task("1", "New")),
3265            (Status::Routed, make_ext_task("2", "Routed")),
3266            (Status::InProgress, make_ext_task("3", "InProgress")),
3267            (Status::NeedsReview, make_ext_task("4", "NeedsReview")),
3268            (Status::InReview, make_ext_task("5", "InReview")),
3269            (Status::Blocked, make_ext_task("6", "Blocked")),
3270        ]);
3271        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3272
3273        ingest_external_tasks(&backend, "owner/repo", &store)
3274            .await
3275            .unwrap();
3276
3277        let counts = store.status_counts("owner/repo").await.unwrap();
3278        assert_eq!(counts.get("new"), Some(&1));
3279        assert_eq!(counts.get("routed"), Some(&1));
3280        assert_eq!(counts.get("in_progress"), Some(&1));
3281        assert_eq!(counts.get("needs_review"), Some(&1));
3282        assert_eq!(counts.get("in_review"), Some(&1));
3283        assert_eq!(counts.get("blocked"), Some(&1));
3284    }
3285
3286    #[serial(cooldown_state)]
3287    #[tokio::test]
3288    async fn ingest_does_not_overwrite_store_authoritative_status() {
3289        cooldown::reset_global_state().await;
3290        // Backend reports task as New (GitHub labels haven't caught up yet)
3291        let backend: Arc<dyn ExternalBackend> =
3292            IngestMockBackend::with_tasks(vec![(Status::New, make_ext_task("99", "My task"))]);
3293        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3294
3295        // Pre-create the task and advance it to Routed in the store
3296        let id = store
3297            .create(&NewTask {
3298                external_id: Some("99".to_string()),
3299                repo: "owner/repo".to_string(),
3300                origin: "github".to_string(),
3301                title: "My task".to_string(),
3302                ..Default::default()
3303            })
3304            .await
3305            .unwrap();
3306        store.update_status(id, TaskStatus::Routed).await.unwrap();
3307
3308        // Ingest should NOT overwrite Routed back to New
3309        ingest_external_tasks(&backend, "owner/repo", &store)
3310            .await
3311            .unwrap();
3312
3313        let task = store.get(id).await.unwrap();
3314        assert_eq!(
3315            task.status,
3316            TaskStatus::Routed,
3317            "ingest must not overwrite store-authoritative status"
3318        );
3319    }
3320
3321    #[serial(cooldown_state)]
3322    #[tokio::test]
3323    async fn ingest_sync_to_project_called_only_for_new_tasks() {
3324        cooldown::reset_global_state().await;
3325
3326        // Two tasks: "10" is pre-existing, "11" is new.
3327        let backend = IngestMockBackend::with_tasks(vec![
3328            (Status::New, make_ext_task("10", "Existing task")),
3329            (Status::New, make_ext_task("11", "New task")),
3330        ]);
3331        let backend_dyn: Arc<dyn ExternalBackend> = Arc::<IngestMockBackend>::clone(&backend);
3332        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3333
3334        // Pre-insert task "10" so it already exists in the store.
3335        store
3336            .create(&NewTask {
3337                external_id: Some("10".to_string()),
3338                repo: "owner/repo".to_string(),
3339                origin: "github".to_string(),
3340                title: "Existing task".to_string(),
3341                ..Default::default()
3342            })
3343            .await
3344            .unwrap();
3345
3346        ingest_external_tasks(&backend_dyn, "owner/repo", &store)
3347            .await
3348            .unwrap();
3349
3350        let syncs = backend.project_syncs.lock().await;
3351        // Only the new task "11" should trigger sync_to_project — not the pre-existing "10".
3352        assert_eq!(
3353            syncs.len(),
3354            1,
3355            "sync_to_project must be called exactly once (for new task only), got: {syncs:?}"
3356        );
3357        assert_eq!(syncs[0].0, "11", "sync_to_project must target the new task");
3358    }
3359
3360    #[serial(cooldown_state)]
3361    #[tokio::test]
3362    async fn ingest_handles_mixed_labeled_and_unlabeled_open_issues() {
3363        // Simulate a backend where some issues have no status label (routable/unlabeled)
3364        // and others carry active status labels.  Both kinds must be ingested and the
3365        // store status must reflect the label where present.
3366        struct MixedBackend;
3367
3368        #[async_trait]
3369        impl ExternalBackend for MixedBackend {
3370            fn name(&self) -> &'static str {
3371                "mixed"
3372            }
3373            async fn create_task(
3374                &self,
3375                _: &str,
3376                _: &str,
3377                _: &[String],
3378            ) -> anyhow::Result<ExternalId> {
3379                Ok(ExternalId("x".into()))
3380            }
3381            async fn get_task(&self, _: &ExternalId) -> anyhow::Result<ExternalTask> {
3382                anyhow::bail!("not implemented")
3383            }
3384            async fn list_by_status(&self, _: Status) -> anyhow::Result<Vec<ExternalTask>> {
3385                Ok(vec![])
3386            }
3387            /// Return a mix: one unlabeled (no status) + one labeled in_progress.
3388            async fn list_active_open_issues(
3389                &self,
3390            ) -> anyhow::Result<Vec<(ExternalTask, Option<Status>)>> {
3391                Ok(vec![
3392                    (make_ext_task("100", "Unlabeled routable"), None),
3393                    (
3394                        make_ext_task("101", "In progress issue"),
3395                        Some(Status::InProgress),
3396                    ),
3397                ])
3398            }
3399            async fn post_comment(&self, _: &ExternalId, _: &str) -> anyhow::Result<()> {
3400                Ok(())
3401            }
3402            async fn set_labels(&self, _: &ExternalId, _: &[String]) -> anyhow::Result<()> {
3403                Ok(())
3404            }
3405            async fn remove_label(&self, _: &ExternalId, _: &str) -> anyhow::Result<()> {
3406                Ok(())
3407            }
3408            async fn get_sub_issues(&self, _: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
3409                Ok(vec![])
3410            }
3411            async fn health_check(&self) -> anyhow::Result<()> {
3412                Ok(())
3413            }
3414            async fn get_mentions(&self, _: &str) -> anyhow::Result<Vec<Mention>> {
3415                Ok(vec![])
3416            }
3417            async fn update_status(&self, _: &ExternalId, _: Status) -> anyhow::Result<()> {
3418                Ok(())
3419            }
3420            async fn has_open_issue_with_title(&self, _: &str, _: &str) -> anyhow::Result<bool> {
3421                Ok(false)
3422            }
3423            async fn sync_to_project(
3424                &self,
3425                _: &ExternalId,
3426                _: Status,
3427            ) -> anyhow::Result<Option<String>> {
3428                Ok(None)
3429            }
3430        }
3431
3432        cooldown::reset_global_state().await;
3433
3434        let backend: Arc<dyn ExternalBackend> = Arc::new(MixedBackend);
3435        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3436
3437        ingest_external_tasks(&backend, "owner/repo", &store)
3438            .await
3439            .unwrap();
3440
3441        let all = store.list_all("owner/repo").await.unwrap();
3442        assert_eq!(all.len(), 2, "both issues must be ingested");
3443
3444        let unlabeled = store
3445            .get_by_external_id("owner/repo", "100")
3446            .await
3447            .unwrap()
3448            .unwrap();
3449        assert_eq!(
3450            unlabeled.status,
3451            TaskStatus::New,
3452            "unlabeled issue should start as New"
3453        );
3454
3455        let in_progress = store
3456            .get_by_external_id("owner/repo", "101")
3457            .await
3458            .unwrap()
3459            .unwrap();
3460        assert_eq!(
3461            in_progress.status,
3462            TaskStatus::InProgress,
3463            "labeled in_progress issue must be synced to InProgress"
3464        );
3465    }
3466
3467    // ── kv_get_prefer_store / kv_set_prefer_store ────────────────────
3468
3469    #[serial(cooldown_state)]
3470    #[tokio::test]
3471    async fn kv_get_prefer_store_reads_from_store() {
3472        cooldown::reset_global_state().await;
3473        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3474
3475        store.kv_set("k1", "store_val").await.unwrap();
3476
3477        let opt = Some(&store);
3478        let val = kv_get_prefer_store(opt, "k1").await;
3479        assert_eq!(val.as_deref(), Some("store_val"));
3480    }
3481
3482    #[serial(cooldown_state)]
3483    #[tokio::test]
3484    async fn kv_get_prefer_store_returns_none_without_store() {
3485        cooldown::reset_global_state().await;
3486        let opt: Option<&Arc<TaskStore>> = None;
3487        let val = kv_get_prefer_store(opt, "k2").await;
3488        assert_eq!(val, None);
3489    }
3490
3491    #[serial(cooldown_state)]
3492    #[tokio::test]
3493    async fn kv_set_prefer_store_writes_to_store() {
3494        cooldown::reset_global_state().await;
3495        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3496
3497        let opt = Some(&store);
3498        kv_set_prefer_store(opt, "k3", "val3").await;
3499
3500        assert_eq!(store.kv_get("k3").await.unwrap().as_deref(), Some("val3"));
3501    }
3502
3503    #[serial(cooldown_state)]
3504    #[tokio::test]
3505    async fn kv_set_prefer_store_noop_without_store() {
3506        cooldown::reset_global_state().await;
3507        let opt: Option<&Arc<TaskStore>> = None;
3508        // Should not panic
3509        kv_set_prefer_store(opt, "k4", "val4").await;
3510    }
3511
3512    // ── try_kv_get_prefer_store / try_kv_set_prefer_store ────────────────
3513
3514    #[serial(cooldown_state)]
3515    #[tokio::test]
3516    async fn try_kv_get_prefer_store_returns_value_from_store() {
3517        cooldown::reset_global_state().await;
3518        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3519        store.kv_set("try_k1", "try_val1").await.unwrap();
3520
3521        let opt = Some(&store);
3522        let result = try_kv_get_prefer_store(opt, "try_k1").await;
3523        assert_eq!(result.unwrap().as_deref(), Some("try_val1"));
3524    }
3525
3526    #[serial(cooldown_state)]
3527    #[tokio::test]
3528    async fn try_kv_get_prefer_store_returns_none_for_missing_key() {
3529        cooldown::reset_global_state().await;
3530        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3531        let opt = Some(&store);
3532
3533        let result = try_kv_get_prefer_store(opt, "nonexistent_key").await;
3534        assert_eq!(result.unwrap(), None); // Key doesn't exist, but query succeeded
3535    }
3536
3537    #[serial(cooldown_state)]
3538    #[tokio::test]
3539    async fn try_kv_get_prefer_store_returns_none_without_store() {
3540        cooldown::reset_global_state().await;
3541        let opt: Option<&Arc<TaskStore>> = None;
3542        let result = try_kv_get_prefer_store(opt, "try_k2").await;
3543        assert_eq!(result.unwrap(), None);
3544    }
3545
3546    #[serial(cooldown_state)]
3547    #[tokio::test]
3548    async fn try_kv_set_prefer_store_writes_to_store() {
3549        cooldown::reset_global_state().await;
3550        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3551        let opt = Some(&store);
3552
3553        let result = try_kv_set_prefer_store(opt, "try_k3", "try_val3").await;
3554        assert!(result.is_ok());
3555        assert_eq!(
3556            store.kv_get("try_k3").await.unwrap().as_deref(),
3557            Some("try_val3")
3558        );
3559    }
3560
3561    #[serial(cooldown_state)]
3562    #[tokio::test]
3563    async fn try_kv_set_prefer_store_ok_without_store() {
3564        cooldown::reset_global_state().await;
3565        let opt: Option<&Arc<TaskStore>> = None;
3566        // Should not error - best effort when store unavailable
3567        let result = try_kv_set_prefer_store(opt, "try_k4", "try_val4").await;
3568        assert!(result.is_ok());
3569    }
3570
3571    // ── dispatching lock tests ──────────────────────────────────────────
3572
3573    #[serial(cooldown_state)]
3574    #[test]
3575    fn dispatching_set_blocks_duplicate_processing() {
3576        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3577
3578        let repo = "owner/repo";
3579        let task_id = "42";
3580        let dispatch_key = format!("{repo}/{task_id}");
3581
3582        // Initially not in the map
3583        assert!(!dispatching.contains_key(&dispatch_key));
3584
3585        // Insert — simulates dispatch starting
3586        dispatching.insert(dispatch_key.clone(), task_id.to_string());
3587
3588        // Now should be blocked
3589        assert!(
3590            dispatching.contains_key(&dispatch_key),
3591            "task should be locked while dispatching"
3592        );
3593
3594        // Remove — simulates review completion
3595        dispatching.remove(&dispatch_key);
3596
3597        // Now should be free again
3598        assert!(
3599            !dispatching.contains_key(&dispatch_key),
3600            "task should be unlocked after review completes"
3601        );
3602    }
3603
3604    #[serial(cooldown_state)]
3605    #[test]
3606    fn dispatching_set_does_not_block_other_tasks() {
3607        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3608
3609        let repo = "owner/repo";
3610
3611        // Lock task 42
3612        let key_42 = format!("{repo}/42");
3613        dispatching.insert(key_42.clone(), "42".to_string());
3614
3615        // Task 43 should NOT be blocked
3616        let key_43 = format!("{repo}/43");
3617        assert!(
3618            dispatching.contains_key(&key_42),
3619            "task 42 should be locked"
3620        );
3621        assert!(
3622            !dispatching.contains_key(&key_43),
3623            "task 43 should not be locked"
3624        );
3625    }
3626
3627    /// Regression test for a6d8b9a.
3628    ///
3629    /// Before the fix, `sync_tick` step 5 transitioned a task to `InReview` and then
3630    /// spawned a `tokio::spawn` review agent WITHOUT first inserting the `dispatch_key`
3631    /// into the dispatching set. This created a window where `review_open_prs` (step 4)
3632    /// in a concurrent sync tick could see the task as `InReview`, find the key absent,
3633    /// and re-dispatch it — silently discarding human `CHANGES_REQUESTED` feedback when
3634    /// the review agent later completed and marked the task `Done`.
3635    ///
3636    /// The fix: insert `dispatch_key` BEFORE `tokio::spawn`, remove it AFTER the future
3637    /// completes. This test verifies that invariant using tokio channels to synchronize
3638    /// with a simulated concurrent caller (review_open_prs).
3639    #[serial(cooldown_state)]
3640    #[tokio::test]
3641    async fn dispatch_key_held_during_review_agent_execution() {
3642        cooldown::reset_global_state().await;
3643
3644        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3645
3646        let dispatch_key = "owner/repo/42".to_string();
3647
3648        // Step 1: insert before spawning — this is the invariant added by a6d8b9a.
3649        dispatching.insert(dispatch_key.clone(), "42".to_string());
3650
3651        let (agent_started_tx, agent_started_rx) = oneshot::channel::<()>();
3652        let (check_done_tx, check_done_rx) = oneshot::channel::<()>();
3653
3654        let dispatching_agent = Arc::clone(&dispatching);
3655        let key_agent = dispatch_key.clone();
3656
3657        // Step 2: spawn review agent — key is ALREADY in the set (fix invariant holds).
3658        let review_task = tokio::spawn(async move {
3659            agent_started_tx.send(()).unwrap();
3660            // Simulate review agent work — hold until the concurrent check is done.
3661            check_done_rx.await.unwrap();
3662            // Remove from dispatching set on completion (mirror of tick.rs/sync.rs).
3663            dispatching_agent.remove(&key_agent);
3664        });
3665
3666        // Step 3: concurrent review_open_prs check — simulates step 4 of another
3667        // sync_tick invocation running while the review agent is still active.
3668        agent_started_rx.await.unwrap();
3669        assert!(
3670            dispatching.contains_key(&dispatch_key),
3671            "dispatch_key must be visible in the dispatching set while the review \
3672             agent is running — without the a6d8b9a fix, review_open_prs would see \
3673             the key absent and re-dispatch the task, silently dropping \
3674             CHANGES_REQUESTED feedback"
3675        );
3676
3677        // Step 4: review agent completes.
3678        check_done_tx.send(()).unwrap();
3679        review_task.await.unwrap();
3680
3681        // Step 5: key released — review_open_prs can now act on the task if needed.
3682        assert!(
3683            !dispatching.contains_key(&dispatch_key),
3684            "dispatch_key must be removed from the dispatching set after the \
3685             review agent completes so subsequent sync ticks can process the task"
3686        );
3687    }
3688
3689    /// Regression test for issue #833.
3690    ///
3691    /// Before the fix, `dispatch_key` was inserted into the `dispatching` HashSet
3692    /// *outside* the spawned task but removed *inside* it.  When the spawned async
3693    /// block panicked (e.g. an `unwrap()` inside `review_and_merge`), Tokio caught
3694    /// the panic and terminated the task without running the cleanup code, leaving
3695    /// the key in the set permanently.  Subsequent stuck-task recovery would reset
3696    /// the task to `NeedsReview`, but the subscriber/sync would see the key still
3697    /// present and skip it forever — a permanent review loop until service restart.
3698    ///
3699    /// The fix introduces `DispatchGuard`, a RAII wrapper whose `Drop` impl removes
3700    /// the key unconditionally, even when the task unwinds via panic.
3701    #[serial(cooldown_state)]
3702    #[tokio::test]
3703    async fn dispatch_guard_releases_key_on_panic() {
3704        cooldown::reset_global_state().await;
3705
3706        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3707        let key = "owner/repo/42".to_string();
3708
3709        // Insert key before spawn — mirrors the production invariant established
3710        // by the a6d8b9a fix (key must be visible before the spawn so concurrent
3711        // review_open_prs callers see it and skip the task).
3712        dispatching.insert(key.clone(), "42".to_string());
3713
3714        // Create the guard (takes ownership of the removal obligation).
3715        let guard = DispatchGuard::new(Arc::clone(&dispatching), key.clone());
3716
3717        // Spawn a task that panics before the normal completion path.
3718        let handle = tokio::spawn(async move {
3719            let _guard = guard; // moved in; Drop removes key whether we panic or not
3720            panic!("simulated panic inside review_and_merge");
3721        });
3722
3723        // Absorb the JoinError — the panic is expected.
3724        let _ = handle.await;
3725
3726        // Key MUST be gone even though the task panicked.
3727        assert!(
3728            !dispatching.contains_key(&key),
3729            "dispatch key must be removed from the dispatching set even when the \
3730             spawned review task panics — without DispatchGuard the key leaks and \
3731             the task gets stuck in a permanent review loop"
3732        );
3733    }
3734
3735    #[serial(cooldown_state)]
3736    #[tokio::test]
3737    async fn sync_tick_reconciles_board_without_ingesting_or_transitioning_tasks() {
3738        cooldown::reset_global_state().await;
3739        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3740        let mock = IngestMockBackend::with_tasks(vec![]);
3741        let backend: Arc<dyn ExternalBackend> = Arc::<IngestMockBackend>::clone(&mock);
3742        let task_manager = Arc::new(TaskManager::with_store(
3743            Arc::clone(&backend),
3744            Arc::clone(&store),
3745            "owner/repo".to_string(),
3746        ));
3747        let dispatching = Arc::new(DashMap::new());
3748        Box::pin(sync_tick(
3749            &backend,
3750            &Arc::new(TmuxManager::new()),
3751            "owner/repo",
3752            &EngineConfig::default(),
3753            &Arc::new(RwLock::new(Router::new_for_test(
3754                RouterConfig::default(),
3755                vec!["claude".to_string()],
3756            ))),
3757            &task_manager,
3758            &store,
3759            &dispatching,
3760            &Arc::new(DashSet::new()),
3761        ))
3762        .await
3763        .unwrap();
3764        assert_eq!(mock.project_reconciliations.load(Ordering::Relaxed), 1);
3765        assert!(mock.project_syncs.lock().await.is_empty());
3766        assert!(mock.status_updates.lock().await.is_empty());
3767        assert!(dispatching.is_empty());
3768        assert!(
3769            store
3770                .existing_external_ids("owner/repo")
3771                .await
3772                .unwrap()
3773                .is_empty()
3774        );
3775    }
3776
3777    #[serial(cooldown_state)]
3778    #[tokio::test]
3779    async fn needs_review_refire_increments_and_fires_when_old_enough() {
3780        cooldown::reset_global_state().await;
3781        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3782        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
3783        let task_manager = Arc::new(TaskManager::with_store(
3784            backend,
3785            Arc::clone(&store),
3786            "owner/repo".to_string(),
3787        ));
3788        let router = reviewer_router();
3789        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3790
3791        let id = store
3792            .upsert_external(&crate::store::UpsertExternal {
3793                repo: "owner/repo",
3794                ext_id: "201",
3795                title: "Needs review old",
3796                body: "",
3797                author: "",
3798                url: "",
3799                labels: &[],
3800                origin: "github",
3801            })
3802            .await
3803            .unwrap();
3804        store
3805            .update_status(id, TaskStatus::NeedsReview)
3806            .await
3807            .unwrap();
3808        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
3809            .bind(id)
3810            .execute(store.pool())
3811            .await
3812            .unwrap();
3813
3814        let t = store.get(id).await.unwrap();
3815        assert_eq!(t.needs_review_refires, 0);
3816
3817        refire_and_escalate_stale_needs_review_global(
3818            &task_manager,
3819            &store,
3820            &router,
3821            &dispatching,
3822            &sweep_repos(),
3823        )
3824        .await
3825        .unwrap();
3826
3827        let t2 = store.get(id).await.unwrap();
3828        assert_eq!(t2.needs_review_refires, 1);
3829        assert_eq!(t2.status, TaskStatus::NeedsReview);
3830    }
3831
3832    #[serial(cooldown_state)]
3833    #[tokio::test]
3834    async fn needs_review_backoff_delays_fire_but_increments_counter() {
3835        cooldown::reset_global_state().await;
3836        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3837        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
3838        let task_manager = Arc::new(TaskManager::with_store(
3839            backend,
3840            Arc::clone(&store),
3841            "owner/repo".to_string(),
3842        ));
3843        let router = reviewer_router();
3844        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3845
3846        let id = store
3847            .upsert_external(&crate::store::UpsertExternal {
3848                repo: "owner/repo",
3849                ext_id: "202",
3850                title: "Needs review backoff",
3851                body: "",
3852                author: "",
3853                url: "",
3854                labels: &[],
3855                origin: "github",
3856            })
3857            .await
3858            .unwrap();
3859        store
3860            .update_status(id, TaskStatus::NeedsReview)
3861            .await
3862            .unwrap();
3863
3864        store
3865            .set_fields(id, &[("needs_review_refires", serde_json::json!(1))])
3866            .await
3867            .unwrap();
3868
3869        // Age 1 minute: stale enough to consider, but backoff at refires=1 needs 2 minutes
3870        let now_minus_1 = (chrono::Utc::now() - chrono::Duration::minutes(1))
3871            .format("%Y-%m-%dT%H:%M:%SZ")
3872            .to_string();
3873        sqlx::query("UPDATE tasks SET updated_at = ? WHERE id = ?")
3874            .bind(now_minus_1.clone())
3875            .bind(id)
3876            .execute(store.pool())
3877            .await
3878            .unwrap();
3879
3880        let before = store.get(id).await.unwrap();
3881        assert_eq!(before.needs_review_refires, 1);
3882
3883        refire_and_escalate_stale_needs_review_global(
3884            &task_manager,
3885            &store,
3886            &router,
3887            &dispatching,
3888            &sweep_repos(),
3889        )
3890        .await
3891        .unwrap();
3892
3893        let after = store.get(id).await.unwrap();
3894        // Counter must not increment on a backoff delay, only on an actual re-fire
3895        assert_eq!(after.needs_review_refires, 1);
3896        assert_eq!(after.updated_at, now_minus_1);
3897        assert_eq!(after.status, TaskStatus::NeedsReview);
3898    }
3899
3900    /// Router with one claude review model, so the refire sweeps see a reviewer.
3901    fn reviewer_router() -> Router {
3902        let mut config = RouterConfig::default();
3903        config.model_map.insert(
3904            "review".to_string(),
3905            HashMap::from([("claude".to_string(), vec!["sonnet".to_string()])]),
3906        );
3907        Router::new_for_test(config, vec!["claude".to_string()])
3908    }
3909
3910    #[serial(cooldown_state)]
3911    #[tokio::test]
3912    async fn needs_review_escalates_to_blocked_after_max_refires() {
3913        cooldown::reset_global_state().await;
3914        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3915        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
3916        let task_manager = Arc::new(TaskManager::with_store(
3917            backend,
3918            Arc::clone(&store),
3919            "owner/repo".to_string(),
3920        ));
3921        let router = reviewer_router();
3922        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3923
3924        let id = store
3925            .upsert_external(&crate::store::UpsertExternal {
3926                repo: "owner/repo",
3927                ext_id: "203",
3928                title: "Needs review escalate",
3929                body: "",
3930                author: "",
3931                url: "",
3932                labels: &[],
3933                origin: "github",
3934            })
3935            .await
3936            .unwrap();
3937        store
3938            .update_status(id, TaskStatus::NeedsReview)
3939            .await
3940            .unwrap();
3941
3942        // Counter at MAX (5): the next pass is attempt 6 and escalates
3943        store
3944            .set_fields(id, &[("needs_review_refires", serde_json::json!(5))])
3945            .await
3946            .unwrap();
3947        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
3948            .bind(id)
3949            .execute(store.pool())
3950            .await
3951            .unwrap();
3952
3953        refire_and_escalate_stale_needs_review_global(
3954            &task_manager,
3955            &store,
3956            &router,
3957            &dispatching,
3958            &sweep_repos(),
3959        )
3960        .await
3961        .unwrap();
3962
3963        let after = store.get(id).await.unwrap();
3964        assert_eq!(after.status, TaskStatus::Blocked);
3965        assert!(after.block_reason.is_some());
3966    }
3967
3968    /// Exactly MAX_NEEDS_REVIEW_REFIRE_ATTEMPTS refires are allowed before
3969    /// escalation. Task stays NeedsReview at 5 refires and only blocks on the 6th
3970    #[serial(cooldown_state)]
3971    #[tokio::test]
3972    async fn needs_review_allows_max_refires_before_escalation() {
3973        cooldown::reset_global_state().await;
3974        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3975        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
3976        let task_manager = Arc::new(TaskManager::with_store(
3977            backend,
3978            Arc::clone(&store),
3979            "owner/repo".to_string(),
3980        ));
3981        let router = reviewer_router();
3982        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3983
3984        let id = store
3985            .upsert_external(&crate::store::UpsertExternal {
3986                repo: "owner/repo",
3987                ext_id: "204",
3988                title: "Max refires test",
3989                body: "",
3990                author: "",
3991                url: "",
3992                labels: &[],
3993                origin: "github",
3994            })
3995            .await
3996            .unwrap();
3997        store
3998            .update_status(id, TaskStatus::NeedsReview)
3999            .await
4000            .unwrap();
4001
4002        // Counter at MAX-1 (4): incrementing hits MAX (5) and must not escalate
4003        store
4004            .set_fields(id, &[("needs_review_refires", serde_json::json!(4))])
4005            .await
4006            .unwrap();
4007        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4008            .bind(id)
4009            .execute(store.pool())
4010            .await
4011            .unwrap();
4012
4013        refire_and_escalate_stale_needs_review_global(
4014            &task_manager,
4015            &store,
4016            &router,
4017            &dispatching,
4018            &sweep_repos(),
4019        )
4020        .await
4021        .unwrap();
4022
4023        let after = store.get(id).await.unwrap();
4024        assert_eq!(after.status, TaskStatus::NeedsReview);
4025        assert_eq!(after.needs_review_refires, 5);
4026        assert!(after.block_reason.is_none());
4027    }
4028
4029    /// Every repo the sweep tests use, so each counts as active
4030    fn sweep_repos() -> HashSet<String> {
4031        ["owner/repo", "inactive/repo", "some/repo", "active/repo"]
4032            .into_iter()
4033            .map(String::from)
4034            .collect()
4035    }
4036
4037    /// A repo outside the active set has no review subscriber (#4024)
4038    #[serial(cooldown_state)]
4039    #[tokio::test]
4040    async fn global_sweeps_leave_tasks_of_a_removed_repo_alone() {
4041        cooldown::reset_global_state().await;
4042        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4043        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4044        let task_manager = Arc::new(TaskManager::with_store(
4045            backend,
4046            Arc::clone(&store),
4047            "active/repo".to_string(),
4048        ));
4049        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4050        let router = router_with_healthy_review_agent();
4051        let active: HashSet<String> = ["active/repo".to_string()].into_iter().collect();
4052
4053        let review_id = store
4054            .create_internal("removed/repo", "Needs review", "", "cron", "1", None)
4055            .await
4056            .unwrap();
4057        store
4058            .update_status(review_id, TaskStatus::NeedsReview)
4059            .await
4060            .unwrap();
4061        let blocked_id = store
4062            .create_internal("removed/repo", "Rebroadcast blocked", "", "cron", "2", None)
4063            .await
4064            .unwrap();
4065        store
4066            .update_status(blocked_id, TaskStatus::Blocked)
4067            .await
4068            .unwrap();
4069        store
4070            .set_block_reason(blocked_id, Some(REBROADCAST_BLOCK_REASON))
4071            .await
4072            .unwrap();
4073        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z'")
4074            .execute(store.pool())
4075            .await
4076            .unwrap();
4077
4078        refire_and_escalate_stale_needs_review_global(
4079            &task_manager,
4080            &store,
4081            &router,
4082            &dispatching,
4083            &active,
4084        )
4085        .await
4086        .unwrap();
4087        auto_recover_rebroadcast_blocked_tasks_global(&router, &task_manager, &store, &active)
4088            .await
4089            .unwrap();
4090
4091        let review = store.get(review_id).await.unwrap();
4092        assert_eq!(review.status, TaskStatus::NeedsReview);
4093        assert_eq!(review.needs_review_refires, 0);
4094        let blocked = store.get(blocked_id).await.unwrap();
4095        assert_eq!(blocked.status, TaskStatus::Blocked);
4096        assert_eq!(
4097            blocked.block_reason.as_deref(),
4098            Some(REBROADCAST_BLOCK_REASON)
4099        );
4100    }
4101
4102    // ── refire_and_escalate_stale_needs_review_global ──────────────────────
4103
4104    fn router_with_healthy_review_agent() -> Router {
4105        let mut config = RouterConfig::default();
4106        config
4107            .model_map
4108            .entry("review".to_string())
4109            .or_default()
4110            .insert("claude".to_string(), vec!["sonnet".to_string()]);
4111        let mut router = Router::new(config);
4112        router.available_agents = vec!["claude".to_string()];
4113        router
4114    }
4115
4116    /// Regression test for #3469: a task stuck in NeedsReview for a repo that
4117    /// is not the TaskManager's own repo must still be refired. The sweep
4118    /// queries via `list_all_by_status_global` and writes through the store
4119    /// directly, bypassing the repo-scoped TaskManager
4120    #[serial(cooldown_state)]
4121    #[tokio::test]
4122    async fn global_refire_sweep_refires_stale_task_from_inactive_repo() {
4123        cooldown::reset_global_state().await;
4124        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4125        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4126        // TaskManager scoped to the active repo — NOT the repo the task belongs to.
4127        let task_manager = Arc::new(TaskManager::with_store(
4128            backend,
4129            Arc::clone(&store),
4130            "active/repo".to_string(),
4131        ));
4132        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4133        let router = router_with_healthy_review_agent();
4134
4135        let id = store
4136            .upsert_external(&crate::store::UpsertExternal {
4137                repo: "inactive/repo",
4138                ext_id: "490",
4139                title: "Stale needs_review from inactive repo",
4140                body: "",
4141                author: "",
4142                url: "",
4143                labels: &[],
4144                origin: "github",
4145            })
4146            .await
4147            .unwrap();
4148        store
4149            .update_status(id, TaskStatus::NeedsReview)
4150            .await
4151            .unwrap();
4152        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4153            .bind(id)
4154            .execute(store.pool())
4155            .await
4156            .unwrap();
4157
4158        refire_and_escalate_stale_needs_review_global(
4159            &task_manager,
4160            &store,
4161            &router,
4162            &dispatching,
4163            &sweep_repos(),
4164        )
4165        .await
4166        .unwrap();
4167
4168        let task = store.get(id).await.unwrap();
4169        assert_eq!(
4170            task.needs_review_refires, 1,
4171            "cross-repo task must be refired"
4172        );
4173        assert_eq!(task.status, TaskStatus::NeedsReview);
4174    }
4175
4176    /// After `MAX_NEEDS_REVIEW_REFIRE_ATTEMPTS` refires, a cross-repo task must
4177    /// escalate to Blocked
4178    #[serial(cooldown_state)]
4179    #[tokio::test]
4180    async fn global_refire_sweep_escalates_cross_repo_task_to_blocked() {
4181        cooldown::reset_global_state().await;
4182        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4183        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4184        let task_manager = Arc::new(TaskManager::with_store(
4185            backend,
4186            Arc::clone(&store),
4187            "active/repo".to_string(),
4188        ));
4189        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4190        let router = router_with_healthy_review_agent();
4191
4192        let id = store
4193            .upsert_external(&crate::store::UpsertExternal {
4194                repo: "inactive/repo",
4195                ext_id: "493",
4196                title: "Exhausted refires from inactive repo",
4197                body: "",
4198                author: "",
4199                url: "",
4200                labels: &[],
4201                origin: "github",
4202            })
4203            .await
4204            .unwrap();
4205        store
4206            .update_status(id, TaskStatus::NeedsReview)
4207            .await
4208            .unwrap();
4209        store
4210            .set_fields(id, &[("needs_review_refires", serde_json::json!(5))])
4211            .await
4212            .unwrap();
4213        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4214            .bind(id)
4215            .execute(store.pool())
4216            .await
4217            .unwrap();
4218
4219        refire_and_escalate_stale_needs_review_global(
4220            &task_manager,
4221            &store,
4222            &router,
4223            &dispatching,
4224            &sweep_repos(),
4225        )
4226        .await
4227        .unwrap();
4228
4229        let task = store.get(id).await.unwrap();
4230        assert_eq!(task.status, TaskStatus::Blocked);
4231        assert_eq!(
4232            task.block_reason.as_deref(),
4233            Some("review agent rebroadcast escalated after repeated retries")
4234        );
4235    }
4236
4237    /// When every review agent is cooled, the sweep must skip entirely.
4238    /// Re-firing while all agents are cooled would waste the refire counter and
4239    /// could escalate a task to Blocked because agents happened to be
4240    /// temporarily unavailable
4241    #[serial(cooldown_state)]
4242    #[tokio::test]
4243    async fn global_refire_sweep_skips_when_all_review_agents_cooled() {
4244        cooldown::reset_global_state().await;
4245        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4246        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4247        let task_manager = Arc::new(TaskManager::with_store(
4248            backend,
4249            Arc::clone(&store),
4250            "active/repo".to_string(),
4251        ));
4252        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4253        // RouterConfig::default() has no "review" model map entries, so the one
4254        // available agent has no healthy model for "review" — all cooled.
4255        let router = Router::new_for_test(RouterConfig::default(), vec!["claude".to_string()]);
4256
4257        let id = store
4258            .upsert_external(&crate::store::UpsertExternal {
4259                repo: "inactive/repo",
4260                ext_id: "494",
4261                title: "Stale needs_review while agents cooled",
4262                body: "",
4263                author: "",
4264                url: "",
4265                labels: &[],
4266                origin: "github",
4267            })
4268            .await
4269            .unwrap();
4270        store
4271            .update_status(id, TaskStatus::NeedsReview)
4272            .await
4273            .unwrap();
4274        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4275            .bind(id)
4276            .execute(store.pool())
4277            .await
4278            .unwrap();
4279
4280        refire_and_escalate_stale_needs_review_global(
4281            &task_manager,
4282            &store,
4283            &router,
4284            &dispatching,
4285            &sweep_repos(),
4286        )
4287        .await
4288        .unwrap();
4289
4290        let task = store.get(id).await.unwrap();
4291        assert_eq!(
4292            task.needs_review_refires, 0,
4293            "counter must not be touched while cooled"
4294        );
4295        assert_eq!(task.status, TaskStatus::NeedsReview);
4296    }
4297
4298    /// With no available review agent at all, the sweep must not count re-fires
4299    /// or escalate to Blocked.
4300    #[serial(cooldown_state)]
4301    #[tokio::test]
4302    async fn global_refire_sweep_skips_when_no_reviewer_available() {
4303        cooldown::reset_global_state().await;
4304        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4305        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4306        let task_manager = Arc::new(TaskManager::with_store(
4307            backend,
4308            Arc::clone(&store),
4309            "active/repo".to_string(),
4310        ));
4311        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4312        let router = Router::new_for_test(RouterConfig::default(), vec![]);
4313
4314        let id = store
4315            .upsert_external(&crate::store::UpsertExternal {
4316                repo: "some/repo",
4317                ext_id: "495",
4318                title: "needs_review with no reviewer",
4319                body: "",
4320                author: "",
4321                url: "",
4322                labels: &[],
4323                origin: "github",
4324            })
4325            .await
4326            .unwrap();
4327        store
4328            .update_status(id, TaskStatus::NeedsReview)
4329            .await
4330            .unwrap();
4331        store
4332            .set_fields(id, &[("needs_review_refires", serde_json::json!(5))])
4333            .await
4334            .unwrap();
4335        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4336            .bind(id)
4337            .execute(store.pool())
4338            .await
4339            .unwrap();
4340
4341        refire_and_escalate_stale_needs_review_global(
4342            &task_manager,
4343            &store,
4344            &router,
4345            &dispatching,
4346            &sweep_repos(),
4347        )
4348        .await
4349        .unwrap();
4350
4351        let task = store.get(id).await.unwrap();
4352        assert_eq!(task.status, TaskStatus::NeedsReview);
4353        assert_eq!(task.needs_review_refires, 5);
4354    }
4355
4356    /// Regression test for #3499: a task blocked with the rebroadcast-escalation
4357    /// reason after surviving one review round (`review_cycles > 0`) must be
4358    /// recovered back to NeedsReview once review agents are routable again.
4359    /// Previously the `review_cycles != 0` filter excluded these tasks, stranding
4360    /// them permanently even when agents recovered.
4361    #[serial(cooldown_state)]
4362    #[tokio::test]
4363    async fn auto_recover_rebroadcast_blocked_task_after_review_cycle() {
4364        cooldown::reset_global_state().await;
4365        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4366        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4367        let task_manager = Arc::new(TaskManager::with_store(
4368            backend,
4369            Arc::clone(&store),
4370            "owner/repo".to_string(),
4371        ));
4372        let router = router_with_healthy_review_agent();
4373
4374        let id = store
4375            .upsert_external(&crate::store::UpsertExternal {
4376                repo: "owner/repo",
4377                ext_id: "511",
4378                title: "Rebroadcast blocked after review",
4379                body: "",
4380                author: "",
4381                url: "",
4382                labels: &[],
4383                origin: "github",
4384            })
4385            .await
4386            .unwrap();
4387        store.update_status(id, TaskStatus::Blocked).await.unwrap();
4388        store
4389            .set_block_reason(
4390                id,
4391                Some("review agent rebroadcast escalated after repeated retries"),
4392            )
4393            .await
4394            .unwrap();
4395        store
4396            .set_fields(id, &[("review_cycles", serde_json::json!(1))])
4397            .await
4398            .unwrap();
4399        store
4400            .set_fields(id, &[("needs_review_refires", serde_json::json!(6))])
4401            .await
4402            .unwrap();
4403        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4404            .bind(id)
4405            .execute(store.pool())
4406            .await
4407            .unwrap();
4408
4409        auto_recover_rebroadcast_blocked_tasks_global(
4410            &router,
4411            &task_manager,
4412            &store,
4413            &sweep_repos(),
4414        )
4415        .await
4416        .unwrap();
4417
4418        let task = store.get(id).await.unwrap();
4419        assert_eq!(
4420            task.status,
4421            TaskStatus::NeedsReview,
4422            "blocked rebroadcast task must be recovered to needs_review"
4423        );
4424        assert!(
4425            task.block_reason.is_none(),
4426            "block_reason must be cleared on recovery"
4427        );
4428        assert_eq!(
4429            task.needs_review_refires, 0,
4430            "needs_review_refires must be reset to 0 on recovery"
4431        );
4432    }
4433
4434    /// A fresh InReview task (<1 min old) should NOT be reset — the review
4435    /// agent may still be starting up.
4436    #[serial(cooldown_state)]
4437    #[tokio::test]
4438    async fn stale_in_review_recovery_skips_fresh_tasks() {
4439        cooldown::reset_global_state().await;
4440        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4441        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4442        let task_manager = Arc::new(TaskManager::with_store(
4443            Arc::clone(&backend),
4444            Arc::clone(&store),
4445            "owner/repo".to_string(),
4446        ));
4447        let tmux = Arc::new(TmuxManager::new());
4448        let router = Arc::new(RwLock::new(Router::new_for_test(
4449            RouterConfig::default(),
4450            vec!["claude".to_string()],
4451        )));
4452        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4453        let auto_merge_in_flight: Arc<DashSet<String>> = Arc::new(DashSet::new());
4454
4455        let id = store
4456            .upsert_external(&crate::store::UpsertExternal {
4457                repo: "owner/repo",
4458                ext_id: "66",
4459                title: "Fresh review",
4460                body: "",
4461                author: "",
4462                url: "",
4463                labels: &[],
4464                origin: "github",
4465            })
4466            .await
4467            .unwrap();
4468        store.update_status(id, TaskStatus::InReview).await.unwrap();
4469        store
4470            .set_fields(id, &[("branch", serde_json::json!("feature/66"))])
4471            .await
4472            .unwrap();
4473        // Don't backdate — task is fresh (just created)
4474
4475        Box::pin(sync_tick(
4476            &backend,
4477            &tmux,
4478            "owner/repo",
4479            &EngineConfig::default(),
4480            &router,
4481            &task_manager,
4482            &store,
4483            &dispatching,
4484            &auto_merge_in_flight,
4485        ))
4486        .await
4487        .unwrap();
4488
4489        // Should stay InReview — too young to be considered orphaned
4490        let task = store.get(id).await.unwrap();
4491        assert_eq!(task.status, TaskStatus::InReview);
4492    }
4493
4494    #[serial(cooldown_state)]
4495    #[tokio::test]
4496    async fn auto_unblock_routes_recoverable_failure() {
4497        cooldown::reset_global_state().await;
4498        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4499        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4500        let task_manager = Arc::new(TaskManager::with_store(
4501            backend,
4502            Arc::clone(&store),
4503            "owner/repo".to_string(),
4504        ));
4505        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4506
4507        let id = store
4508            .upsert_external(&crate::store::UpsertExternal {
4509                repo: "owner/repo",
4510                ext_id: "77",
4511                title: "Blocked task",
4512                body: "",
4513                author: "",
4514                url: "",
4515                labels: &[],
4516                origin: "github",
4517            })
4518            .await
4519            .unwrap();
4520        store.update_status(id, TaskStatus::Blocked).await.unwrap();
4521        store
4522            .set_fields(
4523                id,
4524                &[
4525                    ("agent", serde_json::json!("codex")),
4526                    ("model", serde_json::json!("gpt-5")),
4527                ],
4528            )
4529            .await
4530            .unwrap();
4531
4532        let run_id = store
4533            .start_run(&crate::store::StartRun {
4534                task_id: id,
4535                attempt: 1,
4536                run_type: "agent",
4537                agent: "codex",
4538                model: "gpt-5",
4539                command: "codex --model gpt-5",
4540                prompt: "do thing",
4541            })
4542            .await
4543            .unwrap();
4544        store
4545            .complete_run(&crate::store::CompleteRun {
4546                run_id,
4547                exit_code: Some(1),
4548                stdout: "",
4549                stderr: "",
4550                parsed: "",
4551                outcome: "rate_limit",
4552                error: "rate limit exceeded",
4553                tokens: RunTokenUsage::default(),
4554            })
4555            .await
4556            .unwrap();
4557
4558        auto_unblock_blocked_tasks(
4559            "owner/repo",
4560            &task_manager,
4561            &store,
4562            &dispatching,
4563            &Arc::new(DashSet::new()),
4564        )
4565        .await
4566        .unwrap();
4567
4568        let task = store.get(id).await.unwrap();
4569        assert_eq!(task.status, TaskStatus::New);
4570        assert!(task.agent.is_none());
4571        assert!(task.model.is_none());
4572        assert_eq!(task.auto_unblock_count, 1);
4573        assert_ne!(task.auto_unblock_last_at, "");
4574        assert_eq!(task.auto_unblock_last_reason, "RateLimit");
4575    }
4576
4577    #[serial(cooldown_state)]
4578    #[tokio::test]
4579    async fn auto_unblock_skips_manual_block() {
4580        cooldown::reset_global_state().await;
4581        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4582        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4583        let task_manager = Arc::new(TaskManager::with_store(
4584            backend,
4585            Arc::clone(&store),
4586            "owner/repo".to_string(),
4587        ));
4588        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4589
4590        let id = store
4591            .upsert_external(&crate::store::UpsertExternal {
4592                repo: "owner/repo",
4593                ext_id: "88",
4594                title: "Manually blocked",
4595                body: "",
4596                author: "",
4597                url: "",
4598                labels: &[],
4599                origin: "github",
4600            })
4601            .await
4602            .unwrap();
4603        store.update_status(id, TaskStatus::Blocked).await.unwrap();
4604        store
4605            .set_block_reason(id, Some("waiting on input"))
4606            .await
4607            .unwrap();
4608
4609        let run_id = store
4610            .start_run(&crate::store::StartRun {
4611                task_id: id,
4612                attempt: 1,
4613                run_type: "agent",
4614                agent: "codex",
4615                model: "gpt-5",
4616                command: "codex --model gpt-5",
4617                prompt: "do thing",
4618            })
4619            .await
4620            .unwrap();
4621        store
4622            .complete_run(&crate::store::CompleteRun {
4623                run_id,
4624                exit_code: Some(1),
4625                stdout: "",
4626                stderr: "",
4627                parsed: "",
4628                outcome: "rate_limit",
4629                error: "rate limit exceeded",
4630                tokens: RunTokenUsage::default(),
4631            })
4632            .await
4633            .unwrap();
4634
4635        auto_unblock_blocked_tasks(
4636            "owner/repo",
4637            &task_manager,
4638            &store,
4639            &dispatching,
4640            &Arc::new(DashSet::new()),
4641        )
4642        .await
4643        .unwrap();
4644
4645        let task = store.get(id).await.unwrap();
4646        assert_eq!(task.status, TaskStatus::Blocked);
4647        assert_eq!(task.auto_unblock_count, 0);
4648    }
4649
4650    // ── FailureCategory::as_str ─────────────────────────────────────────
4651
4652    #[serial(cooldown_state)]
4653    #[test]
4654    fn failure_category_as_str() {
4655        // as_str() must return a stable, human-readable string used as the
4656        // reason key in auto_unblock tracking (stored in auto_unblock_last_reason).
4657        assert_eq!(FailureCategory::RateLimit.as_str(), "RateLimit");
4658        assert_eq!(FailureCategory::Timeout.as_str(), "Timeout");
4659        assert_eq!(
4660            FailureCategory::ModelUnavailable.as_str(),
4661            "ModelUnavailable"
4662        );
4663        assert_eq!(FailureCategory::MaxAttempts.as_str(), "MaxAttempts");
4664        assert_eq!(FailureCategory::Unknown.as_str(), "Unknown");
4665        assert_eq!(FailureCategory::SilentExit0.as_str(), "SilentExit0");
4666        assert_eq!(FailureCategory::ConnectionError.as_str(), "ConnectionError");
4667        assert_eq!(FailureCategory::CliFlagError.as_str(), "CliFlagError");
4668        assert_eq!(
4669            FailureCategory::AllAgentsExhausted.as_str(),
4670            "AllAgentsExhausted"
4671        );
4672        assert_eq!(FailureCategory::ParseError.as_str(), "ParseError");
4673        assert_eq!(FailureCategory::FalseFailure.as_str(), "FalseFailure");
4674        assert_eq!(FailureCategory::PushFailed.as_str(), "PushFailed");
4675        assert_eq!(FailureCategory::PrCreateFailed.as_str(), "PrCreateFailed");
4676    }
4677
4678    // ── classify_failure: ModelUnavailable ──────────────────────────────
4679
4680    #[serial(cooldown_state)]
4681    #[test]
4682    fn classify_failure_model_unavailable_phrase_is_recoverable() {
4683        let category = classify_failure(
4684            "model unavailable (anthropic/claude-sonnet-4-6): Model not found: anthropic/claude-sonnet-4-6",
4685            "error",
4686        );
4687        assert_eq!(
4688            category,
4689            FailureCategory::ModelUnavailable,
4690            "\"model unavailable\" error must classify as ModelUnavailable"
4691        );
4692        assert!(
4693            category.is_recoverable(),
4694            "ModelUnavailable must be recoverable so auto-unblock can re-route"
4695        );
4696    }
4697
4698    // ── classify_failure: CreditExhausted (billing cycle) ──────────────
4699    #[serial(cooldown_state)]
4700    #[test]
4701    fn classify_failure_kimi_billing_cycle_is_credit_exhausted_not_rate_limit() {
4702        // Regression test for issue #3529: Kimi's billing-cycle exhaustion message
4703        // contains "usage limit", which also matches the generic rate-limit content
4704        // check. Credit exhaustion must be checked first so the failure is
4705        // categorized as CreditExhausted, not RateLimit, even when the stored
4706        // outcome is still "rate_limit" (e.g. from a task_run recorded before this
4707        // fix landed).
4708        let error = "kimi rate limit: Failed to authenticate. API Error: 403 You've reached \
4709                      your usage limit for this billing cycle. Your quota will be refreshed soon.";
4710        let category = classify_failure(error, "rate_limit");
4711        assert_eq!(
4712            category,
4713            FailureCategory::CreditExhausted,
4714            "billing-cycle exhaustion must classify as CreditExhausted, not RateLimit"
4715        );
4716        assert!(category.is_recoverable());
4717    }
4718
4719    #[serial(cooldown_state)]
4720    #[test]
4721    fn classify_failure_billing_cycle_exhausted_outcome_is_credit_exhausted() {
4722        // New task_runs store the specific "billing_cycle_exhausted" outcome
4723        // (post-fix). classify_failure must recognize it directly too.
4724        let category = classify_failure("some billing message", "billing_cycle_exhausted");
4725        assert_eq!(category, FailureCategory::CreditExhausted);
4726    }
4727
4728    #[serial(cooldown_state)]
4729    #[test]
4730    fn classify_failure_generic_rate_limit_is_still_rate_limit() {
4731        // Genuine rate limits (no billing-cycle/credit-exhaustion markers) must
4732        // remain RateLimit, not be swallowed by the reordered credit-exhaustion check.
4733        let category = classify_failure("429 Too Many Requests", "rate_limit");
4734        assert_eq!(category, FailureCategory::RateLimit);
4735    }
4736
4737    // ── classify_failure: PrCreateFailed ──────────────────────────────
4738    #[serial(cooldown_state)]
4739    #[test]
4740    fn classify_failure_pr_create_failed_specific_patterns() {
4741        // These should be classified as PrCreateFailed
4742        assert_eq!(
4743            classify_failure("create pr failed", ""),
4744            FailureCategory::PrCreateFailed
4745        );
4746        assert_eq!(
4747            classify_failure("failed to create pull request", ""),
4748            FailureCategory::PrCreateFailed
4749        );
4750        assert_eq!(
4751            classify_failure("pull request creation failed", ""),
4752            FailureCategory::PrCreateFailed
4753        );
4754
4755        // These should NOT be classified as PrCreateFailed (they were before the fix)
4756        assert_ne!(
4757            classify_failure("network failed while fetching pull request status", ""),
4758            FailureCategory::PrCreateFailed
4759        );
4760        assert_ne!(
4761            classify_failure("connection failed, retrying pull request check", ""),
4762            FailureCategory::PrCreateFailed
4763        );
4764        assert_ne!(
4765            classify_failure("pull request created but failed to merge", ""),
4766            FailureCategory::PrCreateFailed
4767        );
4768    }
4769
4770    #[serial(cooldown_state)]
4771    #[test]
4772    fn classify_failure_model_not_found_is_recoverable() {
4773        let category = classify_failure("Model not found: gpt-5-ultra", "error");
4774        assert_eq!(
4775            category,
4776            FailureCategory::ModelUnavailable,
4777            "\"model not found\" error must classify as ModelUnavailable"
4778        );
4779        assert!(category.is_recoverable());
4780    }
4781
4782    #[serial(cooldown_state)]
4783    #[test]
4784    fn classify_failure_model_does_not_exist_is_recoverable() {
4785        let category = classify_failure("The model `claude-opus-99` does not exist", "error");
4786        assert_eq!(category, FailureCategory::ModelUnavailable);
4787        assert!(category.is_recoverable());
4788    }
4789
4790    #[serial(cooldown_state)]
4791    #[tokio::test]
4792    async fn auto_unblock_routes_task_blocked_by_model_unavailable() {
4793        cooldown::reset_global_state().await;
4794        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4795        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4796        let task_manager = Arc::new(TaskManager::with_store(
4797            backend,
4798            Arc::clone(&store),
4799            "owner/repo".to_string(),
4800        ));
4801        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4802
4803        let id = store
4804            .upsert_external(&crate::store::UpsertExternal {
4805                repo: "owner/repo",
4806                ext_id: "1202",
4807                title: "Task blocked by model unavailable",
4808                body: "",
4809                author: "",
4810                url: "",
4811                labels: &[],
4812                origin: "github",
4813            })
4814            .await
4815            .unwrap();
4816        store.update_status(id, TaskStatus::Blocked).await.unwrap();
4817
4818        let run_id = store
4819            .start_run(&crate::store::StartRun {
4820                task_id: id,
4821                attempt: 1,
4822                run_type: "agent",
4823                agent: "opencode",
4824                model: "anthropic/claude-sonnet-4-6",
4825                command: "opencode",
4826                prompt: "fix the bug",
4827            })
4828            .await
4829            .unwrap();
4830        store
4831            .complete_run(&crate::store::CompleteRun {
4832                run_id,
4833                exit_code: Some(1),
4834                stdout: "",
4835                stderr: "",
4836                parsed: "",
4837                outcome: "error",
4838                error: "model unavailable (anthropic/claude-sonnet-4-6): Model not found: anthropic/claude-sonnet-4-6",
4839                tokens: RunTokenUsage::default(),
4840            })
4841            .await
4842            .unwrap();
4843
4844        auto_unblock_blocked_tasks(
4845            "owner/repo",
4846            &task_manager,
4847            &store,
4848            &dispatching,
4849            &Arc::new(DashSet::new()),
4850        )
4851        .await
4852        .unwrap();
4853
4854        let task = store.get(id).await.unwrap();
4855        assert_eq!(
4856            task.status,
4857            TaskStatus::New,
4858            "task blocked by model unavailable must be auto-unblocked and re-routed"
4859        );
4860        assert_eq!(task.auto_unblock_count, 1);
4861        assert_eq!(task.auto_unblock_last_reason, "ModelUnavailable");
4862    }
4863
4864    // ── auto_unblock: reason-reset (regression for #1227) ─────────────────
4865
4866    #[serial(cooldown_state)]
4867    #[tokio::test]
4868    async fn auto_unblock_resets_count_when_reason_changes() {
4869        cooldown::reset_global_state().await;
4870        // Regression test for #1227: when a task is blocked for a different failure
4871        // reason than the last auto-unblock, the counter should reset to 0 (immediate
4872        // retry) instead of accumulating across unrelated failures.
4873        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4874        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4875        let task_manager = Arc::new(TaskManager::with_store(
4876            backend,
4877            Arc::clone(&store),
4878            "owner/repo".to_string(),
4879        ));
4880        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4881
4882        let id = store
4883            .upsert_external(&crate::store::UpsertExternal {
4884                repo: "owner/repo",
4885                ext_id: "1227",
4886                title: "Task with different failure reason",
4887                body: "",
4888                author: "",
4889                url: "",
4890                labels: &[],
4891                origin: "github",
4892            })
4893            .await
4894            .unwrap();
4895        store.update_status(id, TaskStatus::Blocked).await.unwrap();
4896        // Simulate: task was previously auto-unblocked for ModelUnavailable (count=1)
4897        store
4898            .set_fields(
4899                id,
4900                &[
4901                    ("auto_unblock_count", serde_json::json!(1)),
4902                    (
4903                        "auto_unblock_last_reason",
4904                        serde_json::json!("ModelUnavailable"),
4905                    ),
4906                ],
4907            )
4908            .await
4909            .unwrap();
4910
4911        // Add a run with a DIFFERENT failure: RateLimit
4912        let run_id = store
4913            .start_run(&crate::store::StartRun {
4914                task_id: id,
4915                attempt: 2,
4916                run_type: "agent",
4917                agent: "claude",
4918                model: "opus",
4919                command: "claude",
4920                prompt: "fix the bug",
4921            })
4922            .await
4923            .unwrap();
4924        store
4925            .complete_run(&crate::store::CompleteRun {
4926                run_id,
4927                exit_code: Some(1),
4928                stdout: "",
4929                stderr: "",
4930                parsed: "",
4931                outcome: "error",
4932                error: "rate limit reached for model opus",
4933                tokens: RunTokenUsage::default(),
4934            })
4935            .await
4936            .unwrap();
4937
4938        auto_unblock_blocked_tasks(
4939            "owner/repo",
4940            &task_manager,
4941            &store,
4942            &dispatching,
4943            &Arc::new(DashSet::new()),
4944        )
4945        .await
4946        .unwrap();
4947
4948        let task = store.get(id).await.unwrap();
4949        assert_eq!(
4950            task.status,
4951            TaskStatus::New,
4952            "task must be auto-unblocked even with count=1 from a different reason"
4953        );
4954        // Count is reset to 0, then incremented to 1 for the new reason
4955        assert_eq!(task.auto_unblock_count, 1);
4956        assert_eq!(
4957            task.auto_unblock_last_reason, "RateLimit",
4958            "reason must be updated to the new failure category"
4959        );
4960    }
4961
4962    /**
4963     * A task blocked for "exceeded max attempts" has already run the configured number
4964     * of times. Auto-unblock must leave it alone — flipping it back to new/needs_review
4965     * on the next tick restarted the very loop that produced the block (#3783)
4966     **/
4967    #[serial(cooldown_state)]
4968    #[tokio::test]
4969    async fn auto_unblock_leaves_max_attempts_blocked_task_alone() {
4970        cooldown::reset_global_state().await;
4971        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4972        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4973        let task_manager = Arc::new(TaskManager::with_store(
4974            backend,
4975            Arc::clone(&store),
4976            "owner/repo".to_string(),
4977        ));
4978        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4979
4980        let id = store
4981            .upsert_external(&crate::store::UpsertExternal {
4982                repo: "owner/repo",
4983                ext_id: "3783",
4984                title: "Approved work sent back to the agent",
4985                body: "",
4986                author: "",
4987                url: "",
4988                labels: &[],
4989                origin: "github",
4990            })
4991            .await
4992            .unwrap();
4993        store.update_status(id, TaskStatus::Blocked).await.unwrap();
4994
4995        let run_id = store
4996            .start_run(&crate::store::StartRun {
4997                task_id: id,
4998                attempt: 11,
4999                run_type: "agent",
5000                agent: "claude",
5001                model: "opus",
5002                command: "claude",
5003                prompt: "fix the bug",
5004            })
5005            .await
5006            .unwrap();
5007        store
5008            .complete_run(&crate::store::CompleteRun {
5009                run_id,
5010                exit_code: Some(1),
5011                stdout: "",
5012                stderr: "",
5013                parsed: "",
5014                outcome: "error",
5015                error: "exceeded max attempts (11/10). Use `/retry` to reset.",
5016                tokens: RunTokenUsage::default(),
5017            })
5018            .await
5019            .unwrap();
5020
5021        auto_unblock_blocked_tasks(
5022            "owner/repo",
5023            &task_manager,
5024            &store,
5025            &dispatching,
5026            &Arc::new(DashSet::new()),
5027        )
5028        .await
5029        .unwrap();
5030
5031        let task = store.get(id).await.unwrap();
5032        assert_eq!(
5033            task.status,
5034            TaskStatus::Blocked,
5035            "max-attempts block must survive the next tick until a person acts"
5036        );
5037        assert_eq!(
5038            task.auto_unblock_count, 0,
5039            "no auto-unblock attempt should be recorded"
5040        );
5041    }
5042
5043    #[serial(cooldown_state)]
5044    #[tokio::test]
5045    async fn auto_unblock_stores_reason_on_first_unblock() {
5046        cooldown::reset_global_state().await;
5047        // When a task is auto-unblocked for the first time, the failure reason
5048        // should be stored so subsequent blocks can be compared.
5049        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5050        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
5051        let task_manager = Arc::new(TaskManager::with_store(
5052            backend,
5053            Arc::clone(&store),
5054            "owner/repo".to_string(),
5055        ));
5056        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
5057
5058        let id = store
5059            .upsert_external(&crate::store::UpsertExternal {
5060                repo: "owner/repo",
5061                ext_id: "1227b",
5062                title: "Task for first-time reason storage",
5063                body: "",
5064                author: "",
5065                url: "",
5066                labels: &[],
5067                origin: "github",
5068            })
5069            .await
5070            .unwrap();
5071        store.update_status(id, TaskStatus::Blocked).await.unwrap();
5072
5073        let run_id = store
5074            .start_run(&crate::store::StartRun {
5075                task_id: id,
5076                attempt: 1,
5077                run_type: "agent",
5078                agent: "claude",
5079                model: "opus",
5080                command: "claude",
5081                prompt: "fix the bug",
5082            })
5083            .await
5084            .unwrap();
5085        store
5086            .complete_run(&crate::store::CompleteRun {
5087                run_id,
5088                exit_code: Some(1),
5089                stdout: "",
5090                stderr: "",
5091                parsed: "",
5092                outcome: "timeout",
5093                error: "task timed out",
5094                tokens: RunTokenUsage::default(),
5095            })
5096            .await
5097            .unwrap();
5098
5099        auto_unblock_blocked_tasks(
5100            "owner/repo",
5101            &task_manager,
5102            &store,
5103            &dispatching,
5104            &Arc::new(DashSet::new()),
5105        )
5106        .await
5107        .unwrap();
5108
5109        let task = store.get(id).await.unwrap();
5110        assert_eq!(task.status, TaskStatus::New);
5111        assert_eq!(task.auto_unblock_count, 1);
5112        assert_eq!(
5113            task.auto_unblock_last_reason, "Timeout",
5114            "reason must be stored on first auto-unblock"
5115        );
5116    }
5117
5118    // ── classify_failure: sparse checkout (regression for substring false positive) ──
5119
5120    #[serial(cooldown_state)]
5121    #[test]
5122    fn classify_failure_sparse_checkout_not_parse_error() {
5123        // Regression test for #1204: "sparse" substring contains "parse",
5124        // so overly broad substring match would misclassify git sparse-checkout errors
5125        let category = classify_failure("error: cannot reset paths in a sparse checkout", "error");
5126        assert_ne!(
5127            category,
5128            FailureCategory::ParseError,
5129            "sparse checkout errors must NOT be classified as ParseError"
5130        );
5131        assert_eq!(
5132            category,
5133            FailureCategory::Unknown,
5134            "sparse checkout errors should be Unknown (not recoverable auto-unblock)"
5135        );
5136    }
5137
5138    #[serial(cooldown_state)]
5139    #[test]
5140    fn classify_failure_sparse_index_not_parse_error() {
5141        // Another sparse-checkout variant that contains "parse"
5142        let category = classify_failure("failed due to sparse index configuration", "error");
5143        assert_ne!(category, FailureCategory::ParseError);
5144        assert_eq!(category, FailureCategory::Unknown);
5145    }
5146
5147    // ── classify_failure: truncated (issue #3473) ───────────────────────
5148
5149    /// A review output truncated by the model's token budget (opencode
5150    /// step_finish reason=length) must classify as recoverable, so a task
5151    /// blocked after repeated truncations can be auto-unblocked instead of
5152    /// requiring manual intervention.
5153    #[serial(cooldown_state)]
5154    #[test]
5155    fn classify_failure_truncated_is_recoverable() {
5156        let category = classify_failure(
5157            "truncated: review output was cut off before completion \
5158             (output/reasoning token budget exceeded)",
5159            "truncated",
5160        );
5161        assert_eq!(
5162            category,
5163            FailureCategory::ParseError,
5164            "truncated outcome must classify as a recoverable parse-style failure"
5165        );
5166        assert!(
5167            category.is_recoverable(),
5168            "truncated failures must be recoverable so auto-unblock can retry"
5169        );
5170    }
5171
5172    #[serial(cooldown_state)]
5173    #[test]
5174    fn dispatching_key_includes_repo_for_cross_project_isolation() {
5175        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
5176
5177        // Lock task 42 in repo A
5178        let key_a = "owner/repo-a/42".to_string();
5179        dispatching.insert(key_a.clone(), "42".to_string());
5180
5181        // Same task ID in repo B should NOT be blocked
5182        let key_b = "owner/repo-b/42".to_string();
5183        assert!(dispatching.contains_key(&key_a));
5184        assert!(
5185            !dispatching.contains_key(&key_b),
5186            "same task ID in different repo should not be blocked"
5187        );
5188    }
5189
5190    #[serial(cooldown_state)]
5191    #[tokio::test]
5192    async fn emit_degraded_agents_writes_metric_when_three_or_more_agents_degraded() {
5193        cooldown::reset_global_state().await;
5194
5195        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5196
5197        // Create a router and set deterministic available agents
5198        let mut config = RouterConfig::default();
5199        config
5200            .model_map
5201            .entry("medium".to_string())
5202            .or_default()
5203            .insert(
5204                "test-agent-3a".to_string(),
5205                vec!["test-model-a".to_string()],
5206            );
5207        config
5208            .model_map
5209            .entry("medium".to_string())
5210            .or_default()
5211            .insert(
5212                "test-agent-3b".to_string(),
5213                vec!["test-model-b".to_string()],
5214            );
5215        config
5216            .model_map
5217            .entry("medium".to_string())
5218            .or_default()
5219            .insert(
5220                "test-agent-3c".to_string(),
5221                vec!["test-model-c".to_string()],
5222            );
5223        config
5224            .model_map
5225            .entry("medium".to_string())
5226            .or_default()
5227            .insert(
5228                "test-agent-3d".to_string(),
5229                vec!["test-model-d".to_string()],
5230            );
5231        let mut router = Router::new(config);
5232        router.available_agents = vec![
5233            "test-agent-3a".to_string(),
5234            "test-agent-3b".to_string(),
5235            "test-agent-3c".to_string(),
5236            "test-agent-3d".to_string(),
5237        ];
5238
5239        // Place three agents into cooldown
5240        set_agent_cooldown("test-agent-3a", 3600).await;
5241        set_agent_cooldown("test-agent-3b", 3600).await;
5242        set_agent_cooldown("test-agent-3c", 3600).await;
5243
5244        // Call the helper and assert KV metrics written
5245        emit_degraded_agents_if_needed(&router.available_agents, &router.config, Some(&store))
5246            .await;
5247
5248        // Count metric always written
5249        let count_val = store
5250            .kv_get("metrics:orch.agents_degraded.count")
5251            .await
5252            .unwrap();
5253        assert_eq!(count_val.as_deref(), Some("3"));
5254
5255        // Alert metric set to "1" when threshold crossed
5256        let alert_val = store
5257            .kv_get("metrics:orch.agents_degraded.alert")
5258            .await
5259            .unwrap();
5260        assert_eq!(alert_val.as_deref(), Some("1"));
5261    }
5262
5263    #[serial(cooldown_state)]
5264    #[tokio::test]
5265    async fn emit_degraded_agents_clears_alert_when_below_threshold() {
5266        cooldown::reset_global_state().await;
5267
5268        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5269
5270        let mut config = RouterConfig::default();
5271        config
5272            .model_map
5273            .entry("medium".to_string())
5274            .or_default()
5275            .insert(
5276                "test-agent-2a".to_string(),
5277                vec!["test-model-a".to_string()],
5278            );
5279        config
5280            .model_map
5281            .entry("medium".to_string())
5282            .or_default()
5283            .insert(
5284                "test-agent-2b".to_string(),
5285                vec!["test-model-b".to_string()],
5286            );
5287        let mut router = Router::new(config);
5288        router.available_agents = vec!["test-agent-2a".to_string(), "test-agent-2b".to_string()];
5289
5290        // Place only two agents into cooldown — below the threshold of 3
5291        set_agent_cooldown("test-agent-2a", 3600).await;
5292        set_agent_cooldown("test-agent-2b", 3600).await;
5293
5294        emit_degraded_agents_if_needed(&router.available_agents, &router.config, Some(&store))
5295            .await;
5296
5297        // Count metric still written (always-emit)
5298        let count_val = store
5299            .kv_get("metrics:orch.agents_degraded.count")
5300            .await
5301            .unwrap();
5302        assert_eq!(count_val.as_deref(), Some("2"));
5303
5304        // Alert metric cleared (set to "0") — no WARN should fire
5305        let alert_val = store
5306            .kv_get("metrics:orch.agents_degraded.alert")
5307            .await
5308            .unwrap();
5309        assert_eq!(alert_val.as_deref(), Some("0"));
5310
5311        let stale_pool_alert = store
5312            .kv_get("metrics:orch.model_pool_stale_persistent.alert")
5313            .await
5314            .unwrap();
5315        assert_eq!(stale_pool_alert.as_deref(), Some("0"));
5316    }
5317
5318    #[serial(cooldown_state)]
5319    #[tokio::test]
5320    async fn emit_degraded_agents_sets_stale_pool_alert_for_persistent_model_failures() {
5321        cooldown::reset_global_state().await;
5322
5323        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5324
5325        let mut config = RouterConfig::default();
5326        config
5327            .model_map
5328            .entry("medium".to_string())
5329            .or_default()
5330            .insert(
5331                "test-agent-stale".to_string(),
5332                vec!["model-a".to_string(), "model-b".to_string()],
5333            );
5334        let mut router = Router::new(config);
5335        router.available_agents = vec!["test-agent-stale".to_string()];
5336
5337        // 50% cooled + persistent reason should trigger stale-pool alert.
5338        record_persistent_model_failure("test-agent-stale", "model-a").await;
5339
5340        emit_degraded_agents_if_needed(&router.available_agents, &router.config, Some(&store))
5341            .await;
5342
5343        let stale_pool_alert = store
5344            .kv_get("metrics:orch.model_pool_stale_persistent.alert")
5345            .await
5346            .unwrap();
5347        assert_eq!(stale_pool_alert.as_deref(), Some("1"));
5348    }
5349
5350    // ── classify_comment tests ──────────────────────────────────────────
5351
5352    #[serial(cooldown_state)]
5353    #[test]
5354    fn classify_skip_already_processed() {
5355        let action = classify_comment(
5356            "@orch /retry",
5357            Some("https://api.github.com/repos/o/r/issues/1"),
5358            "@bot",
5359            true,
5360        );
5361        assert_eq!(action, CommentAction::Skip);
5362    }
5363
5364    #[serial(cooldown_state)]
5365    #[test]
5366    fn classify_skip_irrelevant_comment() {
5367        let action = classify_comment("just a regular comment", None, "@bot", false);
5368        assert_eq!(action, CommentAction::Skip);
5369    }
5370
5371    #[serial(cooldown_state)]
5372    #[test]
5373    fn classify_command_without_mention() {
5374        let action = classify_comment(
5375            "/retry",
5376            Some("https://api.github.com/repos/o/r/issues/42"),
5377            "@bot",
5378            false,
5379        );
5380        assert_eq!(
5381            action,
5382            CommentAction::ExecuteCommand {
5383                command: OwnerCommand::Retry,
5384                issue_num: "42".to_string(),
5385            }
5386        );
5387    }
5388
5389    #[serial(cooldown_state)]
5390    #[test]
5391    fn classify_command_with_mention() {
5392        let action = classify_comment(
5393            "@orch /close",
5394            Some("https://api.github.com/repos/o/r/issues/7"),
5395            "@bot",
5396            false,
5397        );
5398        assert_eq!(
5399            action,
5400            CommentAction::ExecuteCommandForMention {
5401                command: OwnerCommand::Close,
5402                issue_num: "7".to_string(),
5403            }
5404        );
5405    }
5406
5407    #[serial(cooldown_state)]
5408    #[test]
5409    fn classify_mention_without_command() {
5410        let action = classify_comment(
5411            "@orch can you look at this?",
5412            Some("https://api.github.com/repos/o/r/issues/99"),
5413            "@bot",
5414            false,
5415        );
5416        assert_eq!(
5417            action,
5418            CommentAction::CreateMentionTask {
5419                issue_num: Some("99".to_string()),
5420            }
5421        );
5422    }
5423
5424    #[serial(cooldown_state)]
5425    #[test]
5426    fn classify_mention_without_issue_url() {
5427        let action = classify_comment("@orch help me", None, "@bot", false);
5428        assert_eq!(action, CommentAction::CreateMentionTask { issue_num: None });
5429    }
5430
5431    #[serial(cooldown_state)]
5432    #[test]
5433    fn classify_command_no_mention_no_url_skips() {
5434        let action = classify_comment("/retry", None, "@bot", false);
5435        assert_eq!(action, CommentAction::Skip);
5436    }
5437
5438    #[serial(cooldown_state)]
5439    #[test]
5440    fn classify_mention_via_current_user() {
5441        let action = classify_comment(
5442            "@bot please help",
5443            Some("https://api.github.com/repos/o/r/issues/5"),
5444            "@bot",
5445            false,
5446        );
5447        assert_eq!(
5448            action,
5449            CommentAction::CreateMentionTask {
5450                issue_num: Some("5".to_string()),
5451            }
5452        );
5453    }
5454
5455    #[serial(cooldown_state)]
5456    #[test]
5457    fn classify_command_with_mention_no_url_creates_task() {
5458        // @mention + command but no issue URL → can't run command, create mention task
5459        let action = classify_comment("@orch /retry", None, "@bot", false);
5460        assert_eq!(action, CommentAction::CreateMentionTask { issue_num: None });
5461    }
5462
5463    #[serial(cooldown_state)]
5464    #[test]
5465    fn mention_cursor_stops_advancing_after_gap() {
5466        let mut cursor = MentionCursor::default();
5467        cursor.advance("2026-01-01T00:00:00Z");
5468        cursor.block_on_gap();
5469        cursor.advance("2026-01-02T00:00:00Z");
5470
5471        assert_eq!(
5472            cursor.into_last_safe_ts().as_deref(),
5473            Some("2026-01-01T00:00:00Z")
5474        );
5475    }
5476
5477    #[serial(cooldown_state)]
5478    #[test]
5479    fn mention_cursor_tracks_max_timestamp_until_gap() {
5480        let mut cursor = MentionCursor::default();
5481        cursor.advance("2026-01-01T00:00:00Z");
5482        cursor.advance("2026-01-01T00:05:00Z");
5483        cursor.block_on_gap();
5484        cursor.advance("2026-01-01T00:10:00Z");
5485
5486        assert_eq!(
5487            cursor.into_last_safe_ts().as_deref(),
5488            Some("2026-01-01T00:05:00Z")
5489        );
5490    }
5491
5492    // ── scan_comments: acknowledge_mention failure does not advance cursor ────
5493
5494    /// Backend that tracks acknowledge_mention calls and can be configured to fail them.
5495    struct AckTrackingBackend {
5496        ack_calls: Mutex<Vec<String>>,
5497        posted: Mutex<Vec<String>>,
5498        ack_failure: bool,
5499        inner: NoopBackend,
5500    }
5501
5502    impl AckTrackingBackend {
5503        fn new(ack_failure: bool) -> Arc<Self> {
5504            Arc::new(Self {
5505                ack_calls: Mutex::new(Vec::new()),
5506                posted: Mutex::new(Vec::new()),
5507                ack_failure,
5508                inner: NoopBackend,
5509            })
5510        }
5511
5512        fn fail_on_ack(&self) -> bool {
5513            self.ack_failure
5514        }
5515    }
5516
5517    #[async_trait]
5518    impl ExternalBackend for AckTrackingBackend {
5519        fn name(&self) -> &'static str {
5520            "ack-tracking"
5521        }
5522
5523        async fn acknowledge_mention(&self, comment_id: &str) -> anyhow::Result<()> {
5524            self.ack_calls.lock().await.push(comment_id.to_string());
5525            if self.fail_on_ack() {
5526                // Return a transient error to simulate network glitch.
5527                anyhow::bail!("network error: temporary failure posting reaction");
5528            }
5529            Ok(())
5530        }
5531
5532        // Override get_authenticated_user to ensure scan proceeds.
5533        async fn get_authenticated_user(&self) -> anyhow::Result<Option<String>> {
5534            Ok(Some("bot".into()))
5535        }
5536
5537        // Override get_mentions to return empty (we use prefetched comments).
5538        async fn get_mentions(&self, _since: &str) -> anyhow::Result<Vec<Mention>> {
5539            Ok(vec![])
5540        }
5541
5542        // Delegate everything else to NoopBackend.
5543        async fn create_task(&self, t: &str, b: &str, l: &[String]) -> anyhow::Result<ExternalId> {
5544            self.inner.create_task(t, b, l).await
5545        }
5546        async fn get_task(&self, id: &ExternalId) -> anyhow::Result<ExternalTask> {
5547            self.inner.get_task(id).await
5548        }
5549        async fn list_by_status(&self, s: Status) -> anyhow::Result<Vec<ExternalTask>> {
5550            self.inner.list_by_status(s).await
5551        }
5552        async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
5553            self.inner.list_routable().await
5554        }
5555        async fn post_comment(&self, id: &ExternalId, b: &str) -> anyhow::Result<()> {
5556            self.posted.lock().await.push(b.to_string());
5557            self.inner.post_comment(id, b).await
5558        }
5559        async fn set_labels(&self, id: &ExternalId, l: &[String]) -> anyhow::Result<()> {
5560            self.inner.set_labels(id, l).await
5561        }
5562        async fn remove_label(&self, id: &ExternalId, l: &str) -> anyhow::Result<()> {
5563            self.inner.remove_label(id, l).await
5564        }
5565        async fn get_sub_issues(&self, id: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
5566            self.inner.get_sub_issues(id).await
5567        }
5568        async fn create_sub_task(
5569            &self,
5570            p: &ExternalId,
5571            t: &str,
5572            b: &str,
5573            l: &[String],
5574        ) -> anyhow::Result<ExternalId> {
5575            self.inner.create_sub_task(p, t, b, l).await
5576        }
5577        async fn ensure_status_label(&self, l: &str) -> anyhow::Result<()> {
5578            self.inner.ensure_status_label(l).await
5579        }
5580        async fn has_open_issue_with_title(&self, t: &str, l: &str) -> anyhow::Result<bool> {
5581            self.inner.has_open_issue_with_title(t, l).await
5582        }
5583        async fn health_check(&self) -> anyhow::Result<()> {
5584            self.inner.health_check().await
5585        }
5586        async fn is_pr_merged(&self, b: &str) -> anyhow::Result<bool> {
5587            self.inner.is_pr_merged(b).await
5588        }
5589        async fn update_status(&self, id: &ExternalId, s: Status) -> anyhow::Result<()> {
5590            self.inner.update_status(id, s).await
5591        }
5592    }
5593
5594    /// When acknowledge_mention fails for CreateMentionTask, the cursor must NOT
5595    /// advance and no mention task should be created. The next tick will retry.
5596    #[serial(cooldown_state)]
5597    #[tokio::test]
5598    async fn create_mention_task_skips_on_ack_failure() {
5599        cooldown::reset_global_state().await;
5600        let backend: Arc<AckTrackingBackend> = AckTrackingBackend::new(true);
5601        let store: Arc<TaskStore> = Arc::new(TaskStore::open_memory().await.unwrap());
5602
5603        // Set an initial cursor so we can verify it wasn't advanced.
5604        let initial_ts = "2026-01-01T00:00:00Z";
5605        store
5606            .kv_set("mentions_last_checked", initial_ts)
5607            .await
5608            .unwrap();
5609
5610        let mention = Mention {
5611            id: "mention-abc".into(),
5612            body: "@bot can you help?".into(),
5613            author: "alice".into(),
5614            created_at: "2026-01-02T00:00:00Z".into(),
5615            issue_url: Some("https://api.github.com/repos/owner/repo/issues/42".into()),
5616        };
5617
5618        let backend_trait: Arc<dyn ExternalBackend> = Arc::<AckTrackingBackend>::clone(&backend);
5619        let task_manager = Arc::new(TaskManager::new(Arc::clone(&backend_trait)));
5620
5621        scan_comments(
5622            &backend_trait,
5623            Some(&store),
5624            "owner/repo",
5625            &task_manager,
5626            Some(&[mention]),
5627        )
5628        .await
5629        .unwrap();
5630
5631        // acknowledge_mention was called with the mention ID.
5632        let calls = backend.ack_calls.lock().await;
5633        assert_eq!(
5634            &**calls,
5635            &["mention-abc"],
5636            "acknowledge_mention must be called"
5637        );
5638
5639        // No mention task was created.
5640        let all = store.list_all("owner/repo").await.unwrap();
5641        assert!(
5642            all.is_empty(),
5643            "no task should be created when acknowledge_mention fails"
5644        );
5645
5646        // Cursor was NOT advanced.
5647        let cursor = store.kv_get("mentions_last_checked").await.unwrap();
5648        assert_eq!(
5649            cursor.as_deref(),
5650            Some(initial_ts),
5651            "cursor must not advance when acknowledge_mention fails"
5652        );
5653    }
5654
5655    /// When acknowledge_mention fails for ExecuteCommandForMention, handle_slash_command
5656    /// must NOT be called and no mention task should be created.
5657    #[serial(cooldown_state)]
5658    #[tokio::test]
5659    async fn execute_command_for_mention_skips_on_ack_failure() {
5660        cooldown::reset_global_state().await;
5661        let backend: Arc<AckTrackingBackend> = AckTrackingBackend::new(true);
5662        let store: Arc<TaskStore> = Arc::new(TaskStore::open_memory().await.unwrap());
5663
5664        let initial_ts = "2026-01-01T00:00:00Z";
5665        store
5666            .kv_set("mentions_last_checked", initial_ts)
5667            .await
5668            .unwrap();
5669
5670        // Mention with a slash command and issue URL → ExecuteCommandForMention.
5671        let mention = Mention {
5672            id: "mention-def".into(),
5673            body: "@bot /close".into(),
5674            author: "alice".into(),
5675            created_at: "2026-01-02T00:00:00Z".into(),
5676            issue_url: Some("https://api.github.com/repos/owner/repo/issues/42".into()),
5677        };
5678
5679        let backend_trait: Arc<dyn ExternalBackend> = Arc::<AckTrackingBackend>::clone(&backend);
5680        let task_manager = Arc::new(TaskManager::new(Arc::clone(&backend_trait)));
5681
5682        scan_comments(
5683            &backend_trait,
5684            Some(&store),
5685            "owner/repo",
5686            &task_manager,
5687            Some(&[mention]),
5688        )
5689        .await
5690        .unwrap();
5691
5692        // acknowledge_mention was called.
5693        let calls = backend.ack_calls.lock().await;
5694        assert_eq!(&**calls, &["mention-def"]);
5695
5696        // No mention task was created (handle_slash_command was not called).
5697        let all = store.list_all("owner/repo").await.unwrap();
5698        assert!(
5699            all.is_empty(),
5700            "no task should be created when acknowledge_mention fails for ExecuteCommandForMention"
5701        );
5702
5703        // Cursor was NOT advanced.
5704        let cursor = store.kv_get("mentions_last_checked").await.unwrap();
5705        assert_eq!(
5706            cursor.as_deref(),
5707            Some(initial_ts),
5708            "cursor must not advance when acknowledge_mention fails for ExecuteCommandForMention"
5709        );
5710    }
5711
5712    // ── slash commands retry after a transient GitHub error ──────────────────
5713
5714    /// GitHub stub whose `get_issue` fails once, then succeeds.
5715    #[derive(Default)]
5716    struct FlakyGh {
5717        get_issue_calls: AtomicUsize,
5718    }
5719
5720    #[async_trait]
5721    impl CommandGh for FlakyGh {
5722        async fn get_issue(&self, _repo: &str, number: &str) -> anyhow::Result<GitHubIssue> {
5723            if self.get_issue_calls.fetch_add(1, Ordering::SeqCst) == 0 {
5724                anyhow::bail!("network error: temporary failure fetching issue");
5725            }
5726            Ok(GitHubIssue {
5727                number: number.parse().unwrap_or(0),
5728                id: 1,
5729                title: "t".into(),
5730                body: None,
5731                state: "open".into(),
5732                labels: Vec::new(),
5733                user: GitHubUser {
5734                    login: "alice".into(),
5735                },
5736                created_at: String::new(),
5737                updated_at: String::new(),
5738                html_url: String::new(),
5739                node_id: None,
5740                pull_request: None,
5741                author_association: Some("OWNER".into()),
5742                state_reason: None,
5743            })
5744        }
5745
5746        async fn is_collaborator(&self, _repo: &str, _username: &str) -> anyhow::Result<bool> {
5747            Ok(true)
5748        }
5749
5750        async fn close_issue(&self, _repo: &str, _number: &str) -> anyhow::Result<()> {
5751            Ok(())
5752        }
5753    }
5754
5755    /// A transient `get_issue` failure must leave the cursor before the comment so
5756    /// the next scan retries it — and the command must then run exactly once.
5757    #[serial(cooldown_state)]
5758    #[tokio::test]
5759    async fn slash_command_retries_after_transient_fetch_failure() {
5760        cooldown::reset_global_state().await;
5761        let backend: Arc<AckTrackingBackend> = AckTrackingBackend::new(false);
5762        let store: Arc<TaskStore> = Arc::new(TaskStore::open_memory().await.unwrap());
5763        let initial_ts = "2026-01-01T00:00:00Z";
5764
5765        let mention = Mention {
5766            id: "mention-ghi".into(),
5767            body: "@bot /unblock".into(),
5768            author: "alice".into(),
5769            created_at: "2026-01-02T00:00:00Z".into(),
5770            issue_url: Some("https://api.github.com/repos/owner/repo/issues/42".into()),
5771        };
5772
5773        let backend_trait: Arc<dyn ExternalBackend> = Arc::<AckTrackingBackend>::clone(&backend);
5774        let task_manager = Arc::new(TaskManager::new(Arc::clone(&backend_trait)));
5775        let gh = FlakyGh::default();
5776        let mut cursor = MentionCursor::default();
5777
5778        // First scan: get_issue fails.
5779        let advanced_safely = handle_slash_command(
5780            &backend_trait,
5781            Some(&store),
5782            "owner/repo",
5783            &task_manager,
5784            &gh,
5785            &mention,
5786            &OwnerCommand::Unblock,
5787            &mut cursor,
5788        )
5789        .await
5790        .unwrap();
5791
5792        assert!(
5793            !advanced_safely,
5794            "a transient fetch failure must stop cursor advancement"
5795        );
5796        assert!(
5797            cursor.last_safe_ts.is_none(),
5798            "cursor must stay before the comment so the next tick retries it"
5799        );
5800        assert!(
5801            store.list_all("owner/repo").await.unwrap().is_empty(),
5802            "no mention task on a transient failure"
5803        );
5804        assert!(
5805            backend.posted.lock().await.is_empty(),
5806            "nothing is posted to GitHub on a transient failure"
5807        );
5808        assert_eq!(
5809            filter_mentions_by_since(slice::from_ref(&mention), initial_ts).len(),
5810            1,
5811            "the comment is still in range for the next scan"
5812        );
5813
5814        // Second scan: get_issue succeeds and the command runs.
5815        let mut cursor = MentionCursor::default();
5816        let advanced_safely = handle_slash_command(
5817            &backend_trait,
5818            Some(&store),
5819            "owner/repo",
5820            &task_manager,
5821            &gh,
5822            &mention,
5823            &OwnerCommand::Unblock,
5824            &mut cursor,
5825        )
5826        .await
5827        .unwrap();
5828
5829        assert!(advanced_safely);
5830        assert_eq!(
5831            cursor.into_last_safe_ts().as_deref(),
5832            Some("2026-01-02T00:00:00Z"),
5833            "cursor advances once the command actually runs"
5834        );
5835        assert_eq!(
5836            backend.posted.lock().await.len(),
5837            1,
5838            "the command runs once, posting a single confirmation"
5839        );
5840        assert_eq!(
5841            store.list_all("owner/repo").await.unwrap().len(),
5842            1,
5843            "one sentinel mention task for the executed command"
5844        );
5845    }
5846
5847    /// In the plain `ExecuteCommand` arm the GitHub client points at a dead port
5848    /// under `cfg(test)`, so `get_issue` always fails — the cursor must stay put.
5849    #[serial(cooldown_state)]
5850    #[tokio::test]
5851    async fn execute_command_keeps_cursor_on_fetch_failure() {
5852        cooldown::reset_global_state().await;
5853        let backend: Arc<AckTrackingBackend> = AckTrackingBackend::new(false);
5854        let store: Arc<TaskStore> = Arc::new(TaskStore::open_memory().await.unwrap());
5855
5856        let initial_ts = "2026-01-01T00:00:00Z";
5857        store
5858            .kv_set("mentions_last_checked", initial_ts)
5859            .await
5860            .unwrap();
5861
5862        // Slash command without a mention → ExecuteCommand.
5863        let comment = Mention {
5864            id: "comment-jkl".into(),
5865            body: "/unblock".into(),
5866            author: "alice".into(),
5867            created_at: "2026-01-02T00:00:00Z".into(),
5868            issue_url: Some("https://api.github.com/repos/owner/repo/issues/42".into()),
5869        };
5870
5871        let backend_trait: Arc<dyn ExternalBackend> = Arc::<AckTrackingBackend>::clone(&backend);
5872        let task_manager = Arc::new(TaskManager::new(Arc::clone(&backend_trait)));
5873
5874        scan_comments(
5875            &backend_trait,
5876            Some(&store),
5877            "owner/repo",
5878            &task_manager,
5879            Some(&[comment]),
5880        )
5881        .await
5882        .unwrap();
5883
5884        assert!(
5885            backend.posted.lock().await.is_empty(),
5886            "nothing is posted when the issue fetch fails"
5887        );
5888        assert_eq!(
5889            store
5890                .kv_get("mentions_last_checked")
5891                .await
5892                .unwrap()
5893                .as_deref(),
5894            Some(initial_ts),
5895            "cursor must not advance past a command that never ran"
5896        );
5897    }
5898
5899    // ── is_ci_failure_block ────────────────────────────────────────────────
5900
5901    fn make_task(block_reason: Option<String>) -> Task {
5902        Task {
5903            id: 1,
5904            external_id: None,
5905            repo: "owner/repo".to_string(),
5906            origin: String::new(),
5907            title: String::new(),
5908            body: String::new(),
5909            status: TaskStatus::Blocked,
5910            source: String::new(),
5911            source_id: String::new(),
5912            author: String::new(),
5913            url: String::new(),
5914            labels: vec![],
5915            agent: None,
5916            model: None,
5917            complexity: String::new(),
5918            estimate: 0,
5919            route_reason: String::new(),
5920            agent_profile: String::new(),
5921            route_attempts: 0,
5922            attempts: 0,
5923            branch: String::new(),
5924            worktree: String::new(),
5925            worktree_cleaned: false,
5926            owner_node: None,
5927            summary: String::new(),
5928            last_error: String::new(),
5929            parent_id: None,
5930            block_reason,
5931            pr_number: Some(42),
5932            pr_review_context: String::new(),
5933            last_review_ts: String::new(),
5934            review_ts_map: String::new(),
5935            last_comment_review_ts: String::new(),
5936            merge_conflict_retries: 0,
5937            ci_merge_failures: 0,
5938            pr_create_failures: 0,
5939            push_failures: 0,
5940            network_retries: 0,
5941            timeout_retries: 0,
5942            review_agent_failures: 0,
5943            review_resets: 0,
5944            review_cycles: 0,
5945            review_invocations: 0,
5946            needs_review_refires: 0,
5947            input_tokens: 0,
5948            output_tokens: 0,
5949            total_cost_usd: 0.0,
5950            model_reroute_chain: String::new(),
5951            limit_reroute_chain: String::new(),
5952            memory: vec![],
5953            delegations: vec![],
5954            auto_unblock_count: 0,
5955            auto_unblock_last_at: String::new(),
5956            auto_unblock_last_reason: String::new(),
5957            ci_recovery_count: 0,
5958            no_code_reroutes: 0,
5959            no_code_last_agent: String::new(),
5960            created_at: String::new(),
5961            updated_at: String::new(),
5962        }
5963    }
5964
5965    #[serial(cooldown_state)]
5966    #[test]
5967    fn is_ci_failure_block_matches_ci_failure_limit() {
5968        let task = make_task(Some(
5969            "CI failure limit (3) reached during auto-merge".to_string(),
5970        ));
5971        assert!(is_ci_failure_block(&task));
5972    }
5973
5974    #[serial(cooldown_state)]
5975    #[test]
5976    fn is_ci_failure_block_matches_ci_checks_timeout() {
5977        let task = make_task(Some(
5978            "CI checks timed out after 3 auto-merge failures".to_string(),
5979        ));
5980        assert!(is_ci_failure_block(&task));
5981    }
5982
5983    #[serial(cooldown_state)]
5984    #[test]
5985    fn is_ci_failure_block_false_for_regular_block() {
5986        let task = make_task(Some("waiting on input".to_string()));
5987        assert!(!is_ci_failure_block(&task));
5988    }
5989
5990    #[serial(cooldown_state)]
5991    #[test]
5992    fn is_ci_failure_block_false_for_none_reason() {
5993        let task = make_task(None);
5994        assert!(!is_ci_failure_block(&task));
5995    }
5996
5997    // ── ci_failure_unblock_cooldown_elapsed ────────────────────────────────
5998
5999    #[serial(cooldown_state)]
6000    #[test]
6001    fn ci_failure_cooldown_zero_count_immediate() {
6002        // count=0 means never attempted — always allow immediate attempt
6003        let task = make_task(None);
6004        assert!(ci_failure_unblock_cooldown_elapsed(&task));
6005    }
6006
6007    #[serial(cooldown_state)]
6008    #[test]
6009    fn ci_failure_cooldown_empty_last_at_immediate() {
6010        // count=1 but empty last_at means it was never recorded — allow immediate
6011        let mut task = make_task(None);
6012        task.auto_unblock_count = 1;
6013        assert!(ci_failure_unblock_cooldown_elapsed(&task));
6014    }
6015
6016    #[serial(cooldown_state)]
6017    #[test]
6018    fn ci_failure_cooldown_invalid_timestamp_immediate() {
6019        // invalid timestamp should not block
6020        let mut task = make_task(None);
6021        task.auto_unblock_count = 1;
6022        task.auto_unblock_last_at = "not-a-timestamp".to_string();
6023        assert!(ci_failure_unblock_cooldown_elapsed(&task));
6024    }
6025
6026    #[serial(cooldown_state)]
6027    #[test]
6028    fn ci_failure_cooldown_high_count_still_eventually_elapses() {
6029        // count >= 3 must NOT be a permanent block — verify a sufficiently old
6030        // timestamp eventually elapses the cooldown.
6031        let long_ago = (chrono::Utc::now() - chrono::Duration::days(30)).to_rfc3339();
6032        for count in [3i32, 5, 99] {
6033            let mut task = make_task(None);
6034            task.auto_unblock_count = count;
6035            task.auto_unblock_last_at = long_ago.clone();
6036            assert!(
6037                ci_failure_unblock_cooldown_elapsed(&task),
6038                "count={count} should eventually elapse"
6039            );
6040        }
6041    }
6042
6043    #[serial(cooldown_state)]
6044    #[test]
6045    fn ci_failure_cooldown_recent_timestamp_still_blocks() {
6046        // A recent timestamp should still gate the check at every count.
6047        let just_now = chrono::Utc::now().to_rfc3339();
6048        for count in [1i32, 2, 3, 99] {
6049            let mut task = make_task(None);
6050            task.auto_unblock_count = count;
6051            task.auto_unblock_last_at = just_now.clone();
6052            assert!(
6053                !ci_failure_unblock_cooldown_elapsed(&task),
6054                "count={count} should still be cooling down"
6055            );
6056        }
6057    }
6058
6059    // ── auto_unblock_ci_failure_blocked_tasks_global ───────────────────────
6060
6061    #[serial(cooldown_state)]
6062    #[tokio::test]
6063    async fn global_ci_failure_sweep_returns_ok_with_no_blocked_tasks() {
6064        cooldown::reset_global_state().await;
6065        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6066        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6067        let task_manager = Arc::new(TaskManager::with_store(
6068            backend,
6069            Arc::clone(&store),
6070            "owner/repo".to_string(),
6071        ));
6072        auto_unblock_ci_failure_blocked_tasks_global(
6073            &task_manager,
6074            &store,
6075            &Arc::new(DashSet::new()),
6076        )
6077        .await
6078        .unwrap();
6079    }
6080
6081    #[serial(cooldown_state)]
6082    #[tokio::test]
6083    async fn global_ci_failure_sweep_skips_tasks_in_cooldown() {
6084        // Tasks with a fresh auto_unblock_last_at are skipped before reaching GhHttp,
6085        // so this test works without a real GitHub token.
6086        cooldown::reset_global_state().await;
6087        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6088        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6089        let task_manager = Arc::new(TaskManager::with_store(
6090            backend,
6091            Arc::clone(&store),
6092            "active/repo".to_string(),
6093        ));
6094
6095        // Insert a CI-failure task from an inactive repo with a very recent last_at.
6096        let id = store
6097            .upsert_external(&crate::store::UpsertExternal {
6098                repo: "inactive/repo",
6099                ext_id: "10",
6100                title: "Old blocked task",
6101                body: "",
6102                author: "",
6103                url: "",
6104                labels: &[],
6105                origin: "github",
6106            })
6107            .await
6108            .unwrap();
6109        store.update_status(id, TaskStatus::Blocked).await.unwrap();
6110        let just_now = chrono::Utc::now().to_rfc3339();
6111        store
6112            .set_fields(
6113                id,
6114                &[
6115                    (
6116                        "block_reason",
6117                        serde_json::json!("CI failure limit reached during auto-merge"),
6118                    ),
6119                    ("pr_number", serde_json::json!(99)),
6120                    ("auto_unblock_count", serde_json::json!(1)),
6121                    ("auto_unblock_last_at", serde_json::json!(just_now)),
6122                ],
6123            )
6124            .await
6125            .unwrap();
6126
6127        auto_unblock_ci_failure_blocked_tasks_global(
6128            &task_manager,
6129            &store,
6130            &Arc::new(DashSet::new()),
6131        )
6132        .await
6133        .unwrap();
6134
6135        // Cooldown not elapsed — task must remain untouched.
6136        let task = store.get(id).await.unwrap();
6137        assert_eq!(
6138            task.auto_unblock_count, 1,
6139            "task in cooldown must not be processed"
6140        );
6141    }
6142
6143    #[serial(cooldown_state)]
6144    #[tokio::test]
6145    async fn global_ci_failure_sweep_ignores_non_ci_failure_blocks() {
6146        cooldown::reset_global_state().await;
6147        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6148        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6149        let task_manager = Arc::new(TaskManager::with_store(
6150            backend,
6151            Arc::clone(&store),
6152            "active/repo".to_string(),
6153        ));
6154
6155        // Insert a non-CI-failure blocked task — should be ignored entirely.
6156        let id = store
6157            .upsert_external(&crate::store::UpsertExternal {
6158                repo: "inactive/repo",
6159                ext_id: "20",
6160                title: "Manual block",
6161                body: "",
6162                author: "",
6163                url: "",
6164                labels: &[],
6165                origin: "github",
6166            })
6167            .await
6168            .unwrap();
6169        store.update_status(id, TaskStatus::Blocked).await.unwrap();
6170        store
6171            .set_fields(
6172                id,
6173                &[("block_reason", serde_json::json!("waiting for human input"))],
6174            )
6175            .await
6176            .unwrap();
6177
6178        auto_unblock_ci_failure_blocked_tasks_global(
6179            &task_manager,
6180            &store,
6181            &Arc::new(DashSet::new()),
6182        )
6183        .await
6184        .unwrap();
6185
6186        // Non-CI-failure tasks must remain blocked.
6187        let task = store.get(id).await.unwrap();
6188        assert_eq!(task.status, TaskStatus::Blocked);
6189    }
6190
6191    // ── auto_recover_rebroadcast_blocked_tasks_global ───────────────────────
6192
6193    #[serial(cooldown_state)]
6194    #[tokio::test]
6195    async fn global_rebroadcast_sweep_returns_ok_with_no_blocked_tasks() {
6196        cooldown::reset_global_state().await;
6197        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6198        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6199        let task_manager = Arc::new(TaskManager::with_store(
6200            backend,
6201            Arc::clone(&store),
6202            "owner/repo".to_string(),
6203        ));
6204        let router = router_with_healthy_review_agent();
6205        auto_recover_rebroadcast_blocked_tasks_global(
6206            &router,
6207            &task_manager,
6208            &store,
6209            &sweep_repos(),
6210        )
6211        .await
6212        .unwrap();
6213    }
6214
6215    #[serial(cooldown_state)]
6216    #[tokio::test]
6217    async fn global_rebroadcast_sweep_ignores_non_rebroadcast_blocks() {
6218        cooldown::reset_global_state().await;
6219        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6220        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6221        let task_manager = Arc::new(TaskManager::with_store(
6222            backend,
6223            Arc::clone(&store),
6224            "active/repo".to_string(),
6225        ));
6226        let router = router_with_healthy_review_agent();
6227
6228        let id = store
6229            .upsert_external(&crate::store::UpsertExternal {
6230                repo: "inactive/repo",
6231                ext_id: "30",
6232                title: "Manual block",
6233                body: "",
6234                author: "",
6235                url: "",
6236                labels: &[],
6237                origin: "github",
6238            })
6239            .await
6240            .unwrap();
6241        store.update_status(id, TaskStatus::Blocked).await.unwrap();
6242        store
6243            .set_fields(
6244                id,
6245                &[("block_reason", serde_json::json!("waiting for human input"))],
6246            )
6247            .await
6248            .unwrap();
6249        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
6250            .bind(id)
6251            .execute(store.pool())
6252            .await
6253            .unwrap();
6254
6255        auto_recover_rebroadcast_blocked_tasks_global(
6256            &router,
6257            &task_manager,
6258            &store,
6259            &sweep_repos(),
6260        )
6261        .await
6262        .unwrap();
6263
6264        let task = store.get(id).await.unwrap();
6265        assert_eq!(task.status, TaskStatus::Blocked);
6266        assert_eq!(
6267            task.block_reason.as_deref(),
6268            Some("waiting for human input")
6269        );
6270    }
6271
6272    #[serial(cooldown_state)]
6273    #[tokio::test]
6274    async fn global_rebroadcast_sweep_skips_freshly_blocked_tasks() {
6275        cooldown::reset_global_state().await;
6276        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6277        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6278        let task_manager = Arc::new(TaskManager::with_store(
6279            backend,
6280            Arc::clone(&store),
6281            "active/repo".to_string(),
6282        ));
6283        let router = router_with_healthy_review_agent();
6284
6285        let id = store
6286            .upsert_external(&crate::store::UpsertExternal {
6287                repo: "inactive/repo",
6288                ext_id: "40",
6289                title: "Just escalated",
6290                body: "",
6291                author: "",
6292                url: "",
6293                labels: &[],
6294                origin: "github",
6295            })
6296            .await
6297            .unwrap();
6298        store.update_status(id, TaskStatus::Blocked).await.unwrap();
6299        store
6300            .set_fields(
6301                id,
6302                &[("block_reason", serde_json::json!(REBROADCAST_BLOCK_REASON))],
6303            )
6304            .await
6305            .unwrap();
6306        // No backdating — escalated "just now" in this same tick, must not be
6307        // immediately un-blocked by the same pass.
6308
6309        auto_recover_rebroadcast_blocked_tasks_global(
6310            &router,
6311            &task_manager,
6312            &store,
6313            &sweep_repos(),
6314        )
6315        .await
6316        .unwrap();
6317
6318        let task = store.get(id).await.unwrap();
6319        assert_eq!(task.status, TaskStatus::Blocked);
6320    }
6321
6322    /// Regression test for #3532: a rebroadcast-blocked task whose repo is not
6323    /// the TaskManager's own repo must still be recovered. The sweep queries
6324    /// across all repos and writes through the store directly, bypassing the
6325    /// repo-scoped TaskManager
6326    #[serial(cooldown_state)]
6327    #[tokio::test]
6328    async fn global_rebroadcast_sweep_recovers_task_from_inactive_repo() {
6329        cooldown::reset_global_state().await;
6330        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6331        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6332        let task_manager = Arc::new(TaskManager::with_store(
6333            backend,
6334            Arc::clone(&store),
6335            "active/repo".to_string(),
6336        ));
6337        let router = router_with_healthy_review_agent();
6338
6339        let id = store
6340            .upsert_external(&crate::store::UpsertExternal {
6341                repo: "inactive/repo",
6342                ext_id: "458",
6343                title: "Rebroadcast blocked from inactive repo",
6344                body: "",
6345                author: "",
6346                url: "",
6347                labels: &[],
6348                origin: "github",
6349            })
6350            .await
6351            .unwrap();
6352        store.update_status(id, TaskStatus::Blocked).await.unwrap();
6353        store
6354            .set_fields(
6355                id,
6356                &[
6357                    ("block_reason", serde_json::json!(REBROADCAST_BLOCK_REASON)),
6358                    ("needs_review_refires", serde_json::json!(6)),
6359                ],
6360            )
6361            .await
6362            .unwrap();
6363        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
6364            .bind(id)
6365            .execute(store.pool())
6366            .await
6367            .unwrap();
6368
6369        auto_recover_rebroadcast_blocked_tasks_global(
6370            &router,
6371            &task_manager,
6372            &store,
6373            &sweep_repos(),
6374        )
6375        .await
6376        .unwrap();
6377
6378        let task = store.get(id).await.unwrap();
6379        assert_eq!(
6380            task.status,
6381            TaskStatus::NeedsReview,
6382            "cross-repo rebroadcast-blocked task must be recovered to needs_review"
6383        );
6384        assert!(
6385            task.block_reason.is_none(),
6386            "block_reason must be cleared on recovery"
6387        );
6388        assert_eq!(
6389            task.needs_review_refires, 0,
6390            "needs_review_refires must be reset to 0 on recovery"
6391        );
6392    }
6393
6394    /// With no routable review agent, the sweep must not reset the task, that
6395    /// would immediately re-exhaust the refire counter
6396    #[serial(cooldown_state)]
6397    #[tokio::test]
6398    async fn global_rebroadcast_sweep_skips_reset_without_routable_review_agent() {
6399        cooldown::reset_global_state().await;
6400        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6401        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6402        let task_manager = Arc::new(TaskManager::with_store(
6403            backend,
6404            Arc::clone(&store),
6405            "active/repo".to_string(),
6406        ));
6407        // RouterConfig::default() has no "review" model map entries, so the one
6408        // available agent has no healthy model for "review" — nothing is routable.
6409        let router = Router::new_for_test(RouterConfig::default(), vec!["claude".to_string()]);
6410
6411        let id = store
6412            .upsert_external(&crate::store::UpsertExternal {
6413                repo: "inactive/repo",
6414                ext_id: "459",
6415                title: "Rebroadcast blocked, no routable agents",
6416                body: "",
6417                author: "",
6418                url: "",
6419                labels: &[],
6420                origin: "github",
6421            })
6422            .await
6423            .unwrap();
6424        store.update_status(id, TaskStatus::Blocked).await.unwrap();
6425        store
6426            .set_fields(
6427                id,
6428                &[("block_reason", serde_json::json!(REBROADCAST_BLOCK_REASON))],
6429            )
6430            .await
6431            .unwrap();
6432        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
6433            .bind(id)
6434            .execute(store.pool())
6435            .await
6436            .unwrap();
6437
6438        auto_recover_rebroadcast_blocked_tasks_global(
6439            &router,
6440            &task_manager,
6441            &store,
6442            &sweep_repos(),
6443        )
6444        .await
6445        .unwrap();
6446
6447        let task = store.get(id).await.unwrap();
6448        assert_eq!(task.status, TaskStatus::Blocked);
6449        assert_eq!(task.block_reason.as_deref(), Some(REBROADCAST_BLOCK_REASON));
6450    }
6451}