Skip to main content

orch/engine/
sync.rs

1//! Sync tick — periodic operations that run less frequently than the core tick.
2//!
3//! The sync tick runs every ~45 seconds and handles:
4//! - Worktree cleanup for done tasks
5//! - Merged-PR detection
6//! - @mention scanning and internal task creation
7//! - PR review processing and follow-up dispatch
8//! - Stale InReview detection
9//! - Owner /slash command scanning
10//! - Skill repository syncing
11
12use super::EngineConfig;
13use super::cleanup::{check_merged_prs, cleanup_done_worktrees, prune_old_run_output};
14use super::commands::{CommandOutcome, CommandStoreOps, parse_command, validate_and_run_command};
15use super::review_poll::review_open_prs;
16use crate::backends::ExternalId;
17use crate::backends::ExternalTask;
18use crate::backends::{ExternalBackend, Mention, Status};
19use crate::cmd::CommandErrorContext;
20use crate::config;
21use crate::engine;
22use crate::engine::auto_merge::{is_pr_behind, required_checks_state};
23use crate::engine::cleanup::cleanup_task_worktree;
24use crate::engine::commands::CommandGh;
25use crate::engine::commands::OwnerCommand;
26use crate::engine::cooldown;
27use crate::engine::cooldown::{
28    cooldown_reason, github_circuit_remaining_secs, is_agent_in_cooldown, is_github_circuit_open,
29    is_model_in_cooldown, refresh_degraded_agents,
30};
31use crate::engine::dependencies;
32use crate::engine::router::Router;
33use crate::engine::router::RouterConfig;
34use crate::engine::router::config as router_config;
35use crate::engine::runner::agents::patterns;
36use crate::engine::runner::git_ops;
37use crate::engine::tasks;
38use crate::engine::tasks::{TaskManager, status_to_task_status};
39use crate::github::http::GhHttp;
40use crate::github::types::extract_issue_number_from_url;
41use crate::home;
42use crate::store;
43use crate::store::Task;
44use crate::store::TaskRun;
45use crate::store::TaskStatus;
46use crate::store::TaskStore;
47use crate::tmux::TmuxManager;
48use dashmap::{DashMap, DashSet};
49use futures::future;
50use std::collections::HashMap;
51use std::collections::HashSet;
52use std::path::PathBuf;
53use std::sync::Arc;
54use std::time::Duration;
55use std::time::Instant;
56use tokio::fs;
57use tokio::sync::OnceCell;
58use tokio::sync::RwLock;
59use tokio::time;
60
61/// Read a KV value from the store.
62///
63/// Returns `None` if the store is unavailable, the key doesn't exist, OR if
64/// the store query fails. Use [`try_kv_get_prefer_store`] if you need to
65/// distinguish between "key not found" and actual errors.
66async fn kv_get_prefer_store(store: Option<&Arc<TaskStore>>, key: &str) -> Option<String> {
67    match try_kv_get_prefer_store(store, key).await {
68        Ok(opt) => opt,
69        Err(e) => {
70            tracing::error!(key, err = %e, "kv_get failed");
71            None
72        }
73    }
74}
75
76/// Try to read a KV value from the store, returning errors instead of
77/// silently discarding them.
78///
79/// Returns `Ok(None)` if the store is unavailable or the key doesn't exist.
80/// Returns `Err` if the store query fails.
81async fn try_kv_get_prefer_store(
82    store: Option<&Arc<TaskStore>>,
83    key: &str,
84) -> anyhow::Result<Option<String>> {
85    if let Some(s) = store {
86        Ok(s.kv_get(key).await?)
87    } else {
88        Ok(None)
89    }
90}
91
92/// Write a KV value to the store.
93///
94/// Logs warnings on failure but provides no way for callers to know if the
95/// write succeeded. Use [`try_kv_set_prefer_store`] if you need to handle
96/// errors or guarantee persistence (e.g., for circuit breaker state).
97async fn kv_set_prefer_store(store: Option<&Arc<TaskStore>>, key: &str, value: &str) {
98    if let Err(e) = try_kv_set_prefer_store(store, key, value).await {
99        tracing::error!(key, err = %e, "kv_set failed");
100    }
101}
102
103/// Try to write a KV value to the store, returning errors instead of
104/// silently logging them.
105///
106/// Returns `Ok(())` if the store is unavailable (best-effort) or if the
107/// write succeeds. Returns `Err` if the store query fails.
108async fn try_kv_set_prefer_store(
109    store: Option<&Arc<TaskStore>>,
110    key: &str,
111    value: &str,
112) -> anyhow::Result<()> {
113    if let Some(s) = store {
114        s.kv_set(key, value).await?;
115    }
116    Ok(())
117}
118
119#[derive(Debug, Clone)]
120pub(crate) struct ReviewTaskSnapshot {
121    pub external: ExternalTask,
122    pub stored: Task,
123}
124
125async fn prefetch_review_tasks(
126    store: &Arc<TaskStore>,
127    repo: &str,
128) -> anyhow::Result<(
129    Vec<ReviewTaskSnapshot>,
130    Vec<ReviewTaskSnapshot>,
131    Vec<ReviewTaskSnapshot>,
132)> {
133    // Approved tasks wait on the same merge path as a review in progress
134    let mut in_review = store.list_by_status(repo, TaskStatus::InReview).await?;
135    in_review.extend(store.list_by_status(repo, TaskStatus::Approved).await?);
136    let in_review = in_review
137        .into_iter()
138        .map(|stored| ReviewTaskSnapshot {
139            external: tasks::store_task_to_external(&stored),
140            stored,
141        })
142        .collect();
143    let needs_review = store
144        .list_by_status(repo, TaskStatus::NeedsReview)
145        .await?
146        .into_iter()
147        .map(|stored| ReviewTaskSnapshot {
148            external: tasks::store_task_to_external(&stored),
149            stored,
150        })
151        .collect();
152
153    // Also prefetch blocked tasks that have a branch or PR number recorded.
154    // These can be merged externally (or manually) but previously were invisible
155    // to the merged-PR checker, leaving tasks stuck in Blocked after merges.
156    let blocked_candidates = store
157        .list_by_status(repo, TaskStatus::Blocked)
158        .await?
159        .into_iter()
160        .filter(|stored| stored.pr_number.is_some() || !stored.branch.is_empty())
161        .map(|stored| ReviewTaskSnapshot {
162            external: tasks::store_task_to_external(&stored),
163            stored,
164        })
165        .collect();
166
167    Ok((in_review, needs_review, blocked_candidates))
168}
169
170async fn fetch_comments_since(
171    backend: &Arc<dyn ExternalBackend>,
172    since: &str,
173) -> anyhow::Result<Vec<Mention>> {
174    backend.get_mentions(since).await
175}
176
177fn filter_mentions_by_since(comments: &[Mention], since: &str) -> Vec<Mention> {
178    comments
179        .iter()
180        .filter(|comment| comment.created_at.as_str() > since)
181        .cloned()
182        .collect()
183}
184
185#[derive(Debug, Clone, Copy, PartialEq, Eq)]
186enum FailureCategory {
187    FalseFailure,
188    RateLimit,
189    CreditExhausted,
190    SilentExit0,
191    Timeout,
192    ConnectionError,
193    CliFlagError,
194    AllAgentsExhausted,
195    ParseError,
196    MaxAttempts,
197    PushFailed,
198    PrCreateFailed,
199    ModelUnavailable,
200    Unknown,
201}
202
203impl FailureCategory {
204    /// Whether auto-unblock may put the task back in the queue on its own.
205    ///
206    /// `MaxAttempts` is deliberately absent: the task already ran the configured
207    /// number of times, so an automatic retry just restarts the loop that produced
208    /// the block. It stays blocked until `orch task unblock` or `orch task retry`
209    /// (#3783)
210    fn is_recoverable(self) -> bool {
211        matches!(
212            self,
213            FailureCategory::FalseFailure
214                | FailureCategory::RateLimit
215                | FailureCategory::CreditExhausted
216                | FailureCategory::SilentExit0
217                | FailureCategory::Timeout
218                | FailureCategory::ConnectionError
219                | FailureCategory::CliFlagError
220                | FailureCategory::AllAgentsExhausted
221                | FailureCategory::ParseError
222                | FailureCategory::ModelUnavailable
223        )
224    }
225
226    fn as_str(self) -> &'static str {
227        match self {
228            FailureCategory::FalseFailure => "FalseFailure",
229            FailureCategory::RateLimit => "RateLimit",
230            FailureCategory::CreditExhausted => "CreditExhausted",
231            FailureCategory::SilentExit0 => "SilentExit0",
232            FailureCategory::Timeout => "Timeout",
233            FailureCategory::ConnectionError => "ConnectionError",
234            FailureCategory::CliFlagError => "CliFlagError",
235            FailureCategory::AllAgentsExhausted => "AllAgentsExhausted",
236            FailureCategory::ParseError => "ParseError",
237            FailureCategory::MaxAttempts => "MaxAttempts",
238            FailureCategory::PushFailed => "PushFailed",
239            FailureCategory::PrCreateFailed => "PrCreateFailed",
240            FailureCategory::ModelUnavailable => "ModelUnavailable",
241            FailureCategory::Unknown => "Unknown",
242        }
243    }
244}
245
246fn classify_failure(error: &str, outcome: &str) -> FailureCategory {
247    let lower = error.to_lowercase();
248
249    if outcome == "timeout" || lower.contains("timeout") {
250        return FailureCategory::Timeout;
251    }
252
253    // Credit exhaustion is a transient condition (credits replenish, billing cycles
254    // reset). Classifying it as recoverable allows auto-unblock to retry once the
255    // credit-related cooldown expires, instead of permanently blocking the task.
256    //
257    // Checked before the generic rate-limit branch below: billing-cycle exhaustion
258    // messages (e.g. Kimi's "usage limit for this billing cycle") also contain
259    // "usage limit"/"rate limit" substrings, so checking outcome/content-based
260    // rate-limit patterns first would misclassify them as plain RateLimit (issue
261    // #3529). `outcome` also carries the specific string from `classify_run_outcome`
262    // when the task_run already recorded it.
263    if outcome == "billing_cycle_exhausted"
264        || outcome == "credit_exhausted"
265        || outcome == "org_level_disabled"
266        || cooldown::detect_credit_exhaustion(&lower).is_some()
267    {
268        return FailureCategory::CreditExhausted;
269    }
270
271    if outcome == "rate_limit" || lower.contains("rate limit") || lower.contains("usage limit") {
272        return FailureCategory::RateLimit;
273    }
274
275    if error.trim().is_empty() {
276        return FailureCategory::FalseFailure;
277    }
278
279    if lower.contains("all agents exhausted") {
280        return FailureCategory::AllAgentsExhausted;
281    }
282
283    if lower.contains("max attempts") || lower.contains("exceeded max attempts") {
284        return FailureCategory::MaxAttempts;
285    }
286
287    if lower.contains("output-format=stream-json") && lower.contains("requires --verbose") {
288        return FailureCategory::CliFlagError;
289    }
290
291    if lower.contains("model unavailable")
292        || lower.contains("model not found")
293        || (lower.contains("does not exist") && lower.contains("model"))
294    {
295        return FailureCategory::ModelUnavailable;
296    }
297
298    if lower.contains("push failed") {
299        return FailureCategory::PushFailed;
300    }
301
302    if lower.contains("create pr failed")
303        || lower.contains("failed to create pull request")
304        || lower.contains("pull request creation failed")
305    {
306        return FailureCategory::PrCreateFailed;
307    }
308
309    if lower.contains("exit 0") {
310        return FailureCategory::SilentExit0;
311    }
312
313    if outcome == "parse_error"
314        || outcome == "truncated"
315        || lower.contains("invalid response")
316        || lower.contains("parse error")
317        || lower.contains("truncated")
318    {
319        return FailureCategory::ParseError;
320    }
321
322    if patterns::detect_network_error(&lower).is_some() {
323        return FailureCategory::ConnectionError;
324    }
325
326    FailureCategory::Unknown
327}
328
329fn classify_failure_from_run(run: &TaskRun) -> Option<FailureCategory> {
330    // Completed successfully — no failure to classify.
331    if run.outcome == "success" && run.error.trim().is_empty() {
332        return None;
333    }
334
335    // Skip runs with no meaningful data. Some runs are created then never
336    // populated (aborted/stale work). Treat these as non-failures so they
337    // don't block auto-unblock logic which considers recent failures.
338    // Historically an all-NULL/empty run would be promoted to Unknown which
339    // prevented auto-unblock. Ensure we early-return None for truly empty
340    // runs (no outcome, no error, no exit_code, no output).
341    if run.outcome.trim().is_empty()
342        && run.error.trim().is_empty()
343        && run.exit_code.is_none()
344        && run.stdout.trim().is_empty()
345        && run.stderr.trim().is_empty()
346    {
347        return None;
348    }
349
350    let has_output = !run.stdout.trim().is_empty() || !run.stderr.trim().is_empty();
351    if run.exit_code == Some(0) && !has_output && run.outcome != "success" {
352        return Some(FailureCategory::SilentExit0);
353    }
354
355    let mut category = classify_failure(&run.error, &run.outcome);
356
357    // Promote a bare "false failure" to Unknown only when we have a
358    // concrete non-zero exit code or other signs the run truly failed.
359    // Do NOT promote when exit_code is None (incomplete run) — those are
360    // handled above by skipping. This prevents NULL/aborted runs from
361    // turning recoverable FalseFailure into non-recoverable Unknown.
362    // Only promote when we have explicit evidence: a non-zero exit code
363    // or non-empty stdout/stderr. If exit_code is None we keep FalseFailure
364    // so the run remains recoverable.
365    if category == FailureCategory::FalseFailure {
366        let has_nonzero_exit = run.exit_code.is_some() && run.exit_code != Some(0);
367        let has_output = !run.stdout.trim().is_empty() || !run.stderr.trim().is_empty();
368        if has_nonzero_exit || has_output {
369            category = FailureCategory::Unknown;
370        }
371    }
372
373    Some(category)
374}
375
376/// Resolve the parent task ID for a mention.
377///
378/// - If the mention is on a PR, look up the task that owns that PR via `pr_number`.
379/// - If the mention is on an issue, look up the task by external_id (issue number).
380///
381/// Returns:
382/// - `Ok(Some(id))` — parent found
383/// - `Ok(None)` — no matching parent (genuine miss, e.g. unknown issue/PR)
384/// - `Err(_)` — lookup failed (DB error); caller should defer/retry
385async fn resolve_mention_parent_id(
386    store: &TaskStore,
387    repo: &str,
388    issue_num: &str,
389    is_pr: bool,
390) -> anyhow::Result<Option<i64>> {
391    if is_pr {
392        if let Ok(pr_num) = issue_num.parse::<i32>() {
393            store.resolve_id_by_pr_number(repo, pr_num).await
394        } else {
395            Ok(None)
396        }
397    } else {
398        store.resolve_id_by_external(repo, issue_num).await
399    }
400}
401
402/// Create a sentinel mention task in the store and advance the cursor.
403async fn record_mention_task(
404    store: &TaskStore,
405    repo: &str,
406    mention: &Mention,
407    title: &str,
408    body: &str,
409    parent_id: Option<i64>,
410    cursor: &mut MentionCursor,
411) -> bool {
412    match store
413        .create_internal(repo, title, body, "mention", &mention.id, parent_id)
414        .await
415    {
416        Ok(task_id) => {
417            tracing::info!(task_id, mention_id = %mention.id, "created mention task");
418            cursor.advance(&mention.created_at);
419            true
420        }
421        Err(e) => {
422            tracing::warn!(mention_id = %mention.id, err = %e, "failed to create mention task");
423            false
424        }
425    }
426}
427
428/// Advance the mention cursor if the timestamp is newer.
429fn advance_cursor(last_safe_ts: &mut Option<String>, ts: &str) {
430    if last_safe_ts.as_deref() < Some(ts) {
431        *last_safe_ts = Some(ts.to_string());
432    }
433}
434
435#[derive(Debug, Default)]
436struct MentionCursor {
437    last_safe_ts: Option<String>,
438    blocked_by_gap: bool,
439}
440
441impl MentionCursor {
442    fn advance(&mut self, ts: &str) {
443        if self.blocked_by_gap {
444            return;
445        }
446        advance_cursor(&mut self.last_safe_ts, ts);
447    }
448
449    fn block_on_gap(&mut self) {
450        self.blocked_by_gap = true;
451    }
452
453    fn into_last_safe_ts(self) -> Option<String> {
454        self.last_safe_ts
455    }
456}
457
458/// What to do with a comment in the unified scan loop.
459#[derive(Debug, PartialEq)]
460pub(crate) enum CommentAction {
461    /// Already processed or not relevant — skip.
462    Skip,
463    /// Slash command without @mention — execute the command.
464    ExecuteCommand {
465        command: OwnerCommand,
466        issue_num: String,
467    },
468    /// @mention with a slash command — execute and record sentinel task.
469    ExecuteCommandForMention {
470        command: OwnerCommand,
471        issue_num: String,
472    },
473    /// @mention without command — create internal task for agent to respond.
474    CreateMentionTask { issue_num: Option<String> },
475}
476
477/// Classify a comment into the appropriate action.
478///
479/// Pure function — no I/O, no async. Determines what the scan loop should do
480/// with each comment based on its content and whether it was already processed.
481pub(crate) fn classify_comment(
482    body: &str,
483    issue_url: Option<&str>,
484    current_user: &str,
485    already_processed: bool,
486) -> CommentAction {
487    if already_processed {
488        return CommentAction::Skip;
489    }
490
491    let is_mention =
492        body.contains(current_user) || body.contains("@orch") || body.contains("@orchestrator");
493    let command = parse_command(body);
494    let issue_num = issue_url.and_then(extract_issue_number_from_url);
495
496    match (command, is_mention, issue_num) {
497        (Some(cmd), true, Some(num)) => CommentAction::ExecuteCommandForMention {
498            command: cmd,
499            issue_num: num,
500        },
501        (Some(cmd), false, Some(num)) => CommentAction::ExecuteCommand {
502            command: cmd,
503            issue_num: num,
504        },
505        // Command but no issue URL — if it's a mention, create task; otherwise skip
506        (Some(_), true, None) => CommentAction::CreateMentionTask { issue_num: None },
507        // Mention without command — create task for agent
508        (None, true, issue_num) => CommentAction::CreateMentionTask { issue_num },
509        // Neither mention nor command — skip
510        (Some(_), false, None) | (None, false, _) => CommentAction::Skip,
511    }
512}
513
514/// Handle a slash command from a mention. Records a sentinel task and advances
515/// the cursor on success; logs a warning and returns early if the mention has
516/// no valid issue URL.
517///
518/// Execute a slash command mentioned by `@orch` on a comment.
519///
520/// Returns:
521/// - `Ok(true)` — command processed and cursor may continue advancing
522/// - `Ok(false)` — mention task insert failed, or validation hit a transient
523///   GitHub error; caller must stop cursor advancement so the next tick retries
524/// - `Err(_)` — DB lookup failed for parent_id; caller must NOT advance cursor
525///   so this mention is retried on the next sync
526#[expect(
527    clippy::too_many_arguments,
528    reason = "the sync tick hands over its shared state piece by piece"
529)]
530async fn handle_slash_command(
531    backend: &Arc<dyn ExternalBackend>,
532    store: Option<&Arc<TaskStore>>,
533    repo: &str,
534    task_manager: &Arc<TaskManager>,
535    gh: &dyn CommandGh,
536    mention: &Mention,
537    command: &OwnerCommand,
538    cursor: &mut MentionCursor,
539) -> anyhow::Result<bool> {
540    let Some(issue_num) = mention
541        .issue_url
542        .as_deref()
543        .and_then(extract_issue_number_from_url)
544    else {
545        tracing::warn!(comment_id = %mention.id, "slash command without valid issue number");
546        return Ok(true);
547    };
548
549    let store_opt: Option<Arc<dyn CommandStoreOps>> =
550        store.map(|s| Arc::clone(s) as Arc<dyn CommandStoreOps>);
551    let outcome = validate_and_run_command(
552        backend,
553        gh,
554        repo,
555        &issue_num,
556        command,
557        &mention.author,
558        &store_opt,
559        task_manager,
560    )
561    .await;
562
563    // Extract is_pr from the outcome (avoids a second get_issue call)
564    let is_pr = match &outcome {
565        CommandOutcome::Executed { is_pr }
566        | CommandOutcome::NotOpen { is_pr }
567        | CommandOutcome::NotCollaborator { is_pr } => *is_pr,
568        CommandOutcome::FetchFailed | CommandOutcome::CollaboratorCheckFailed => {
569            // Nothing was executed and nothing was posted to GitHub. Leave the cursor
570            // before this comment so the next tick retries the command.
571            tracing::warn!(
572                comment_id = %mention.id,
573                command = %command,
574                "deferring @orch slash command after a transient GitHub error"
575            );
576            return Ok(false);
577        }
578    };
579
580    // Record a sentinel task for the mention (with correct parent_id).
581    // On DB lookup failure, defer so the mention can be retried on next sync.
582    if let Some(s) = store {
583        let parent_id = match resolve_mention_parent_id(s, repo, &issue_num, is_pr).await {
584            Ok(parent_id) => parent_id,
585            Err(e) => {
586                tracing::warn!(
587                    comment_id = %mention.id,
588                    err = %e,
589                    "deferring mention task due to parent lookup failure"
590                );
591                return Err(e);
592            }
593        };
594
595        let (title, body) = match &outcome {
596            CommandOutcome::NotOpen { .. } => (
597                format!(
598                    "Skipped @orch command on closed #{issue_num} from @{}",
599                    mention.author
600                ),
601                format!(
602                    "Mention by @{}:\n\n{}\n\n**Skipped:** target is closed",
603                    mention.author, mention.body
604                ),
605            ),
606            CommandOutcome::NotCollaborator { .. } => (
607                format!(
608                    "Skipped @orch command on #{issue_num} from non-collaborator @{}",
609                    mention.author
610                ),
611                format!(
612                    "Mention by @{}:\n\n{}\n\n**Skipped:** author is not a collaborator",
613                    mention.author, mention.body
614                ),
615            ),
616            CommandOutcome::Executed { .. } => (
617                format!(
618                    "Respond to mention by @{} on task #{issue_num} — command executed",
619                    mention.author
620                ),
621                format!(
622                    "Mention by @{} on #{issue_num}:\n\n{}\n\n**Status:** Command executed",
623                    mention.author, mention.body
624                ),
625            ),
626            CommandOutcome::FetchFailed | CommandOutcome::CollaboratorCheckFailed => {
627                tracing::debug!("skipping mention task record due to earlier command failure");
628                return Ok(true);
629            }
630        };
631
632        let created = record_mention_task(s, repo, mention, &title, &body, parent_id, cursor).await;
633        if !created {
634            return Ok(false);
635        }
636    }
637    Ok(true)
638}
639
640fn auto_unblock_cooldown_elapsed(count: i32, last_at: &str) -> bool {
641    if count == 0 {
642        return true;
643    }
644
645    let required = match count {
646        1 => chrono::Duration::minutes(30),
647        2 => chrono::Duration::hours(2),
648        _ => return false,
649    };
650
651    if last_at.trim().is_empty() {
652        return true;
653    }
654
655    match chrono::DateTime::parse_from_rfc3339(last_at) {
656        Ok(ts) => chrono::Utc::now() - ts.with_timezone(&chrono::Utc) >= required,
657        Err(_) => true,
658    }
659}
660
661fn is_ci_failure_block(task: &Task) -> bool {
662    let reason = task.block_reason.as_deref().unwrap_or("");
663    reason.contains("CI failure limit") || reason.contains("CI checks timed out")
664}
665
666fn ci_failure_unblock_cooldown_elapsed(task: &Task) -> bool {
667    if task.auto_unblock_count == 0 {
668        return true;
669    }
670
671    // Short cooldowns — the PR may close quickly after the first check.
672    // We never permanently block (no `_ => return false`): eventually the PR
673    // will close or a human will intervene.
674    let required = match task.auto_unblock_count {
675        1 => chrono::Duration::minutes(10),
676        2 => chrono::Duration::hours(1),
677        3 => chrono::Duration::hours(6),
678        _ => chrono::Duration::hours(24),
679    };
680
681    let last_at = &task.auto_unblock_last_at;
682    if last_at.trim().is_empty() {
683        return true;
684    }
685
686    match chrono::DateTime::parse_from_rfc3339(last_at) {
687        Ok(ts) => chrono::Utc::now() - ts.with_timezone(&chrono::Utc) >= required,
688        Err(_) => true,
689    }
690}
691
692async fn try_unblock_ci_failure_task(
693    gh: &GhHttp,
694    repo: &str,
695    task_manager: &Arc<TaskManager>,
696    store: &Arc<TaskStore>,
697    task: &Task,
698    auto_merge_in_flight: &Arc<DashSet<String>>,
699) -> anyhow::Result<bool> {
700    let Some(pr_number) = task.pr_number.and_then(|n| u64::try_from(n).ok()) else {
701        tracing::debug!(
702            task_id = task.id,
703            "CI failure blocked task has no PR number"
704        );
705        return Ok(false);
706    };
707
708    let pr = match gh.get_pr(repo, pr_number).await {
709        Ok(p) => p,
710        Err(e) => {
711            tracing::warn!(task_id = task.id, pr_number, err = %e, "failed to get PR status for CI failure block");
712            return Ok(false);
713        }
714    };
715
716    tracing::debug!(
717        task_id = task.id,
718        pr_number,
719        merged = pr.merged,
720        state = ?pr.state,
721        "verifying PR state for CI failure blocked task"
722    );
723
724    if pr.merged.unwrap_or(false) || pr.state.eq_ignore_ascii_case("closed") {
725        tracing::info!(
726            task_id = task.id,
727            pr_number,
728            merged = pr.merged,
729            state = %pr.state,
730            "PR merged/closed — unblocking CI failure blocked task"
731        );
732        let ext_id = task
733            .external_id
734            .clone()
735            .unwrap_or_else(|| format!("internal:{}", task.id));
736        // Update the store directly by the already-resolved numeric id instead of going
737        // through `task_manager.update_task_status`, which resolves by `self.repo` — this
738        // sweep runs cross-repo, so `task.repo` may not match the caller's own repo scope.
739        task_manager
740            .mark_task_done_by_store_id(task.id, &task.repo, &ExternalId(ext_id))
741            .await?;
742        let fields = vec![
743            ("block_reason", serde_json::Value::Null),
744            ("auto_unblock_count", serde_json::json!(0)),
745            ("auto_unblock_last_at", serde_json::json!("")),
746            ("auto_unblock_last_reason", serde_json::json!("")),
747        ];
748        store.set_fields(task.id, &fields).await?;
749        return Ok(true);
750    }
751
752    // Skip if a recovery merge is already in flight for this task. The poll loop
753    // can take up to 10 minutes, and the same task may be seen by both the per-repo
754    // sync_tick and the global sweep in the same tick window.
755    let task_id_str = task
756        .external_id
757        .clone()
758        .unwrap_or_else(|| format!("internal:{}", task.id));
759    if auto_merge_in_flight.contains(&task_id_str) {
760        tracing::debug!(
761            task_id = task.id,
762            pr_number,
763            "CI-failure recovery merge already in flight — skipping"
764        );
765        return Ok(false);
766    }
767
768    // Give the PR an active chance to recover: merge the base branch into the PR
769    // branch to re-trigger CI, then re-enable GitHub auto-merge. GitHub's queue
770    // can silently stall even when checks are green and the branch is up to date,
771    // so after arming auto-merge we poll for completion and fall back to a direct
772    // merge once the PR is mergeable and CI is passing.
773    let branch_update_result = gh.update_pr_branch(repo, pr_number).await;
774    match branch_update_result {
775        Ok(()) => {
776            tracing::info!(
777                task_id = task.id,
778                pr_number,
779                "updated CI-failure-blocked PR branch against base to retrigger CI"
780            );
781        }
782        Err(ref e) => {
783            // warn, not debug, so a silently-broken corrective retry doesn't look like a working one (#3561)
784            tracing::warn!(
785                task_id = task.id,
786                pr_number,
787                err = %e,
788                outcome = "branch_update_failed",
789                "failed to update CI-failure-blocked PR branch (may already be up to date)"
790            );
791        }
792    }
793
794    if let Err(e) = gh.enable_auto_merge(repo, pr_number).await {
795        tracing::warn!(task_id = task.id, pr_number, err = %e, "failed to re-enable auto-merge after branch update");
796    }
797
798    // Record this attempt so the cooldown advances even though the actual
799    // poll/merge now runs in the background.
800    let pr_state = pr.state.as_str();
801    let new_reason = if let Err(ref e) = branch_update_result {
802        format!(
803            "CI failure limit reached during auto-merge (PR #{pr_number} still open, state: {pr_state}, branch update failed: {e})"
804        )
805    } else {
806        format!(
807            "CI failure limit reached during auto-merge (PR #{pr_number} still open, state: {pr_state})"
808        )
809    };
810    let now = chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string();
811    // Increment auto_unblock_count so the cooldown backoff (24h, 3d, never) kicks in
812    // and we don't hammer the GitHub API every tick for a PR that stays open.
813    let new_count = task.auto_unblock_count.saturating_add(1);
814    let fields = vec![
815        ("block_reason", serde_json::json!(&new_reason)),
816        ("auto_unblock_count", serde_json::json!(new_count)),
817        ("auto_unblock_last_at", serde_json::json!(&now)),
818    ];
819    // Use set_fields_silent so the probe write does not bump updated_at on stale
820    // blocked tasks — triage queries keyed off updated_at must only see real
821    // status/result changes, not periodic re-checks of an open PR.
822    store.set_fields_silent(task.id, &fields).await?;
823
824    // Spawn the poll and direct-merge fallback in the background so the
825    // recovery sweep does not block the main tick loop (or sync_tick) while
826    // CI runs. The in-flight guard prevents double-spawns across ticks.
827    auto_merge_in_flight.insert(task_id_str.clone());
828    let gh_clone = gh.clone();
829    let repo_string = repo.to_string();
830    let task_manager_clone = Arc::clone(task_manager);
831    let store_clone = Arc::clone(store);
832    let task_clone = task.clone();
833    let in_flight_clone = Arc::clone(auto_merge_in_flight);
834
835    tokio::spawn(async move {
836        let _guard = scopeguard::guard((), |()| {
837            in_flight_clone.remove(&task_id_str);
838        });
839
840        match poll_and_merge_recovered_pr(&gh_clone, &repo_string, pr_number, task_clone.id).await {
841            Ok(true) => {
842                tracing::info!(
843                    task_id = task_clone.id,
844                    pr_number,
845                    "CI-failure-blocked PR merged during recovery sweep"
846                );
847                let ext_id = task_clone
848                    .external_id
849                    .clone()
850                    .unwrap_or_else(|| format!("internal:{}", task_clone.id));
851                let fields = [
852                    ("block_reason", serde_json::Value::Null),
853                    ("auto_unblock_count", serde_json::json!(0)),
854                    ("auto_unblock_last_at", serde_json::json!("")),
855                    ("auto_unblock_last_reason", serde_json::json!("")),
856                ];
857                if let Err(e) = task_manager_clone
858                    .update_task_status_and_result_by_store_id(
859                        task_clone.id,
860                        &task_clone.repo,
861                        &ExternalId(ext_id),
862                        Status::Done,
863                        &fields,
864                    )
865                    .await
866                {
867                    tracing::warn!(
868                        task_id = task_clone.id,
869                        pr_number,
870                        err = %e,
871                        "failed to mark CI-failure-blocked task done after recovery merge"
872                    );
873                }
874                if let Err(e) =
875                    cleanup_task_worktree(&format!("{}", task_clone.id), &repo_string, &store_clone)
876                        .await
877                {
878                    tracing::warn!(
879                        task_id = task_clone.id,
880                        pr_number,
881                        err = %e,
882                        "cleanup after CI-failure recovery merge failed"
883                    );
884                }
885            }
886            Ok(false) => {
887                tracing::debug!(
888                    task_id = task_clone.id,
889                    pr_number,
890                    "CI-failure-blocked PR not merged this recovery tick — poll completed without merge"
891                );
892            }
893            Err(e) => {
894                tracing::warn!(
895                    task_id = task_clone.id,
896                    pr_number,
897                    err = %e,
898                    "error while polling CI-failure-blocked PR for recovery"
899                );
900            }
901        }
902    });
903
904    Ok(false)
905}
906
907/// Poll a recovered CI-failure PR until it merges or its checks settle, then
908/// attempt a direct merge if GitHub's auto-merge queue has not completed it.
909///
910/// GitHub's `enablePullRequestAutoMerge` hands the PR off to GitHub's queue and
911/// returns immediately; the queue can stall silently. This helper mirrors the
912/// primary `auto_merge_pr` polling pattern but is scoped to the recovery sweep:
913/// it waits for required checks to pass, verifies the PR is not behind the base
914/// branch, and calls `gh.merge_pr()` directly when the PR is ready but still
915/// open. Returns `Ok(true)` when the PR is merged/closed, `Ok(false)` when the
916/// PR needs to wait for a future tick.
917async fn poll_and_merge_recovered_pr(
918    gh: &GhHttp,
919    repo: &str,
920    pr_number: u64,
921    task_id: i64,
922) -> anyhow::Result<bool> {
923    // The branch can fall behind while CI runs. Retry the update-branch call a
924    // few times inside the poll loop instead of exiting and waiting for the next
925    // 24-hour recovery cooldown.
926    const MAX_BEHIND_RETRIES: u32 = 3;
927
928    let max_wait_secs: u64 = config::get("workflow.ci_poll_max_wait_secs")
929        .ok()
930        .and_then(|s| s.parse().ok())
931        .unwrap_or(600);
932    let base_interval_secs: u64 = config::get("workflow.ci_poll_interval_secs")
933        .ok()
934        .and_then(|s| s.parse().ok())
935        .unwrap_or(15);
936    let max_wait = Duration::from_secs(max_wait_secs);
937    let start = Instant::now();
938    let mut poll_count: u32 = 0;
939    let mut behind_retries: u32 = 0;
940
941    let initial_pr = gh.get_pr(repo, pr_number).await?;
942    let base_branch = initial_pr.base.ref_.clone();
943    let required_contexts = gh
944        .get_required_status_check_contexts(repo, &base_branch)
945        .await?;
946    let repo_has_workflows = if required_contexts.is_empty() {
947        gh.has_workflows(repo).await?
948    } else {
949        true
950    };
951
952    loop {
953        let pr = gh.get_pr(repo, pr_number).await?;
954        if pr.merged.unwrap_or(false) || pr.state.eq_ignore_ascii_case("closed") {
955            tracing::info!(
956                task_id = task_id,
957                pr_number,
958                "PR merged/closed during recovery poll"
959            );
960            return Ok(true);
961        }
962
963        if pr.mergeable == Some(false) {
964            tracing::info!(
965                task_id = task_id,
966                pr_number,
967                outcome = "poll_exited_without_merge",
968                reason = "merge_conflicts",
969                "PR has merge conflicts after recovery update — leaving for next recovery cycle"
970            );
971            return Ok(false);
972        }
973
974        if is_pr_behind(&pr) {
975            if behind_retries >= MAX_BEHIND_RETRIES {
976                tracing::info!(
977                    task_id = task_id,
978                    pr_number,
979                    behind_retries,
980                    max_behind_retries = MAX_BEHIND_RETRIES,
981                    outcome = "poll_exited_without_merge",
982                    reason = "behind_retry_exhausted",
983                    "PR still behind base after recovery retries — leaving for next recovery cycle"
984                );
985                return Ok(false);
986            }
987            behind_retries += 1;
988            match gh.update_pr_branch(repo, pr_number).await {
989                Ok(()) => {
990                    tracing::info!(
991                        task_id = task_id,
992                        pr_number,
993                        behind_retries,
994                        "PR was behind base — updated branch and continuing recovery poll"
995                    );
996                    continue;
997                }
998                Err(e) => {
999                    tracing::warn!(
1000                        task_id = task_id,
1001                        pr_number,
1002                        behind_retries,
1003                        err = %e,
1004                        outcome = "branch_update_failed",
1005                        "failed to update behind PR branch during recovery poll"
1006                    );
1007                    return Ok(false);
1008                }
1009            }
1010        }
1011
1012        let head_sha = pr.head.sha.clone();
1013        let (state, total, passing, failing, pending) = if required_contexts.is_empty() {
1014            gh.get_combined_status(
1015                repo,
1016                &head_sha,
1017                repo_has_workflows,
1018                pr.mergeable_state.as_deref(),
1019            )
1020            .await?
1021        } else {
1022            let check_runs = gh.get_check_runs(repo, &head_sha).await?;
1023            let statuses = gh.get_commit_status_contexts(repo, &head_sha).await?;
1024            required_checks_state(&required_contexts, &check_runs, &statuses)
1025        };
1026
1027        tracing::info!(
1028            task_id = task_id,
1029            pr_number,
1030            state = %state,
1031            total,
1032            passing,
1033            failing,
1034            pending,
1035            "CI recovery status check"
1036        );
1037
1038        match state.as_str() {
1039            "success" => {
1040                tracing::info!(
1041                    task_id = task_id,
1042                    pr_number,
1043                    "CI green after recovery — attempting direct merge"
1044                );
1045                match gh.merge_pr(repo, pr_number, true, &head_sha).await {
1046                    Ok(()) => {
1047                        tracing::info!(
1048                            task_id = task_id,
1049                            pr_number,
1050                            "direct merge succeeded after recovery"
1051                        );
1052                        return Ok(true);
1053                    }
1054                    Err(e) => {
1055                        let err = e.to_string().to_ascii_lowercase();
1056                        tracing::warn!(
1057                            task_id = task_id,
1058                            pr_number,
1059                            err = %e,
1060                            "direct merge failed after recovery"
1061                        );
1062                        // If the PR became merged between the get_pr call and the
1063                        // merge attempt, treat it as success rather than a failed tick.
1064                        if err.contains("already merged")
1065                            || err.contains("no merge commit")
1066                            || err.contains("pull request already merged")
1067                        {
1068                            return Ok(true);
1069                        }
1070                        // The head moved after we read it, and updating the branch would skip CI on the new commit
1071                        if err.contains("head branch was modified")
1072                            || err.contains("sha does not match")
1073                        {
1074                            tracing::info!(
1075                                task_id = task_id,
1076                                pr_number,
1077                                outcome = "poll_exited_without_merge",
1078                                reason = "head_moved",
1079                                "merge rejected because the head sha changed"
1080                            );
1081                            return Ok(false);
1082                        }
1083                        // GitHub may reject the merge because the branch fell behind
1084                        // again while CI was finishing. Retry the branch update inside
1085                        // the poll window instead of waiting for the next sweep.
1086                        if (err.contains("head branch is out of date")
1087                            || err.contains("behind")
1088                            || err.contains("409"))
1089                            && behind_retries < MAX_BEHIND_RETRIES
1090                        {
1091                            behind_retries += 1;
1092                            match gh.update_pr_branch(repo, pr_number).await {
1093                                Ok(()) => {
1094                                    tracing::info!(
1095                                        task_id = task_id,
1096                                        pr_number,
1097                                        behind_retries,
1098                                        "merge rejected due to out-of-date branch — updated and continuing recovery poll"
1099                                    );
1100                                    continue;
1101                                }
1102                                Err(update_err) => {
1103                                    tracing::warn!(
1104                                        task_id = task_id,
1105                                        pr_number,
1106                                        behind_retries,
1107                                        err = %update_err,
1108                                        outcome = "branch_update_failed",
1109                                        "failed to update out-of-date PR branch during recovery poll"
1110                                    );
1111                                }
1112                            }
1113                        }
1114                        tracing::info!(
1115                            task_id = task_id,
1116                            pr_number,
1117                            outcome = "poll_exited_without_merge",
1118                            reason = "direct_merge_failed",
1119                            "direct merge did not succeed — leaving for next recovery cycle"
1120                        );
1121                        return Ok(false);
1122                    }
1123                }
1124            }
1125            "failure" => {
1126                tracing::info!(
1127                    task_id = task_id,
1128                    pr_number,
1129                    outcome = "poll_exited_without_merge",
1130                    reason = "ci_failure",
1131                    "CI still failing after recovery — will retry later"
1132                );
1133                return Ok(false);
1134            }
1135            _ => {}
1136        }
1137
1138        if start.elapsed() >= max_wait {
1139            tracing::info!(
1140                task_id = task_id,
1141                pr_number,
1142                outcome = "poll_exited_without_merge",
1143                reason = "poll_timeout",
1144                "CI still pending after recovery poll timeout — will retry later"
1145            );
1146            return Ok(false);
1147        }
1148
1149        poll_count += 1;
1150        let multiplier = (2_u64).pow(poll_count.saturating_sub(1)).min(4);
1151        let sleep_secs = base_interval_secs.saturating_mul(multiplier);
1152        time::sleep(Duration::from_secs(sleep_secs)).await;
1153    }
1154}
1155
1156/// Exact `block_reason` written when a task's NeedsReview refire counter exhausts
1157/// during a rebroadcast escalation (`refire_and_escalate_stale_needs_review_global`).
1158/// Shared with `auto_recover_rebroadcast_blocked_tasks_global` so escalation and
1159/// recovery never drift apart on the literal string
1160const REBROADCAST_BLOCK_REASON: &str = "review agent rebroadcast escalated after repeated retries";
1161
1162/// Global sweep: auto-recover rebroadcast-blocked tasks across all repos.
1163///
1164/// When a NeedsReview refire counter exhausts, the task is blocked with
1165/// `REBROADCAST_BLOCK_REASON`. Once any review agent is routable again, this
1166/// sweep resets matching tasks to `NeedsReview` so the subscriber can pick them up.
1167/// Writes through the store via `*_by_store_id` helpers, not a repo-scoped
1168/// `TaskManager`. Called once per main tick from `tick_global_sweeps`.
1169///
1170/// Before resetting, the attached PR's live state is checked (mirroring
1171/// `try_unblock_ci_failure_task`): merged means the task is marked done, closed
1172/// unmerged means it stays blocked with an annotated reason, and only an open PR
1173/// (or no PR) falls through to `NeedsReview`. A 5-minute block age prevents
1174/// same-tick escalate/recover flapping. Reset-to-NeedsReview still requires at
1175/// least one routable review agent
1176pub(crate) async fn auto_recover_rebroadcast_blocked_tasks_global(
1177    router: &Router,
1178    task_manager: &Arc<TaskManager>,
1179    store: &Arc<TaskStore>,
1180    active_repos: &HashSet<String>,
1181) -> anyhow::Result<()> {
1182    const MIN_BLOCK_AGE_MINUTES: i64 = 5;
1183
1184    let all_blocked = match store.list_all_by_status_global(TaskStatus::Blocked).await {
1185        Ok(tasks) => tasks,
1186        Err(e) => {
1187            tracing::warn!(err = %e, "global rebroadcast-blocked sweep: failed to list blocked tasks");
1188            return Ok(());
1189        }
1190    };
1191
1192    let now = chrono::Utc::now();
1193    let candidates: Vec<_> = all_blocked
1194        .into_iter()
1195        .filter(|t| {
1196            if t.block_reason.as_deref() != Some(REBROADCAST_BLOCK_REASON) {
1197                return false;
1198            }
1199            // A repo outside the active set has no review subscriber, so a reset would loop
1200            if !active_repos.contains(&t.repo) {
1201                return false;
1202            }
1203            // Only recover tasks blocked long enough to rule out same-tick escalations.
1204            match chrono::DateTime::parse_from_rfc3339(&t.updated_at) {
1205                Ok(updated_at) => {
1206                    let age = now - updated_at.with_timezone(&chrono::Utc);
1207                    age.num_minutes() >= MIN_BLOCK_AGE_MINUTES
1208                }
1209                Err(_) => true,
1210            }
1211        })
1212        .collect();
1213
1214    if candidates.is_empty() {
1215        return Ok(());
1216    }
1217
1218    let gh = match GhHttp::new() {
1219        Ok(g) => Some(g),
1220        Err(e) => {
1221            tracing::warn!(err = %e, "global rebroadcast-blocked sweep: failed to create GhHttp — skipping PR-state checks this tick");
1222            None
1223        }
1224    };
1225
1226    // Skip reset-to-NeedsReview when no review agent is routable, re-firing
1227    // while cooled would immediately re-exhaust the counter
1228    let any_routable =
1229        !router.available_agents.is_empty() && router.healthy_agent_count("review") > 0;
1230
1231    for task in candidates {
1232        let ext_id = task
1233            .external_id
1234            .clone()
1235            .unwrap_or_else(|| format!("internal:{}", task.id));
1236
1237        if let (Some(gh), Some(pr_number)) =
1238            (&gh, task.pr_number.and_then(|n| u64::try_from(n).ok()))
1239        {
1240            match gh.get_pr(&task.repo, pr_number).await {
1241                Ok(pr) => {
1242                    if pr.merged.unwrap_or(false) {
1243                        tracing::info!(
1244                            task_id = task.id,
1245                            repo = %task.repo,
1246                            pr_number,
1247                            "global rebroadcast-blocked sweep: PR already merged — marking task done"
1248                        );
1249                        if let Err(e) = task_manager
1250                            .mark_task_done_by_store_id(
1251                                task.id,
1252                                &task.repo,
1253                                &ExternalId(ext_id.clone()),
1254                            )
1255                            .await
1256                        {
1257                            tracing::warn!(task_id = task.id, repo = %task.repo, err = %e, "global rebroadcast-blocked sweep: failed to mark merged task done");
1258                            continue;
1259                        }
1260                        let fields: &[(&str, serde_json::Value)] = &[
1261                            ("block_reason", serde_json::Value::Null),
1262                            ("needs_review_refires", serde_json::json!(0)),
1263                        ];
1264                        if let Err(e) = store.set_fields(task.id, fields).await {
1265                            tracing::warn!(task_id = task.id, err = %e, "global rebroadcast-blocked sweep: failed to clear fields after marking done");
1266                        }
1267                        continue;
1268                    }
1269                    if pr.state.eq_ignore_ascii_case("closed") {
1270                        tracing::info!(
1271                            task_id = task.id,
1272                            repo = %task.repo,
1273                            pr_number,
1274                            "global rebroadcast-blocked sweep: PR closed unmerged — leaving task blocked, not re-dispatching"
1275                        );
1276                        let new_reason = format!(
1277                            "{REBROADCAST_BLOCK_REASON} (PR #{pr_number} closed unmerged — no further auto-recovery)"
1278                        );
1279                        let fields: &[(&str, serde_json::Value)] =
1280                            &[("block_reason", serde_json::json!(new_reason))];
1281                        // set_fields_silent: this is a terminal annotation, not a real
1282                        // status change — don't bump updated_at so triage queries keyed
1283                        // off it aren't confused by a task that stays blocked forever.
1284                        if let Err(e) = store.set_fields_silent(task.id, fields).await {
1285                            tracing::warn!(task_id = task.id, err = %e, "global rebroadcast-blocked sweep: failed to annotate closed-unmerged block reason");
1286                        }
1287                        continue;
1288                    }
1289                    // PR still open — fall through to the reset-to-NeedsReview path.
1290                }
1291                Err(e) => {
1292                    tracing::warn!(task_id = task.id, repo = %task.repo, pr_number, err = %e, "global rebroadcast-blocked sweep: failed to get PR status — skipping this tick");
1293                    continue;
1294                }
1295            }
1296        }
1297
1298        if !any_routable {
1299            continue;
1300        }
1301
1302        if let Err(e) = task_manager
1303            .update_task_status_by_store_id(
1304                task.id,
1305                &task.repo,
1306                &ExternalId(ext_id.clone()),
1307                Status::NeedsReview,
1308            )
1309            .await
1310        {
1311            tracing::warn!(task_id = task.id, repo = %task.repo, err = %e, "global rebroadcast-blocked sweep: failed to reset task to needs_review — skipping");
1312            continue;
1313        }
1314
1315        let fields: &[(&str, serde_json::Value)] = &[
1316            ("block_reason", serde_json::Value::Null),
1317            ("needs_review_refires", serde_json::json!(0)),
1318        ];
1319        if let Err(e) = store.set_fields(task.id, fields).await {
1320            tracing::warn!(task_id = task.id, err = %e, "global rebroadcast-blocked sweep: failed to clear block_reason/needs_review_refires after status reset");
1321        }
1322
1323        tracing::info!(
1324            task_id = task.id,
1325            repo = %task.repo,
1326            ext_id = %ext_id,
1327            "global rebroadcast-blocked sweep: auto-recovered rebroadcast-blocked task — review agents now available"
1328        );
1329    }
1330
1331    Ok(())
1332}
1333
1334async fn auto_unblock_blocked_tasks(
1335    repo: &str,
1336    task_manager: &Arc<TaskManager>,
1337    store: &Arc<TaskStore>,
1338    dispatching: &Arc<DashMap<String, String>>,
1339    auto_merge_in_flight: &Arc<DashSet<String>>,
1340) -> anyhow::Result<()> {
1341    let blocked = match store.list_by_status(repo, TaskStatus::Blocked).await {
1342        Ok(tasks) => tasks,
1343        Err(e) => {
1344            tracing::warn!(err = %e, "failed to list blocked tasks — skipping auto-unblock this tick");
1345            return Ok(());
1346        }
1347    };
1348
1349    if blocked.is_empty() {
1350        return Ok(());
1351    }
1352
1353    // Batch-load all runs for blocked tasks in a single query instead of N individual queries.
1354    let task_ids: Vec<i64> = blocked.iter().map(|t| t.id).collect();
1355    let mut runs_by_task = match store.get_runs_batch(&task_ids).await {
1356        Ok(map) => map,
1357        Err(e) => {
1358            tracing::warn!(err = %e, "get_runs_batch failed — skipping auto-unblock this tick");
1359            return Ok(());
1360        }
1361    };
1362
1363    // Separate CI-failure blocked tasks from other blocked tasks.
1364    let (ci_failure_tasks, other_blocked): (Vec<_>, Vec<_>) =
1365        blocked.into_iter().partition(is_ci_failure_block);
1366
1367    // Handle CI-failure blocked tasks by verifying PR state.
1368    if !ci_failure_tasks.is_empty() {
1369        let gh = match GhHttp::new() {
1370            Ok(g) => Some(g),
1371            Err(e) => {
1372                tracing::warn!(err = %e, "failed to create GhHttp — skipping CI failure unblock this tick");
1373                None
1374            }
1375        };
1376        for task in ci_failure_tasks {
1377            if ci_failure_unblock_cooldown_elapsed(&task)
1378                && let Some(ref gh) = gh
1379                && let Err(e) = try_unblock_ci_failure_task(
1380                    gh,
1381                    repo,
1382                    task_manager,
1383                    store,
1384                    &task,
1385                    auto_merge_in_flight,
1386                )
1387                .await
1388            {
1389                tracing::warn!(task_id = task.id, err = %e, "CI failure unblock check failed");
1390            }
1391        }
1392    }
1393
1394    // Process regular blocked tasks (those without a block_reason).
1395    for task in other_blocked {
1396        if task.block_reason.is_some() {
1397            continue;
1398        }
1399
1400        let task_id_str = task
1401            .external_id
1402            .clone()
1403            .unwrap_or_else(|| format!("internal:{}", task.id));
1404        let dispatch_key = format!("{repo}/{task_id_str}");
1405        if dispatching.contains_key(&dispatch_key) {
1406            continue;
1407        }
1408
1409        let runs = runs_by_task.remove(&task.id).unwrap_or_default();
1410        let relevant_runs: Vec<_> = runs
1411            .into_iter()
1412            .filter(|r| r.run_type == "agent" || r.run_type == "review")
1413            .collect();
1414        let mut failures = Vec::new();
1415        let mut has_review_failure = false;
1416        for run in relevant_runs.iter().rev() {
1417            if let Some(category) = classify_failure_from_run(run) {
1418                if run.run_type == "review" {
1419                    has_review_failure = true;
1420                }
1421                failures.push(category);
1422            }
1423            if failures.len() >= 3 {
1424                break;
1425            }
1426        }
1427
1428        if failures.is_empty() || !failures.iter().all(|f| f.is_recoverable()) {
1429            continue;
1430        }
1431
1432        // Determine the most recent failure category as the "reason key" for this unblock.
1433        let reason_key = failures.first().map_or("", |f| f.as_str());
1434
1435        // Detect reason changes to reset exponential backoff.
1436        // New reasons bypass old cooldown windows and start fresh attempt counters.
1437        let current_reason = task.auto_unblock_last_reason.clone();
1438        let reason_changed = reason_key != current_reason;
1439
1440        // Compute effective counter: when the reason changes, start from 0;
1441        // otherwise use the accumulated count for exponential backoff.
1442        let cooldown_count = if reason_changed {
1443            0
1444        } else {
1445            task.auto_unblock_count
1446        };
1447
1448        // Gate on attempt count using the effective count — if the reason changed, the
1449        // counter was reset above so cooldown_count is 0 and a new reason always gets a
1450        // fresh attempt regardless of how many times the old reason fired.
1451        if cooldown_count >= 3 {
1452            continue;
1453        }
1454
1455        if !auto_unblock_cooldown_elapsed(cooldown_count, &task.auto_unblock_last_at) {
1456            continue;
1457        }
1458
1459        // Compute the new counter value: when reason changed, cooldown_count is 0,
1460        // so incrementing gives 1 (first attempt for the new reason). When the reason is the
1461        // same, advance from current count for exponential backoff.
1462        let new_count = cooldown_count + 1;
1463        let now = chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string();
1464
1465        let ext_id = task
1466            .external_id
1467            .clone()
1468            .unwrap_or_else(|| format!("internal:{}", task.id));
1469        let new_status = if has_review_failure {
1470            Status::NeedsReview
1471        } else {
1472            Status::New
1473        };
1474        let new_status_str = if has_review_failure {
1475            "needs_review"
1476        } else {
1477            "new"
1478        };
1479
1480        // First update the status; only increment the counter if this succeeds.
1481        // This prevents transient GitHub API errors from exhausting the retry quota.
1482        if let Err(e) = task_manager
1483            .update_task_status(&ExternalId(ext_id), new_status)
1484            .await
1485        {
1486            tracing::warn!(
1487                task_id = task.id,
1488                err = %e,
1489                "auto-unblock failed to update status — counter not incremented"
1490            );
1491            continue;
1492        }
1493
1494        // Status update succeeded; now record the counter increment and clear related fields.
1495        let mut fields: Vec<(&str, serde_json::Value)> = vec![
1496            ("auto_unblock_count", serde_json::json!(new_count)),
1497            ("auto_unblock_last_at", serde_json::json!(now)),
1498            ("auto_unblock_last_reason", serde_json::json!(reason_key)),
1499            ("agent", serde_json::Value::Null),
1500            ("model", serde_json::Value::Null),
1501        ];
1502        fields.push(("no_code_reroutes", serde_json::json!(0)));
1503        fields.push(("no_code_last_agent", serde_json::json!("")));
1504        if has_review_failure {
1505            fields.push(("review_agent_failures", serde_json::json!(0)));
1506            fields.push(("review_invocations", serde_json::json!(0)));
1507            fields.push(("review_resets", serde_json::json!(0)));
1508        }
1509        if let Err(e) = store.set_fields(task.id, &fields).await {
1510            tracing::warn!(task_id = task.id, err = %e, "failed to set auto_unblock fields after successful status update");
1511            // Task is unblocked (correct behavior) but counter hasn't advanced.
1512            // Slightly over-eager retries are better than permanent task stalls.
1513        }
1514
1515        if let Err(e) = store
1516            .append_activity(
1517                task.id,
1518                "auto_unblock",
1519                Some("blocked"),
1520                Some(new_status_str),
1521                None,
1522                None,
1523                Some(&serde_json::json!({
1524                    "reason": reason_key,
1525                    "count": if reason_changed { 1 } else { task.auto_unblock_count + 1 },
1526                    "failures": failures.iter().map(|f| format!("{f:?}")).collect::<Vec<_>>(),
1527                })),
1528            )
1529            .await
1530        {
1531            tracing::warn!(task_id = task.id, err = %e, "failed to log auto_unblock activity");
1532        }
1533
1534        tracing::info!(
1535            task_id = task.id,
1536            failures = ?failures,
1537            review_failure = has_review_failure,
1538            "auto-unblocked task with recoverable failures"
1539        );
1540    }
1541
1542    Ok(())
1543}
1544
1545/// Global sweep for CI-failure blocked tasks from inactive or removed projects.
1546///
1547/// `auto_unblock_blocked_tasks` is called from `sync_tick` and only runs for the
1548/// currently-active repo. Tasks that were blocked for a project that is later removed
1549/// from orch config never get processed and accumulate in the DB indefinitely.
1550///
1551/// This function queries all blocked tasks across all repos and handles the CI-failure
1552/// cases (block_reason LIKE '%CI failure%' or '%CI checks timed out%') so they are
1553/// eventually cleaned up regardless of project activity. Called once per main tick from
1554/// `tick_global_sweeps`.
1555pub(crate) async fn auto_unblock_ci_failure_blocked_tasks_global(
1556    task_manager: &Arc<TaskManager>,
1557    store: &Arc<TaskStore>,
1558    auto_merge_in_flight: &Arc<DashSet<String>>,
1559) -> anyhow::Result<()> {
1560    let all_blocked = match store.list_all_by_status_global(TaskStatus::Blocked).await {
1561        Ok(tasks) => tasks,
1562        Err(e) => {
1563            tracing::warn!(err = %e, "global CI-failure blocked sweep: failed to list blocked tasks");
1564            return Ok(());
1565        }
1566    };
1567
1568    let ci_failure_tasks: Vec<_> = all_blocked
1569        .into_iter()
1570        .filter(is_ci_failure_block)
1571        .collect();
1572    if ci_failure_tasks.is_empty() {
1573        return Ok(());
1574    }
1575
1576    let gh = match GhHttp::new() {
1577        Ok(g) => g,
1578        Err(e) => {
1579            tracing::warn!(err = %e, "global CI-failure blocked sweep: failed to create GhHttp");
1580            return Ok(());
1581        }
1582    };
1583
1584    for task in ci_failure_tasks {
1585        if !ci_failure_unblock_cooldown_elapsed(&task) {
1586            continue;
1587        }
1588        if let Err(e) = try_unblock_ci_failure_task(
1589            &gh,
1590            &task.repo,
1591            task_manager,
1592            store,
1593            &task,
1594            auto_merge_in_flight,
1595        )
1596        .await
1597        {
1598            tracing::warn!(
1599                task_id = task.id,
1600                repo = %task.repo,
1601                err = %e,
1602                "global CI-failure blocked sweep: unblock check failed"
1603            );
1604        }
1605    }
1606
1607    Ok(())
1608}
1609
1610/// Global sweep: re-fire stale NeedsReview tasks and escalate persistently-stuck ones
1611/// to Blocked, across all repos.
1612///
1613/// Two cases where the event-driven subscriber in `subscribers/review.rs` can miss
1614/// a NeedsReview task: all review slots are busy when the event fires, or the
1615/// broadcast receiver lagged and dropped it. Re-publish the NeedsReview status
1616/// after a short idle window so the subscriber can pick it up when a slot is free.
1617/// Using a status re-fire rather than spawning the review agent directly keeps this
1618/// path stateless. The subscriber is still the sole spawner and uses the InReview
1619/// transition as its atomic guard (issue #857)
1620///
1621/// Queries across all repos and writes through the store via `*_by_store_id`
1622/// TaskManager helpers. Called once per main tick from `tick_global_sweeps`
1623pub(crate) async fn refire_and_escalate_stale_needs_review_global(
1624    task_manager: &Arc<TaskManager>,
1625    store: &Arc<TaskStore>,
1626    router: &Router,
1627    dispatching: &Arc<DashMap<String, String>>,
1628    active_repos: &HashSet<String>,
1629) -> anyhow::Result<()> {
1630    const MIN_STALE_NEEDS_REVIEW_MINUTES: i64 = 1;
1631    const MAX_NEEDS_REVIEW_REFIRE_ATTEMPTS: u64 = 5;
1632
1633    let needs_review_tasks = match store
1634        .list_all_by_status_global(TaskStatus::NeedsReview)
1635        .await
1636    {
1637        Ok(tasks) => tasks,
1638        Err(e) => {
1639            tracing::warn!(err = %e, "global NeedsReview refire sweep: failed to list needs_review tasks");
1640            return Ok(());
1641        }
1642    };
1643
1644    if needs_review_tasks.is_empty() {
1645        return Ok(());
1646    }
1647
1648    // Re-firing while every review agent is cooled wastes the counter and can
1649    // escalate a task to Blocked while the subscriber would only bail again
1650    let all_review_agents_cooled = router.healthy_agent_count("review") == 0;
1651    if all_review_agents_cooled {
1652        tracing::info!(
1653            count = needs_review_tasks.len(),
1654            "global NeedsReview refire sweep: no review agent available — skipping this pass"
1655        );
1656        return Ok(());
1657    }
1658
1659    for stored in needs_review_tasks {
1660        // No review subscriber listens for a repo outside the active set
1661        if !active_repos.contains(&stored.repo) {
1662            continue;
1663        }
1664
1665        let external = tasks::store_task_to_external(&stored);
1666        let ext_id = external.id.clone();
1667        let repo = stored.repo.clone();
1668
1669        let age_minutes = match chrono::DateTime::parse_from_rfc3339(&external.updated_at) {
1670            Ok(updated_at) => {
1671                let age = chrono::Utc::now() - updated_at.with_timezone(&chrono::Utc);
1672                if age.num_minutes() < MIN_STALE_NEEDS_REVIEW_MINUTES {
1673                    continue;
1674                }
1675                Some(age.num_minutes())
1676            }
1677            Err(_) => None,
1678        };
1679
1680        // Skip tasks actively being dispatched by their owning repo's engine loop.
1681        let dispatch_key = format!("{}/{}", repo, ext_id.0);
1682        if dispatching.contains_key(&dispatch_key) {
1683            continue;
1684        }
1685
1686        let current_refires = u64::try_from(stored.needs_review_refires).unwrap_or(0);
1687        let new_refires = current_refires + 1;
1688
1689        if new_refires > MAX_NEEDS_REVIEW_REFIRE_ATTEMPTS {
1690            tracing::warn!(
1691                task_id = stored.id,
1692                repo = %repo,
1693                new_refires,
1694                "global NeedsReview refire sweep: escalating to Blocked after repeated refires"
1695            );
1696            let fields = [
1697                ("block_reason", serde_json::json!(REBROADCAST_BLOCK_REASON)),
1698                (
1699                    "last_error",
1700                    serde_json::json!(format!("escalated after {} retries", new_refires)),
1701                ),
1702            ];
1703            if let Err(e) = task_manager
1704                .update_task_status_and_result_by_store_id(
1705                    stored.id,
1706                    &repo,
1707                    &ext_id,
1708                    Status::Blocked,
1709                    &fields,
1710                )
1711                .await
1712            {
1713                tracing::error!(task_id = stored.id, repo = %repo, err = %e, "global NeedsReview refire sweep: escalation to Blocked failed — skipping to avoid silent auto-unblock loop");
1714                continue;
1715            }
1716            if let Err(e) = store::store_increment_by_id(
1717                &Some(Arc::clone(store)),
1718                stored.id,
1719                "needs_review_refires",
1720            )
1721            .await
1722            {
1723                tracing::warn!(
1724                    task_id = stored.id,
1725                    repo = %repo,
1726                    err = %e,
1727                    "global NeedsReview refire sweep: failed to increment needs_review_refires after escalation"
1728                );
1729            }
1730            continue;
1731        }
1732
1733        // Exponential backoff: MIN * 2^(current_refires)
1734        let required_minutes =
1735            MIN_STALE_NEEDS_REVIEW_MINUTES * (1i64 << u32::try_from(current_refires).unwrap_or(0));
1736
1737        let should_fire = match age_minutes {
1738            Some(age) => age >= required_minutes,
1739            None => true,
1740        };
1741
1742        if !should_fire {
1743            continue;
1744        }
1745
1746        if let Err(e) = task_manager
1747            .update_task_status_by_store_id(stored.id, &repo, &ext_id, Status::NeedsReview)
1748            .await
1749        {
1750            tracing::warn!(
1751                task_id = stored.id,
1752                repo = %repo,
1753                err = %e,
1754                "global NeedsReview refire sweep: failed to re-fire NeedsReview event — not incrementing counter"
1755            );
1756            continue;
1757        }
1758
1759        let fired_refires = match store::store_increment_by_id(
1760            &Some(Arc::clone(store)),
1761            stored.id,
1762            "needs_review_refires",
1763        )
1764        .await
1765        {
1766            Ok(v) => v,
1767            Err(e) => {
1768                tracing::warn!(
1769                    task_id = stored.id,
1770                    repo = %repo,
1771                    err = %e,
1772                    "global NeedsReview refire sweep: failed to increment needs_review_refires after re-fire"
1773                );
1774                current_refires + 1
1775            }
1776        };
1777
1778        tracing::info!(
1779            task_id = stored.id,
1780            repo = %repo,
1781            age_minutes,
1782            refires = fired_refires,
1783            required_minutes,
1784            "global NeedsReview refire sweep: re-firing NeedsReview event for stale task"
1785        );
1786    }
1787
1788    Ok(())
1789}
1790
1791/// Sync tick — runs every 45s.
1792///
1793/// Handles less-frequent operations:
1794/// - Cleanup finished worktrees
1795/// - Check for merged PRs → mark tasks done
1796/// - Scan for @mentions
1797#[expect(
1798    clippy::too_many_arguments,
1799    reason = "the engine passes its shared handles one by one"
1800)]
1801pub(crate) async fn sync_tick(
1802    backend: &Arc<dyn ExternalBackend>,
1803    tmux: &Arc<TmuxManager>,
1804    repo: &str,
1805    config: &EngineConfig,
1806    router: &Arc<RwLock<Router>>,
1807    task_manager: &Arc<TaskManager>,
1808    store: &Arc<TaskStore>,
1809    dispatching: &Arc<DashMap<String, String>>,
1810    auto_merge_in_flight: &Arc<DashSet<String>>,
1811) -> anyhow::Result<()> {
1812    // One node polls GitHub and decides for the whole cluster
1813    if !store.is_leader() {
1814        return Ok(());
1815    }
1816
1817    tracing::debug!("sync tick");
1818
1819    let mut phases: Vec<(&str, u128)> = Vec::new();
1820    let mut at = Instant::now();
1821
1822    // Global GitHub 5xx circuit breaker — skip non-critical background work
1823    // during sustained GitHub outages to reduce churn and wasted API calls.
1824    let gh_circuit_open = is_github_circuit_open();
1825
1826    if gh_circuit_open {
1827        let remaining = github_circuit_remaining_secs();
1828        tracing::debug!(
1829            remaining_secs = remaining,
1830            "GitHub 5xx circuit breaker open — skipping non-critical sync work"
1831        );
1832        // Persist circuit breaker state as a metric for operators.
1833        // This is critical for avoiding thundering herd after restart during outages.
1834        if let Err(e) =
1835            try_kv_set_prefer_store(Some(store), "metrics:orch.github_5xx_circuit.open", "1").await
1836        {
1837            tracing::error!(err = %e, "failed to persist circuit breaker state — may cause thundering herd after restart");
1838        }
1839    } else if let Err(e) =
1840        try_kv_set_prefer_store(Some(store), "metrics:orch.github_5xx_circuit.open", "0").await
1841    {
1842        tracing::error!(err = %e, "failed to clear circuit breaker state");
1843    }
1844
1845    // 0. Ingest all active external tasks into the unified store.
1846    //    This ensures the store has data for tasks created before dual-write was added.
1847    if !gh_circuit_open {
1848        if let Err(e) = ingest_external_tasks(backend, repo, store).await {
1849            tracing::debug!(err = %e, "external task ingest failed");
1850        }
1851        phases.push(("ingest", at.elapsed().as_millis()));
1852        at = Instant::now();
1853
1854        // Reconcile existing board members too: labels can change without a
1855        // local task transition, and a human/automation may move a card. The
1856        // backend only writes drifted fields through the shared GitHub client.
1857        if let Err(e) = backend.reconcile_project_statuses().await {
1858            tracing::warn!(err = %e, "project status reconciliation failed");
1859        }
1860        phases.push(("board_reconcile", at.elapsed().as_millis()));
1861    }
1862    at = Instant::now();
1863
1864    // 1. Cleanup worktrees for done tasks (background — must not block routing/dispatch)
1865    if !gh_circuit_open {
1866        let backend = Arc::clone(backend);
1867        let repo = repo.to_string();
1868        let task_manager = Arc::clone(task_manager);
1869        let store = Arc::clone(store);
1870        tokio::spawn(async move {
1871            if let Err(e) = cleanup_done_worktrees(&backend, &repo, &task_manager, &store).await {
1872                tracing::warn!(err = %e, "worktree cleanup failed");
1873            }
1874            if let Err(e) = prune_old_run_output(&store).await {
1875                tracing::warn!(err = %e, "task run pruning failed");
1876            }
1877        });
1878    }
1879
1880    let (in_review_tasks, needs_review_tasks, blocked_review_candidates) =
1881        match prefetch_review_tasks(store, repo).await {
1882            Ok(tasks) => tasks,
1883            Err(e) => {
1884                tracing::warn!(err = %e, "failed to prefetch review tasks");
1885                (vec![], vec![], vec![])
1886            }
1887        };
1888    phases.push(("prefetch_review", at.elapsed().as_millis()));
1889    at = Instant::now();
1890
1891    if !gh_circuit_open {
1892        let mention_fallback = chrono::Utc::now() - chrono::Duration::hours(24);
1893        let mention_since = kv_get_prefer_store(Some(store), "mentions_last_checked")
1894            .await
1895            .unwrap_or_else(|| mention_fallback.format("%Y-%m-%dT%H:%M:%SZ").to_string());
1896        let shared_since = mention_since.clone();
1897
1898        let cached_router_config = router.read().await.config.clone();
1899        // Include blocked tasks that have branch/pr info when checking for merged PRs.
1900        let mut combined_needs = needs_review_tasks.clone();
1901        combined_needs.extend(blocked_review_candidates.clone());
1902
1903        let (merge_result, comments_result, review_result) = tokio::join!(
1904            check_merged_prs(
1905                backend,
1906                repo,
1907                store,
1908                task_manager,
1909                &in_review_tasks,
1910                &combined_needs,
1911            ),
1912            fetch_comments_since(backend, &shared_since),
1913            async {
1914                let gh = match GhHttp::new() {
1915                    Ok(g) => g,
1916                    Err(e) => {
1917                        tracing::warn!(err = %e, "failed to create GitHub HTTP client for review poll");
1918                        return Ok(());
1919                    }
1920                };
1921                Box::pin(review_open_prs(
1922                    backend,
1923                    repo,
1924                    config,
1925                    task_manager,
1926                    store,
1927                    dispatching,
1928                    auto_merge_in_flight,
1929                    &in_review_tasks,
1930                    &gh,
1931                    &cached_router_config,
1932                ))
1933                .await
1934            }
1935        );
1936
1937        if let Err(e) = merge_result {
1938            tracing::warn!(err = %e, "PR merge check failed");
1939        }
1940
1941        match comments_result {
1942            Ok(comments) => {
1943                let filtered = filter_mentions_by_since(&comments, &mention_since);
1944
1945                if let Err(e) =
1946                    scan_comments(backend, Some(store), repo, task_manager, Some(&filtered)).await
1947                {
1948                    tracing::warn!(err = %e, "comment scan failed");
1949                }
1950            }
1951            Err(e) => {
1952                tracing::warn!(err = %e, "comment fetch failed");
1953            }
1954        }
1955
1956        if let Err(e) = review_result {
1957            tracing::warn!(err = %e, "PR review failed");
1958        }
1959
1960        phases.push(("merge_comments_review", at.elapsed().as_millis()));
1961    }
1962    at = Instant::now();
1963
1964    // 5. Stale InReview recovery lives in `tick_recover_stuck_tasks`, the single owner of an
1965    // in_review task whose review session is gone or hung. It ran here too, after one minute,
1966    // which beat a finished review's result to the engine and dropped it as stale
1967
1968    // 5c. Detect stale InProgress tasks and recover them.
1969    //
1970    // Similar to stale InReview detection, we check for InProgress tasks whose
1971    // agent tmux session has died (crash, OOM, etc.) and reset them to Routed
1972    // so they can be re-dispatched.
1973    let in_progress_tasks: Vec<_> = match store.list_by_status(repo, TaskStatus::InProgress).await {
1974        Ok(tasks) => tasks
1975            .into_iter()
1976            .map(|stored| {
1977                let external = tasks::store_task_to_external(&stored);
1978                (stored, external)
1979            })
1980            .collect(),
1981        Err(e) => {
1982            tracing::warn!(err = %e, "failed to list in_progress tasks — skipping stale InProgress check");
1983            vec![]
1984        }
1985    };
1986
1987    if !in_progress_tasks.is_empty() {
1988        // Fetch all live tmux sessions once instead of one subprocess call per task.
1989        let live_sessions: HashSet<String> = tmux
1990            .list_sessions()
1991            .await
1992            .unwrap_or_default()
1993            .into_iter()
1994            .map(|s| s.name)
1995            .collect();
1996
1997        for (_stored, external) in &in_progress_tasks {
1998            // A task is only considered stale if it has been in InProgress for > 5 minutes.
1999            const MIN_STALE_MINUTES: i64 = 5;
2000
2001            let task_id = &external.id.0;
2002
2003            // Skip tasks currently being processed by the main tick (dispatch + agent flow).
2004            let dispatch_key = format!("{repo}/{task_id}");
2005            {
2006                if dispatching.contains_key(&dispatch_key) {
2007                    tracing::debug!(
2008                        task_id = %task_id,
2009                        "task locked by dispatch flow, skipping stale check"
2010                    );
2011                    continue;
2012                }
2013            }
2014
2015            // Skip tasks that just transitioned to InProgress — allow time for the
2016            // agent to start its tmux session before treating it as stale.
2017            match chrono::DateTime::parse_from_rfc3339(&external.updated_at) {
2018                Ok(updated_at) => {
2019                    let age = chrono::Utc::now() - updated_at.with_timezone(&chrono::Utc);
2020                    if age.num_minutes() < MIN_STALE_MINUTES {
2021                        tracing::debug!(
2022                            task_id = %task_id,
2023                            age_seconds = age.num_seconds(),
2024                            "InProgress task is too young to be considered stale, skipping"
2025                        );
2026                        continue;
2027                    }
2028                }
2029                Err(e) => {
2030                    tracing::warn!(
2031                        task_id = %task_id,
2032                        ts = %external.updated_at,
2033                        err = %e,
2034                        "invalid updated_at timestamp — treating task as potentially stale"
2035                    );
2036                }
2037            }
2038
2039            let agent_session = tmux.session_name(repo, task_id);
2040            if live_sessions.contains(&agent_session) {
2041                // Agent is still alive — skip.
2042                continue;
2043            }
2044
2045            // No agent session exists. The agent is dead regardless of
2046            // session tracking — hard restart, crash, or flag not persisted.
2047            // Reset to Routed so the dispatcher can re-spawn it.
2048            {
2049                tracing::warn!(
2050                    task_id = %task_id,
2051                    session = %agent_session,
2052                    "InProgress task has no active agent session — resetting to Routed"
2053                );
2054                match task_manager
2055                    .update_task_status_if(&external.id, Status::Routed, Status::InProgress)
2056                    .await
2057                {
2058                    Err(e) => {
2059                        tracing::error!(task_id = %task_id, err = %e, "failed to reset stale InProgress task — task may be stuck in InProgress indefinitely");
2060                    }
2061                    Ok(false) => {
2062                        tracing::debug!(
2063                            task_id = %task_id,
2064                            "stale InProgress reset skipped — task already transitioned (concurrent Done/Blocked/NeedsReview)"
2065                        );
2066                    }
2067                    Ok(true) => {
2068                        // Successfully reset to Routed
2069                    }
2070                }
2071            }
2072        }
2073    }
2074
2075    phases.push(("stale_in_progress", at.elapsed().as_millis()));
2076    at = Instant::now();
2077    // 5c. Auto-unblock tasks with recoverable failures.
2078    if let Err(e) =
2079        auto_unblock_blocked_tasks(repo, task_manager, store, dispatching, auto_merge_in_flight)
2080            .await
2081    {
2082        tracing::warn!(err = %e, "auto-unblock failed");
2083    }
2084
2085    phases.push(("auto_unblock", at.elapsed().as_millis()));
2086
2087    engine::log_phase_timings("sync", repo, &phases);
2088    Ok(())
2089}
2090
2091/**
2092 * Skill repos and agent health are the same whichever project asks, so they run once per
2093 * sync pass rather than once per project. Called after every project's `sync_tick`
2094 **/
2095pub(crate) async fn sync_shared_tail(router: &Arc<RwLock<Router>>, store: Option<&Arc<TaskStore>>) {
2096    let Some(store) = store else {
2097        return;
2098    };
2099    if !store.is_leader() {
2100        return;
2101    }
2102    match skills_sync().await {
2103        Ok(()) => {
2104            // Clone the Arc handle before releasing the read guard so the read lock is not
2105            // held across the async invalidation call
2106            let llm = router.read().await.llm_router_handle();
2107            llm.invalidate_skills_catalog().await;
2108        }
2109        Err(e) => {
2110            tracing::warn!(err = %e, "skills sync failed");
2111        }
2112    }
2113
2114    // Pre-emptive health check: refresh degraded-agent flags from the rate_limits table.
2115    // Clone what is needed out of the router first, so the write lock is not starved for the
2116    // duration of the DB query
2117    let (available_agents, config) = {
2118        let r = router.read().await;
2119        (r.available_agents.clone(), r.config.clone())
2120    };
2121    let model_checker = |agent: &str| -> bool {
2122        for comp in &["simple", "medium", "complex", "review"] {
2123            if config.has_available_model_for_complexity(agent, comp) {
2124                return true;
2125            }
2126        }
2127        false
2128    };
2129    refresh_degraded_agents(
2130        store,
2131        &available_agents,
2132        &model_checker,
2133        router_config::health_check_window_hours(),
2134        router_config::degraded_rate_limit_threshold(),
2135    )
2136    .await;
2137
2138    emit_degraded_agents_if_needed(&available_agents, &config, Some(store)).await;
2139}
2140
2141/// Scan for @mentions and handle them.
2142///
2143/// Checks recent issue comments for @orch (and legacy @orchestrator) mentions.
2144/// Unified comment scanner — handles both @mentions and slash commands in one pass.
2145///
2146/// For each comment, classifies it via [`classify_comment`] and dispatches:
2147/// - Slash commands → validate + execute (with sentinel task if it was an @mention)
2148/// - @mentions without commands → create internal task for agent to respond
2149/// - Everything else → skip
2150async fn scan_comments(
2151    backend: &Arc<dyn ExternalBackend>,
2152    store: Option<&Arc<TaskStore>>,
2153    repo: &str,
2154    task_manager: &Arc<TaskManager>,
2155    prefetched_comments: Option<&[Mention]>,
2156) -> anyhow::Result<()> {
2157    let current_user = match backend.get_authenticated_user().await {
2158        Ok(Some(u)) => format!("@{u}"),
2159        Ok(None) => {
2160            tracing::debug!("backend does not support user identity, skipping comment scan");
2161            return Ok(());
2162        }
2163        Err(e) => {
2164            tracing::warn!(err = %e, "failed to get current user, skipping comment scan");
2165            return Ok(());
2166        }
2167    };
2168
2169    let comments = if let Some(c) = prefetched_comments {
2170        c.to_vec()
2171    } else {
2172        let fallback = chrono::Utc::now() - chrono::Duration::hours(24);
2173        let since_str = match kv_get_prefer_store(store, "mentions_last_checked").await {
2174            Some(ts) => ts,
2175            None => fallback.format("%Y-%m-%dT%H:%M:%SZ").to_string(),
2176        };
2177        match backend.get_mentions(&since_str).await {
2178            Ok(m) => m,
2179            Err(e) => {
2180                tracing::warn!(err = %e, "failed to get comments");
2181                return Ok(());
2182            }
2183        }
2184    };
2185
2186    // Dedup set: mention tasks already created (by source_id).
2187    let existing_mentions: HashSet<String> = if let Some(s) = store {
2188        match s.list_source_ids_by_source(repo, "mention", 30).await {
2189            Ok(ids) => ids.into_iter().collect(),
2190            Err(e) => {
2191                tracing::warn!(err = %e, "failed to load mention source IDs — skipping to prevent duplicates");
2192                return Ok(());
2193            }
2194        }
2195    } else {
2196        HashSet::new()
2197    };
2198
2199    // Lazily create GhHttp — only allocate when we actually need to make GitHub
2200    // API calls.  This avoids network initialization in test environments where
2201    // the code path never reaches an API call.
2202    let gh_cell: OnceCell<GhHttp> = OnceCell::new();
2203    let get_gh = || async { gh_cell.get_or_try_init(|| async { GhHttp::new() }).await };
2204
2205    // Pre-classify all comments up front so we can batch the is_pull_request
2206    // checks for CreateMentionTask entries before entering the main loop.
2207    let actions: Vec<CommentAction> = comments
2208        .iter()
2209        .map(|c| {
2210            classify_comment(
2211                &c.body,
2212                c.issue_url.as_deref(),
2213                &current_user,
2214                existing_mentions.contains(&c.id),
2215            )
2216        })
2217        .collect();
2218
2219    // Collect (comment_index, issue_num) for all CreateMentionTask entries that
2220    // need an is_pull_request check, then resolve them all concurrently with
2221    // join_all to avoid N × RTT serial API calls inside the loop.
2222    let pr_check_inputs: Vec<(usize, String)> = actions
2223        .iter()
2224        .enumerate()
2225        .filter_map(|(i, action)| {
2226            if let CommentAction::CreateMentionTask {
2227                issue_num: Some(num),
2228            } = action
2229            {
2230                Some((i, num.clone()))
2231            } else {
2232                None
2233            }
2234        })
2235        .collect();
2236
2237    // Only create GhHttp if we actually have PR checks to perform.
2238    let is_pr_values: Vec<bool> = if pr_check_inputs.is_empty() {
2239        Vec::new()
2240    } else {
2241        let gh = get_gh().await?;
2242        future::join_all(
2243            pr_check_inputs
2244                .iter()
2245                .map(|(_, num)| async move { gh.is_pull_request(repo, num).await }),
2246        )
2247        .await
2248    };
2249
2250    // Build a lookup map: comment_index → is_pr.
2251    let is_pr_map: HashMap<usize, bool> = pr_check_inputs
2252        .iter()
2253        .map(|(i, _)| *i)
2254        .zip(is_pr_values)
2255        .collect();
2256
2257    let mut cursor = MentionCursor::default();
2258
2259    for (comment_idx, (comment, action)) in comments.iter().zip(actions).enumerate() {
2260        match action {
2261            CommentAction::Skip => {
2262                cursor.advance(&comment.created_at);
2263            }
2264
2265            CommentAction::ExecuteCommand { command, issue_num } => {
2266                let store_opt: Option<Arc<dyn CommandStoreOps>> =
2267                    store.map(|s| Arc::clone(s) as Arc<dyn CommandStoreOps>);
2268                let gh = get_gh().await?;
2269                let outcome = validate_and_run_command(
2270                    backend,
2271                    gh,
2272                    repo,
2273                    &issue_num,
2274                    &command,
2275                    &comment.author,
2276                    &store_opt,
2277                    task_manager,
2278                )
2279                .await;
2280                match outcome {
2281                    // Validation hit a transient GitHub error: nothing ran and nothing
2282                    // was posted. Stop the cursor so the next tick retries this comment.
2283                    CommandOutcome::FetchFailed | CommandOutcome::CollaboratorCheckFailed => {
2284                        tracing::warn!(
2285                            comment_id = %comment.id,
2286                            command = %command,
2287                            "deferring slash command after a transient GitHub error"
2288                        );
2289                        cursor.block_on_gap();
2290                        break;
2291                    }
2292                    // The outcome comment (success or error) was already posted to
2293                    // GitHub, so re-processing would create duplicates.
2294                    CommandOutcome::Executed { .. }
2295                    | CommandOutcome::NotOpen { .. }
2296                    | CommandOutcome::NotCollaborator { .. } => cursor.advance(&comment.created_at),
2297                }
2298            }
2299
2300            CommentAction::ExecuteCommandForMention { command, .. } => {
2301                if let Err(e) = backend.acknowledge_mention(&comment.id).await {
2302                    tracing::debug!(err = %e, mention_id = %comment.id, "failed to acknowledge mention");
2303                    // Do NOT advance the cursor or call handle_slash_command — the GitHub
2304                    // reaction was never posted. Retry on next tick once the network
2305                    // glitch has resolved.
2306                } else {
2307                    // Parent lookup failure propagates via `?` — scan_comments returns early
2308                    // without advancing the cursor so the mention is retried on next sync.
2309                    let gh = get_gh().await?;
2310                    let advanced_safely = handle_slash_command(
2311                        backend,
2312                        store,
2313                        repo,
2314                        task_manager,
2315                        gh,
2316                        comment,
2317                        &command,
2318                        &mut cursor,
2319                    )
2320                    .await?;
2321                    if !advanced_safely {
2322                        cursor.block_on_gap();
2323                        break;
2324                    }
2325                }
2326            }
2327
2328            CommentAction::CreateMentionTask { issue_num } => {
2329                if let Err(e) = backend.acknowledge_mention(&comment.id).await {
2330                    tracing::debug!(err = %e, mention_id = %comment.id, "failed to acknowledge mention");
2331                    // Do NOT record the mention task or advance the cursor — the GitHub
2332                    // reaction was never posted. Retry on next tick once the network
2333                    // glitch has resolved.
2334                } else {
2335                    // Use the pre-fetched value — no serial API call here.
2336                    // issue_num may be None (e.g. bot mentioned with no issue URL), in
2337                    // which case is_pr_map has no entry; default to false (the None arm
2338                    // of the downstream match ignores is_pr entirely).
2339                    let is_pr = is_pr_map.get(&comment_idx).copied().unwrap_or(false);
2340
2341                    let (title, task_body) = match (&issue_num, is_pr) {
2342                        (Some(num), false) => (
2343                            format!("Respond to mention by @{} on task #{num}", comment.author),
2344                            format!(
2345                                "Mention by @{} on GitHub issue #{num}:\n\n{}\n\n---\nThis mention was posted on issue #{num}. Review the full issue to understand the context before responding.",
2346                                comment.author, comment.body
2347                            ),
2348                        ),
2349                        (Some(num), true) => (
2350                            format!("Respond to mention by @{} on PR #{num}", comment.author),
2351                            format!(
2352                                "Mention by @{} on GitHub PR #{num}:\n\n{}\n\n---\nThis mention was posted on PR #{num}. Review the full PR (diff, description, and comments) to understand the context before responding.",
2353                                comment.author, comment.body
2354                            ),
2355                        ),
2356                        (None, _) => (
2357                            format!("Respond to mention by @{}", comment.author),
2358                            format!("Mention by @{}:\n\n{}", comment.author, comment.body),
2359                        ),
2360                    };
2361
2362                    if let Some(s) = store {
2363                        // On DB lookup failure, do NOT record the mention task or advance
2364                        // the cursor — the mention will be retried on next sync.
2365                        let parent_id = match issue_num {
2366                            Some(ref num) => {
2367                                match resolve_mention_parent_id(s, repo, num, is_pr).await {
2368                                    Ok(id) => id,
2369                                    Err(e) => {
2370                                        tracing::warn!(
2371                                            mention_id = %comment.id,
2372                                            err = %e,
2373                                            "deferring mention task due to parent lookup failure"
2374                                        );
2375                                        return Err(anyhow::anyhow!(
2376                                            "deferring mention {} due to parent lookup failure",
2377                                            comment.id
2378                                        ));
2379                                    }
2380                                }
2381                            }
2382                            None => None,
2383                        };
2384                        if !record_mention_task(
2385                            s,
2386                            repo,
2387                            comment,
2388                            &title,
2389                            &task_body,
2390                            parent_id,
2391                            &mut cursor,
2392                        )
2393                        .await
2394                        {
2395                            cursor.block_on_gap();
2396                            break;
2397                        }
2398                    }
2399                }
2400            }
2401        }
2402    }
2403
2404    if let Some(ts) = cursor.into_last_safe_ts() {
2405        kv_set_prefer_store(store, "mentions_last_checked", &ts).await;
2406    }
2407
2408    Ok(())
2409}
2410
2411/// Sync skill repositories from config.
2412///
2413/// Reads the `skills:` list from config and clones/pulls each repository
2414/// to `~/.orch/skills/{repo}/`. This keeps skill documentation up-to-date
2415/// for agents.
2416async fn skills_sync() -> anyhow::Result<()> {
2417    let skills = match config::get_skills() {
2418        Ok(s) => s,
2419        Err(e) => {
2420            tracing::debug!(err = %e, "no skills configured");
2421            return Ok(());
2422        }
2423    };
2424
2425    if skills.is_empty() {
2426        tracing::debug!("no skills configured, skipping sync");
2427        return Ok(());
2428    }
2429
2430    let skills_base = home::skills_dir()?;
2431    let git_timeout = Duration::from_secs(60);
2432
2433    // Run all skill git operations concurrently — each is fully independent.
2434    // Wall-clock time drops from N × timeout to max(timeout).
2435    let futs: Vec<_> = skills
2436        .into_iter()
2437        .map(|skill| sync_one_skill(skill, skills_base.clone(), git_timeout))
2438        .collect();
2439    future::join_all(futs).await;
2440
2441    Ok(())
2442}
2443
2444/// Sync a single skill repository: `git pull` if it exists, `git clone` otherwise.
2445///
2446/// All errors are logged via `tracing::warn!` so that one failing skill does not
2447/// prevent the others from being updated.
2448async fn sync_one_skill(skill: config::SkillConfig, skills_base: PathBuf, git_timeout: Duration) {
2449    // Validate repo format to prevent path traversal
2450    if skill.repo.contains("..") || skill.repo.matches('/').count() != 1 {
2451        tracing::warn!(repo = %skill.repo, "invalid skill repo format, expected 'owner/repo'");
2452        return;
2453    }
2454
2455    let repo_dir = skills_base.join(&skill.repo);
2456    let repo_url = format!("https://github.com/{}.git", skill.repo);
2457
2458    // Use async metadata check instead of `Path::exists()` to avoid
2459    // performing blocking syscall on the reactor thread.
2460    let repo_exists = fs::metadata(&repo_dir).await.is_ok_and(|m| m.is_dir());
2461
2462    if repo_exists {
2463        // Pull latest changes with timeout
2464        tracing::debug!(repo = %skill.repo, "pulling skill repo");
2465        let pull_result = time::timeout(
2466            git_timeout,
2467            git_ops::git()
2468                .await
2469                .args(["pull", "--ff-only", "--prune"])
2470                .current_dir(&repo_dir)
2471                .output_with_context(),
2472        )
2473        .await;
2474
2475        match pull_result {
2476            Ok(Ok(output)) if !output.status.success() => {
2477                let stderr = String::from_utf8_lossy(&output.stderr);
2478                tracing::warn!(repo = %skill.repo, err = %stderr, "git pull failed");
2479            }
2480            Ok(Ok(_)) => {
2481                tracing::debug!(repo = %skill.repo, "skill repo updated");
2482            }
2483            Ok(Err(e)) => {
2484                tracing::warn!(repo = %skill.repo, err = %e, "git pull error");
2485            }
2486            Err(_) => {
2487                tracing::warn!(repo = %skill.repo, "git pull timed out after 60s");
2488            }
2489        }
2490    } else {
2491        // Clone the repository (shallow for efficiency)
2492        tracing::debug!(repo = %skill.repo, "cloning skill repo");
2493        let Some(parent) = repo_dir.parent() else {
2494            tracing::warn!(repo = %skill.repo, "skill repo path has no parent directory");
2495            return;
2496        };
2497        if let Err(e) = fs::create_dir_all(parent).await {
2498            tracing::warn!(repo = %skill.repo, err = %e, "failed to create parent dir for skill");
2499            return;
2500        }
2501        let Some(repo_dir_str) = repo_dir.to_str() else {
2502            tracing::warn!(repo = %skill.repo, "skill repo path is not valid UTF-8");
2503            return;
2504        };
2505
2506        let clone_result = time::timeout(
2507            git_timeout,
2508            git_ops::git()
2509                .await
2510                .args(["clone", "--depth", "1", &repo_url, repo_dir_str])
2511                .output_with_context(),
2512        )
2513        .await;
2514
2515        match clone_result {
2516            Ok(Ok(output)) if !output.status.success() => {
2517                let stderr = String::from_utf8_lossy(&output.stderr);
2518                tracing::warn!(repo = %skill.repo, err = %stderr, "git clone failed");
2519                // Clean up partial clone to allow retry on next tick
2520                let _ = fs::remove_dir_all(&repo_dir).await;
2521            }
2522            Ok(Ok(_)) => {
2523                tracing::info!(repo = %skill.repo, "skill repo cloned");
2524            }
2525            Ok(Err(e)) => {
2526                tracing::warn!(repo = %skill.repo, err = %e, "git clone error");
2527                let _ = fs::remove_dir_all(&repo_dir).await;
2528            }
2529            Err(_) => {
2530                tracing::warn!(repo = %skill.repo, "git clone timed out after 60s");
2531                let _ = fs::remove_dir_all(&repo_dir).await;
2532            }
2533        }
2534    }
2535}
2536
2537/// Per-agent degradation detail collected during the alerting check.
2538struct DegradedAgentDetail {
2539    /// Agent name (e.g. "claude", "codex").
2540    agent: String,
2541    /// Reason the agent is degraded: the stored cooldown reason when the agent
2542    /// itself is in cooldown, or `"no_available_model"` when all model pools
2543    /// are individually cooled.
2544    reason: String,
2545    /// Models that are individually in cooldown for this agent, deduped across
2546    /// all complexity tiers.
2547    cooled_models: Vec<String>,
2548}
2549
2550/// Per-agent pool health used for stale model pool detection.
2551struct AgentPoolHealth {
2552    agent: String,
2553    cooled_models: Vec<String>,
2554    persistent_failed_models: Vec<String>,
2555    total_configured_models: usize,
2556}
2557
2558/// Inspect router & cooldown state and emit metrics every tick.
2559///
2560/// - `metrics:orch.agents_degraded.count` is written on every call with the
2561///   current number of degraded agents (0 when healthy).
2562/// - When `count >= 3` a WARN log is emitted with full dimensions (agent
2563///   names, cooled models, cooldown reasons) and a dedicated alert metric
2564///   `metrics:orch.agents_degraded.alert` is set to `"1"`. The alert is
2565///   cleared to `"0"` when the count drops below the threshold.
2566pub(crate) async fn emit_degraded_agents_if_needed(
2567    available_agents: &[String],
2568    config: &RouterConfig,
2569    store: Option<&Arc<TaskStore>>,
2570) {
2571    // Edge-triggered: only log when the affected-agents set changes, so a
2572    // persistent condition doesn't flood the log every sync tick.
2573    const STALE_AGENTS_KV: &str = "metrics:orch.model_pool_stale_persistent.agents";
2574
2575    // Complexity tiers to check for configured models.
2576    const COMPLEXITIES: &[&str] = &["simple", "medium", "complex", "review"];
2577    const HEAVILY_COOLED_RATIO_THRESHOLD: f64 = 0.5;
2578
2579    let mut details: Vec<DegradedAgentDetail> = Vec::new();
2580    let mut pool_health: Vec<AgentPoolHealth> = Vec::new();
2581
2582    for agent in available_agents {
2583        let agent_in_cd = is_agent_in_cooldown(agent);
2584
2585        // Collect individually cooled models across all complexity tiers (deduped).
2586        let mut cooled_models: Vec<String> = Vec::new();
2587        let mut persistent_failed_models: Vec<String> = Vec::new();
2588        let mut seen_models = HashSet::new();
2589        for comp in COMPLEXITIES {
2590            if let Some(pool) = config.model_pool_for_complexity(agent, comp) {
2591                for model in pool {
2592                    if seen_models.insert(model.clone()) && is_model_in_cooldown(agent, &model) {
2593                        let key = format!("{agent}:{model}");
2594                        if matches!(
2595                            cooldown_reason(&key).as_deref(),
2596                            Some("persistent_model_error" | "model_not_found")
2597                        ) {
2598                            persistent_failed_models.push(model.clone());
2599                        }
2600                        cooled_models.push(model);
2601                    }
2602                }
2603            }
2604        }
2605        let total_configured_models = seen_models.len();
2606        pool_health.push(AgentPoolHealth {
2607            agent: agent.clone(),
2608            cooled_models: cooled_models.clone(),
2609            persistent_failed_models: persistent_failed_models.clone(),
2610            total_configured_models,
2611        });
2612
2613        // An agent is degraded if it is in agent-level cooldown OR has no
2614        // available (non-cooled) model across any complexity tier.
2615        let has_model = COMPLEXITIES
2616            .iter()
2617            .any(|comp| config.has_available_model_for_complexity(agent, comp));
2618
2619        if agent_in_cd || !has_model {
2620            let reason = if agent_in_cd {
2621                cooldown_reason(agent).unwrap_or_else(|| "agent_cooldown".to_string())
2622            } else {
2623                "no_available_model".to_string()
2624            };
2625            details.push(DegradedAgentDetail {
2626                agent: agent.clone(),
2627                reason,
2628                cooled_models,
2629            });
2630        }
2631    }
2632
2633    let count = details.len();
2634
2635    // Always persist the count metric so operators can scrape it every tick,
2636    // including when the count is 0.
2637    if let Err(e) = try_kv_set_prefer_store(
2638        store,
2639        "metrics:orch.agents_degraded.count",
2640        &count.to_string(),
2641    )
2642    .await
2643    {
2644        tracing::warn!(err = %e, "failed to persist degraded agents count metric");
2645    }
2646
2647    if count >= 3 {
2648        // Build structured dimension strings for the log.
2649        let degraded_agents: Vec<&str> = details.iter().map(|d| d.agent.as_str()).collect();
2650        let cooled_models_dim: Vec<String> = details
2651            .iter()
2652            .filter(|d| !d.cooled_models.is_empty())
2653            .map(|d| format!("{}:[{}]", d.agent, d.cooled_models.join(",")))
2654            .collect();
2655        let reasons_dim: Vec<String> = details
2656            .iter()
2657            .map(|d| format!("{}={}", d.agent, d.reason))
2658            .collect();
2659
2660        tracing::warn!(
2661            degraded_count = count,
2662            degraded_agents = ?degraded_agents,
2663            cooled_models = %cooled_models_dim.join("; "),
2664            cooldown_reasons = %reasons_dim.join("; "),
2665            "multi-agent degradation detected"
2666        );
2667
2668        // Dedicated alert metric: "1" while the threshold is exceeded.
2669        if let Err(e) =
2670            try_kv_set_prefer_store(store, "metrics:orch.agents_degraded.alert", "1").await
2671        {
2672            tracing::warn!(err = %e, "failed to persist degraded agents alert metric");
2673        }
2674    } else if let Err(e) =
2675        try_kv_set_prefer_store(store, "metrics:orch.agents_degraded.alert", "0").await
2676    {
2677        // Clear the alert metric when healthy.
2678        tracing::warn!(err = %e, "failed to clear degraded agents alert metric");
2679    }
2680
2681    // Additional operator signal for dead/stale model pools:
2682    // alert when any agent has at least 50% of configured models cooled and at
2683    // least one persistent model failure marker in cooldown reason.
2684    let heavily_cooled_agents: Vec<String> = pool_health
2685        .iter()
2686        .filter_map(|d| {
2687            if d.total_configured_models == 0 || d.persistent_failed_models.is_empty() {
2688                return None;
2689            }
2690            #[expect(
2691                clippy::cast_precision_loss,
2692                reason = "model counts are far below 2^52"
2693            )]
2694            let ratio = d.cooled_models.len() as f64 / d.total_configured_models as f64;
2695            (ratio >= HEAVILY_COOLED_RATIO_THRESHOLD).then(|| {
2696                format!(
2697                    "{}:{}/{}:{}",
2698                    d.agent,
2699                    d.cooled_models.len(),
2700                    d.total_configured_models,
2701                    d.persistent_failed_models.join(",")
2702                )
2703            })
2704        })
2705        .collect();
2706
2707    let current_dim = heavily_cooled_agents.join(",");
2708    let prev_dim = kv_get_prefer_store(store, STALE_AGENTS_KV)
2709        .await
2710        .unwrap_or_default();
2711
2712    if heavily_cooled_agents.is_empty() {
2713        if !prev_dim.is_empty() {
2714            tracing::info!("agent model pool stale condition cleared");
2715        }
2716        if let Err(e) = try_kv_set_prefer_store(store, STALE_AGENTS_KV, "").await {
2717            tracing::warn!(err = %e, "failed to clear stale model pool agents state");
2718        }
2719        if let Err(e) =
2720            try_kv_set_prefer_store(store, "metrics:orch.model_pool_stale_persistent.alert", "0")
2721                .await
2722        {
2723            tracing::warn!(err = %e, "failed to clear stale model pool alert metric");
2724        }
2725    } else {
2726        if current_dim != prev_dim {
2727            tracing::warn!(
2728                affected_agents = ?heavily_cooled_agents,
2729                "agent model pool appears stale: persistent model failures in heavily cooled pool"
2730            );
2731        }
2732        if let Err(e) = try_kv_set_prefer_store(store, STALE_AGENTS_KV, &current_dim).await {
2733            tracing::warn!(err = %e, "failed to persist stale model pool agents state");
2734        }
2735        if let Err(e) =
2736            try_kv_set_prefer_store(store, "metrics:orch.model_pool_stale_persistent.alert", "1")
2737                .await
2738        {
2739            tracing::warn!(err = %e, "failed to persist stale model pool alert metric");
2740        }
2741    }
2742}
2743
2744/// Ingest all active external tasks into the unified SQLite store.
2745///
2746/// Upserts each task so the store stays in sync with the backend.
2747/// Fetches all open issues (including unlabeled ones) so newly created
2748/// issues appear in the store immediately, not only after they get a
2749/// `status:*` label.  This is best-effort — individual task failures
2750/// are logged and skipped.
2751pub(crate) async fn ingest_external_tasks(
2752    backend: &Arc<dyn ExternalBackend>,
2753    repo: &str,
2754    store: &Arc<TaskStore>,
2755) -> anyhow::Result<()> {
2756    #[derive(Clone)]
2757    struct DuplicateTarget {
2758        id: String,
2759        status_label: String,
2760    }
2761
2762    let mut existing_by_title: HashMap<String, DuplicateTarget> = HashMap::new();
2763    let mut accepted_by_title: HashMap<String, DuplicateTarget> = HashMap::new();
2764
2765    let cutoff = (chrono::Utc::now() - chrono::Duration::hours(24))
2766        .format("%Y-%m-%dT%H:%M:%SZ")
2767        .to_string();
2768    match store.list_for_doctor(repo, &cutoff).await {
2769        Ok(recent) => {
2770            let mut insert_target = |task: &Task| {
2771                if task.origin == "internal" {
2772                    return;
2773                }
2774                let Some(ext_id) = task.external_id.as_ref() else {
2775                    return;
2776                };
2777                let status_label = task
2778                    .labels
2779                    .iter()
2780                    .find(|l| l.starts_with("status:"))
2781                    .cloned()
2782                    .unwrap_or_default();
2783                existing_by_title
2784                    .entry(task.title.clone())
2785                    .or_insert(DuplicateTarget {
2786                        id: ext_id.clone(),
2787                        status_label,
2788                    });
2789            };
2790
2791            // Prefer active tasks over recently done ones when selecting a canonical issue.
2792            for task in recent.iter().filter(|t| t.status != TaskStatus::Done) {
2793                insert_target(task);
2794            }
2795            for task in recent.iter().filter(|t| t.status == TaskStatus::Done) {
2796                insert_target(task);
2797            }
2798        }
2799        Err(e) => {
2800            tracing::warn!(repo, err = %e, "ingest: failed to load recent tasks for dedup");
2801        }
2802    }
2803
2804    // Fetch every open issue, with no `since` filter. The GitHub backend overrides
2805    // list_active_open_issues() to use one list_all_open_issues() request.
2806    //
2807    // A `since` cursor keys on updated_at, so an open issue with no task was skipped
2808    // forever once the cursor passed its updated_at, and the cursor had to be reset,
2809    // sentinelled and periodically ignored to recover (#3545, #3787, #3792). An
2810    // unfiltered listing has no such state and is cheaper too: its URL repeats, so
2811    // `get_all_pages` sends it conditionally and GitHub answers 304 while nothing
2812    // changed. A 304 does not count against the REST quota, a `since` fetch does.
2813    let all_tasks: Vec<(ExternalTask, Option<Status>)> =
2814        match backend.list_active_open_issues().await {
2815            Ok(tasks) => tasks,
2816            Err(e) => {
2817                tracing::warn!(
2818                    repo,
2819                    err = %e,
2820                    "ingest: failed to list open issues — no new issue is ingested this cycle"
2821                );
2822                return Ok(());
2823            }
2824        };
2825
2826    // Upsert into the store, collecting store IDs for batch status check
2827    // Also acknowledge newly detected issues (eyes reaction).
2828    let mut id_status_pairs: Vec<(i64, Status)> = Vec::new();
2829    // Collect (store_id, node_id) for newly ingested tasks so we can batch-fetch
2830    // estimate values from GitHub Projects after the loop.
2831    let mut new_task_node_ids: Vec<(i64, String)> = Vec::new();
2832    // Pre-load existing external IDs for this repo to avoid N+1 queries
2833    // (each get_by_external_id call is an individual SQL query). If the
2834    // lookup fails, fall back to an empty set so we conservatively treat
2835    // tasks as new (they will be upserted below).
2836    let existing_ext_ids: HashSet<String> = match store.existing_external_ids(repo).await {
2837        Ok(s) => s,
2838        Err(e) => {
2839            tracing::warn!(repo = repo, err = %e, "ingest: failed to load existing external ids — proceeding");
2840            HashSet::new()
2841        }
2842    };
2843    for (task, status) in &all_tasks {
2844        // Check if this task already exists in the store before upserting.
2845        // Use the pre-loaded HashSet to avoid issuing N individual queries.
2846        let is_new = !existing_ext_ids.contains(&task.id.0);
2847
2848        if is_new {
2849            let duplicate_target = existing_by_title
2850                .get(&task.title)
2851                .or_else(|| accepted_by_title.get(&task.title));
2852            if let Some(target) = duplicate_target
2853                && target.id != task.id.0
2854            {
2855                let label = target.status_label.as_str();
2856                let should_dedupe =
2857                    match backend.has_open_issue_with_title(&task.title, label).await {
2858                        Ok(true) => true,
2859                        Ok(false) => false,
2860                        Err(e) => {
2861                            tracing::warn!(
2862                                task_id = task.id.0,
2863                                err = %e,
2864                                "ingest: dedup check failed — defaulting to keep issue"
2865                            );
2866                            false
2867                        }
2868                    };
2869
2870                if should_dedupe {
2871                    let comment = format!(
2872                        "Duplicate of #{id} — closing to avoid duplicate work.{footer}",
2873                        id = target.id,
2874                        footer = engine::orch_footer()
2875                    );
2876                    if let Err(e) = backend.post_comment(&task.id, &comment).await {
2877                        tracing::warn!(
2878                            task_id = task.id.0,
2879                            err = %e,
2880                            "ingest: failed to comment on duplicate issue"
2881                        );
2882                    }
2883                    match backend.update_status(&task.id, Status::Done).await {
2884                        Ok(()) => {
2885                            tracing::info!(
2886                                task_id = task.id.0,
2887                                duplicate_of = target.id,
2888                                "ingest: closed duplicate issue"
2889                            );
2890                            continue;
2891                        }
2892                        Err(e) => {
2893                            tracing::warn!(
2894                                task_id = task.id.0,
2895                                err = %e,
2896                                "ingest: failed to close duplicate issue — falling through to ingest normally"
2897                            );
2898                        }
2899                    }
2900                }
2901            }
2902        }
2903
2904        match store.ensure_external_task(repo, task).await {
2905            Ok(store_id) => {
2906                if let Ok(number) = task.id.0.parse::<i64>()
2907                    && let Err(err) = dependencies::sync_issue_dependencies(
2908                        backend.as_ref(),
2909                        store,
2910                        store_id,
2911                        repo,
2912                        Some(number),
2913                        &task.body,
2914                    )
2915                    .await
2916                {
2917                    tracing::warn!(
2918                        task_id = task.id.0,
2919                        err = %err,
2920                        "ingest: dependency sync failed"
2921                    );
2922                }
2923                if let Some(s) = status {
2924                    id_status_pairs.push((store_id, *s));
2925                }
2926                // Acknowledge newly detected issues with an eyes reaction.
2927                if is_new {
2928                    let status_label = task
2929                        .labels
2930                        .iter()
2931                        .find(|l| l.starts_with("status:"))
2932                        .cloned()
2933                        .unwrap_or_default();
2934                    accepted_by_title
2935                        .entry(task.title.clone())
2936                        .or_insert(DuplicateTarget {
2937                            id: task.id.0.clone(),
2938                            status_label,
2939                        });
2940                    if let Err(e) = backend.acknowledge_issue(&task.id).await {
2941                        tracing::debug!(
2942                            task_id = task.id.0,
2943                            err = %e,
2944                            "ingest: acknowledgment failed"
2945                        );
2946                    }
2947                    // Sync to project board only for newly ingested tasks.
2948                    // Collect the returned node_id so we can batch-fetch estimates.
2949                    let task_status = status.unwrap_or(Status::New);
2950                    match backend.sync_to_project(&task.id, task_status).await {
2951                        Ok(Some(node_id)) => {
2952                            new_task_node_ids.push((store_id, node_id));
2953                        }
2954                        Ok(None) => {
2955                            // Project sync not configured or failed — no node_id to collect.
2956                        }
2957                        Err(e) => {
2958                            tracing::debug!(
2959                                task_id = task.id.0,
2960                                err = %e,
2961                                "ingest: project board sync failed"
2962                            );
2963                        }
2964                    }
2965                }
2966            }
2967            Err(e) => {
2968                tracing::debug!(task_id = task.id.0, ?e, "ingest: upsert failed");
2969            }
2970        }
2971    }
2972
2973    // Batch-fetch all upserted tasks in a single query, then sync status for NEW tasks.
2974    // Only sync status from backend → store for NEW tasks (first ingest).
2975    // Once a task exists in the store, its status is authoritative —
2976    // re-ingestion must not overwrite store-first status changes
2977    // (e.g., store has Routed but GitHub still shows New labels).
2978    if !id_status_pairs.is_empty() {
2979        let all_ids: Vec<i64> = id_status_pairs.iter().map(|(id, _)| *id).collect();
2980        let existing_map = match store.get_batch(&all_ids).await {
2981            Ok(map) => map,
2982            Err(e) => {
2983                tracing::warn!(err = %e, task_count = all_ids.len(), "ingest: get_batch failed — skipping initial status sync for newly ingested tasks this tick");
2984                return Ok(());
2985            }
2986        };
2987        for (store_id, status) in id_status_pairs {
2988            if let Some(existing) = existing_map.get(&store_id)
2989                && existing.status == TaskStatus::New
2990            {
2991                let db_status = status_to_task_status(status);
2992                if db_status != TaskStatus::New
2993                    && let Err(e) = store.update_status(store_id, db_status).await
2994                {
2995                    tracing::debug!(?e, "ingest: status sync failed");
2996                }
2997            }
2998        }
2999    }
3000
3001    // Batch-fetch estimate values from GitHub Projects for newly ingested tasks.
3002    // For each task where `tasks.estimate == 0`, populate it from the project board.
3003    if !new_task_node_ids.is_empty() {
3004        let node_ids: Vec<String> = new_task_node_ids.iter().map(|(_, n)| n.clone()).collect();
3005        match backend.get_project_item_estimates(&node_ids).await {
3006            Ok(estimates) => {
3007                for (store_id, node_id) in new_task_node_ids {
3008                    if let Some(&estimate) = estimates.get(&node_id)
3009                        && estimate > 0
3010                        && let Err(e) = store.set_estimate_if_zero(store_id, estimate).await
3011                    {
3012                        tracing::debug!(
3013                            store_id,
3014                            estimate,
3015                            err = %e,
3016                            "ingest: failed to set estimate from project"
3017                        );
3018                    }
3019                }
3020            }
3021            Err(e) => {
3022                tracing::debug!(
3023                    err = %e,
3024                    "ingest: project estimate fetch failed — skipping estimate sync this tick"
3025                );
3026            }
3027        }
3028    }
3029
3030    Ok(())
3031}
3032
3033#[cfg(test)]
3034mod tests {
3035    use super::*;
3036    use super::{CommentAction, MentionCursor, classify_comment};
3037    use crate::backends::test_helpers::NoopBackend;
3038    use crate::backends::{ExternalId, ExternalTask, Mention, Status};
3039    use crate::engine::commands::OwnerCommand;
3040    use crate::engine::cooldown::record_persistent_model_failure;
3041    use crate::engine::cooldown::set_agent_cooldown;
3042    use crate::engine::dispatch_guard::DispatchGuard;
3043    use crate::engine::router::Router;
3044    use crate::engine::router::RouterConfig;
3045    use crate::github::types::GitHubIssue;
3046    use crate::github::types::GitHubUser;
3047    use crate::store::NewTask;
3048    use crate::store::{RunTokenUsage, TaskStore};
3049    use async_trait::async_trait;
3050    use serial_test::serial;
3051    use std::slice;
3052    use std::sync::Arc;
3053    use std::sync::atomic::AtomicUsize;
3054    use std::sync::atomic::Ordering;
3055    use tokio::sync::Mutex;
3056    use tokio::sync::oneshot;
3057
3058    // ── ingest_external_tasks tests ─────────────────────────────────────────
3059
3060    /// Mock backend that returns configurable tasks per status.
3061    struct IngestMockBackend {
3062        /// Stored as (status_label, tasks) pairs since Status doesn't impl Hash.
3063        tasks: Vec<(String, Vec<ExternalTask>)>,
3064        dedup_result: bool,
3065        comments: Mutex<Vec<(String, String)>>,
3066        status_updates: Mutex<Vec<(String, Status)>>,
3067        project_syncs: Mutex<Vec<(String, Status)>>,
3068        project_reconciliations: AtomicUsize,
3069        /// Counts `list_active_open_issues` calls.
3070        list_calls: AtomicUsize,
3071    }
3072
3073    impl IngestMockBackend {
3074        fn with_tasks(tasks: Vec<(Status, ExternalTask)>) -> Arc<Self> {
3075            Self::with_tasks_and_dedup(tasks, false)
3076        }
3077
3078        fn with_tasks_and_dedup(
3079            tasks: Vec<(Status, ExternalTask)>,
3080            dedup_result: bool,
3081        ) -> Arc<Self> {
3082            let mut grouped: Vec<(String, Vec<ExternalTask>)> = Vec::new();
3083            for (status, task) in tasks {
3084                let label = status.as_label().to_string();
3085                if let Some(entry) = grouped.iter_mut().find(|(l, _)| l == &label) {
3086                    entry.1.push(task);
3087                } else {
3088                    grouped.push((label, vec![task]));
3089                }
3090            }
3091            Arc::new(Self {
3092                tasks: grouped,
3093                dedup_result,
3094                comments: Mutex::new(Vec::new()),
3095                status_updates: Mutex::new(Vec::new()),
3096                project_syncs: Mutex::new(Vec::new()),
3097                project_reconciliations: AtomicUsize::new(0),
3098                list_calls: AtomicUsize::new(0),
3099            })
3100        }
3101    }
3102
3103    #[async_trait]
3104    impl ExternalBackend for IngestMockBackend {
3105        fn name(&self) -> &'static str {
3106            "ingest-mock"
3107        }
3108        async fn create_task(&self, _: &str, _: &str, _: &[String]) -> anyhow::Result<ExternalId> {
3109            Ok(ExternalId("new".into()))
3110        }
3111        async fn get_task(&self, _: &ExternalId) -> anyhow::Result<ExternalTask> {
3112            anyhow::bail!("not implemented")
3113        }
3114        async fn list_by_status(&self, status: Status) -> anyhow::Result<Vec<ExternalTask>> {
3115            let label = status.as_label().to_string();
3116            Ok(self
3117                .tasks
3118                .iter()
3119                .find(|(l, _)| l == &label)
3120                .map(|(_, t)| t.clone())
3121                .unwrap_or_default())
3122        }
3123        async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
3124            Ok(vec![])
3125        }
3126        async fn list_active_open_issues(
3127            &self,
3128        ) -> anyhow::Result<Vec<(ExternalTask, Option<Status>)>> {
3129            self.list_calls.fetch_add(1, Ordering::Relaxed);
3130            let mut result = Vec::new();
3131            for (label, tasks) in &self.tasks {
3132                let status = match label.as_str() {
3133                    "status:routed" => Some(Status::Routed),
3134                    "status:in_progress" => Some(Status::InProgress),
3135                    "status:needs_review" => Some(Status::NeedsReview),
3136                    "status:in_review" => Some(Status::InReview),
3137                    "status:approved" => Some(Status::Approved),
3138                    "status:blocked" => Some(Status::Blocked),
3139                    _ => None,
3140                };
3141                for task in tasks {
3142                    result.push((task.clone(), status));
3143                }
3144            }
3145            Ok(result)
3146        }
3147        async fn post_comment(&self, id: &ExternalId, body: &str) -> anyhow::Result<()> {
3148            self.comments
3149                .lock()
3150                .await
3151                .push((id.0.clone(), body.to_string()));
3152            Ok(())
3153        }
3154        async fn set_labels(&self, _: &ExternalId, _: &[String]) -> anyhow::Result<()> {
3155            Ok(())
3156        }
3157        async fn remove_label(&self, _: &ExternalId, _: &str) -> anyhow::Result<()> {
3158            Ok(())
3159        }
3160        async fn get_sub_issues(&self, _: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
3161            Ok(vec![])
3162        }
3163        async fn health_check(&self) -> anyhow::Result<()> {
3164            Ok(())
3165        }
3166        async fn get_mentions(&self, _: &str) -> anyhow::Result<Vec<Mention>> {
3167            Ok(vec![])
3168        }
3169        async fn update_status(&self, id: &ExternalId, status: Status) -> anyhow::Result<()> {
3170            self.status_updates
3171                .lock()
3172                .await
3173                .push((id.0.clone(), status));
3174            Ok(())
3175        }
3176
3177        async fn has_open_issue_with_title(&self, _: &str, _: &str) -> anyhow::Result<bool> {
3178            Ok(self.dedup_result)
3179        }
3180
3181        async fn sync_to_project(
3182            &self,
3183            id: &ExternalId,
3184            status: Status,
3185        ) -> anyhow::Result<Option<String>> {
3186            self.project_syncs.lock().await.push((id.0.clone(), status));
3187            Ok(None) // Mock doesn't have real node IDs.
3188        }
3189
3190        async fn reconcile_project_statuses(&self) -> anyhow::Result<()> {
3191            self.project_reconciliations.fetch_add(1, Ordering::Relaxed);
3192            Ok(())
3193        }
3194    }
3195
3196    fn make_ext_task(id: &str, title: &str) -> ExternalTask {
3197        ExternalTask {
3198            id: ExternalId(id.to_string()),
3199            title: title.to_string(),
3200            body: String::new(),
3201            state: "open".to_string(),
3202            labels: vec![],
3203            author: "user".to_string(),
3204            created_at: "2026-01-01T00:00:00Z".to_string(),
3205            updated_at: "2026-01-01T00:00:00Z".to_string(),
3206            url: format!("https://github.com/owner/repo/issues/{id}"),
3207        }
3208    }
3209
3210    #[serial(cooldown_state)]
3211    #[tokio::test]
3212    async fn ingest_upserts_tasks_into_store() {
3213        cooldown::reset_global_state().await;
3214        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![
3215            (Status::New, make_ext_task("1", "First issue")),
3216            (Status::InProgress, make_ext_task("2", "Second issue")),
3217        ]);
3218        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3219
3220        ingest_external_tasks(&backend, "owner/repo", &store)
3221            .await
3222            .unwrap();
3223
3224        // Both tasks should be in the store
3225        let all = store.list_all("owner/repo").await.unwrap();
3226        assert_eq!(all.len(), 2);
3227
3228        let task1 = store
3229            .get_by_external_id("owner/repo", "1")
3230            .await
3231            .unwrap()
3232            .unwrap();
3233        assert_eq!(task1.title, "First issue");
3234        assert_eq!(task1.status, TaskStatus::New);
3235
3236        let task2 = store
3237            .get_by_external_id("owner/repo", "2")
3238            .await
3239            .unwrap()
3240            .unwrap();
3241        assert_eq!(task2.title, "Second issue");
3242        assert_eq!(task2.status, TaskStatus::InProgress);
3243    }
3244
3245    #[serial(cooldown_state)]
3246    #[tokio::test]
3247    async fn ingest_takes_every_open_issue_whatever_its_age_or_the_last_ingest() {
3248        // An issue opened after a restart reaches the store on the first sync, and so does
3249        // one whose updated_at predates any window a `since` cursor would have used. The
3250        // ingest keeps no cursor, so a stale KV value from an older build changes nothing.
3251        cooldown::reset_global_state().await;
3252        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3253        store
3254            .kv_set("issues_last_ingested:owner/repo", "2026-10-26T00:00:00Z")
3255            .await
3256            .unwrap();
3257
3258        let mut old = make_ext_task("73", "Filed weeks ago, never ingested");
3259        old.updated_at = "2026-01-01T00:00:00Z".to_string();
3260        let backend = IngestMockBackend::with_tasks(vec![(Status::New, old)]);
3261        let backend_dyn: Arc<dyn ExternalBackend> = Arc::<IngestMockBackend>::clone(&backend);
3262        ingest_external_tasks(&backend_dyn, "owner/repo", &store)
3263            .await
3264            .unwrap();
3265        assert!(
3266            store
3267                .get_by_external_id("owner/repo", "73")
3268                .await
3269                .unwrap()
3270                .is_some()
3271        );
3272
3273        // A second sync, standing in for the one right after a restart, sees a new issue.
3274        let backend = IngestMockBackend::with_tasks(vec![(
3275            Status::New,
3276            make_ext_task("78", "Opened after the restart"),
3277        )]);
3278        let backend_dyn: Arc<dyn ExternalBackend> = Arc::<IngestMockBackend>::clone(&backend);
3279        ingest_external_tasks(&backend_dyn, "owner/repo", &store)
3280            .await
3281            .unwrap();
3282        assert_eq!(
3283            backend.list_calls.load(Ordering::Relaxed),
3284            1,
3285            "one sync is one listing"
3286        );
3287        assert!(
3288            store
3289                .get_by_external_id("owner/repo", "78")
3290                .await
3291                .unwrap()
3292                .is_some()
3293        );
3294        assert_eq!(
3295            store
3296                .kv_get("issues_last_ingested:owner/repo")
3297                .await
3298                .unwrap()
3299                .as_deref(),
3300            Some("2026-10-26T00:00:00Z"),
3301            "the ingest must keep no cursor, so a stale one is never read or advanced"
3302        );
3303    }
3304
3305    #[serial(cooldown_state)]
3306    #[tokio::test]
3307    async fn ingest_updates_existing_tasks() {
3308        cooldown::reset_global_state().await;
3309        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![(
3310            Status::Routed,
3311            make_ext_task("42", "Updated title"),
3312        )]);
3313        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3314
3315        // Pre-create the task
3316        store
3317            .create(&NewTask {
3318                external_id: Some("42".to_string()),
3319                repo: "owner/repo".to_string(),
3320                origin: "github".to_string(),
3321                title: "Original title".to_string(),
3322                ..Default::default()
3323            })
3324            .await
3325            .unwrap();
3326
3327        ingest_external_tasks(&backend, "owner/repo", &store)
3328            .await
3329            .unwrap();
3330
3331        // Should have updated the title
3332        let task = store
3333            .get_by_external_id("owner/repo", "42")
3334            .await
3335            .unwrap()
3336            .unwrap();
3337        assert_eq!(task.title, "Updated title");
3338        assert_eq!(task.status, TaskStatus::Routed);
3339    }
3340
3341    #[serial(cooldown_state)]
3342    #[tokio::test]
3343    async fn ingest_closes_duplicate_titles() {
3344        cooldown::reset_global_state().await;
3345
3346        let backend = IngestMockBackend::with_tasks_and_dedup(
3347            vec![(Status::New, make_ext_task("2", "Duplicate issue"))],
3348            true,
3349        );
3350        let backend_dyn: Arc<dyn ExternalBackend> = Arc::<IngestMockBackend>::clone(&backend);
3351        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3352
3353        store
3354            .create(&NewTask {
3355                external_id: Some("1".to_string()),
3356                repo: "owner/repo".to_string(),
3357                origin: "github".to_string(),
3358                title: "Duplicate issue".to_string(),
3359                labels: vec!["status:new".to_string()],
3360                ..Default::default()
3361            })
3362            .await
3363            .unwrap();
3364
3365        ingest_external_tasks(&backend_dyn, "owner/repo", &store)
3366            .await
3367            .unwrap();
3368
3369        let all = store.list_all("owner/repo").await.unwrap();
3370        assert_eq!(all.len(), 1);
3371
3372        let comments = backend.comments.lock().await;
3373        assert_eq!(comments.len(), 1);
3374        assert!(comments[0].1.contains("Duplicate of #1"));
3375
3376        let status_updates = backend.status_updates.lock().await;
3377        assert_eq!(status_updates.len(), 1);
3378        assert_eq!(status_updates[0].0, "2");
3379        assert_eq!(status_updates[0].1, Status::Done);
3380    }
3381
3382    #[serial(cooldown_state)]
3383    #[tokio::test]
3384    async fn ingest_handles_empty_backend() {
3385        cooldown::reset_global_state().await;
3386        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
3387        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3388
3389        // Should not error on empty backend
3390        let result = ingest_external_tasks(&backend, "owner/repo", &store).await;
3391        assert!(result.is_ok());
3392
3393        let all = store.list_all("owner/repo").await.unwrap();
3394        assert!(all.is_empty());
3395    }
3396
3397    #[serial(cooldown_state)]
3398    #[tokio::test]
3399    async fn ingest_syncs_status_correctly_across_statuses() {
3400        cooldown::reset_global_state().await;
3401        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![
3402            (Status::New, make_ext_task("1", "New")),
3403            (Status::Routed, make_ext_task("2", "Routed")),
3404            (Status::InProgress, make_ext_task("3", "InProgress")),
3405            (Status::NeedsReview, make_ext_task("4", "NeedsReview")),
3406            (Status::InReview, make_ext_task("5", "InReview")),
3407            (Status::Blocked, make_ext_task("6", "Blocked")),
3408        ]);
3409        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3410
3411        ingest_external_tasks(&backend, "owner/repo", &store)
3412            .await
3413            .unwrap();
3414
3415        let counts = store.status_counts("owner/repo").await.unwrap();
3416        assert_eq!(counts.get("new"), Some(&1));
3417        assert_eq!(counts.get("routed"), Some(&1));
3418        assert_eq!(counts.get("in_progress"), Some(&1));
3419        assert_eq!(counts.get("needs_review"), Some(&1));
3420        assert_eq!(counts.get("in_review"), Some(&1));
3421        assert_eq!(counts.get("blocked"), Some(&1));
3422    }
3423
3424    #[serial(cooldown_state)]
3425    #[tokio::test]
3426    async fn ingest_does_not_overwrite_store_authoritative_status() {
3427        cooldown::reset_global_state().await;
3428        // Backend reports task as New (GitHub labels haven't caught up yet)
3429        let backend: Arc<dyn ExternalBackend> =
3430            IngestMockBackend::with_tasks(vec![(Status::New, make_ext_task("99", "My task"))]);
3431        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3432
3433        // Pre-create the task and advance it to Routed in the store
3434        let id = store
3435            .create(&NewTask {
3436                external_id: Some("99".to_string()),
3437                repo: "owner/repo".to_string(),
3438                origin: "github".to_string(),
3439                title: "My task".to_string(),
3440                ..Default::default()
3441            })
3442            .await
3443            .unwrap();
3444        store.update_status(id, TaskStatus::Routed).await.unwrap();
3445
3446        // Ingest should NOT overwrite Routed back to New
3447        ingest_external_tasks(&backend, "owner/repo", &store)
3448            .await
3449            .unwrap();
3450
3451        let task = store.get(id).await.unwrap();
3452        assert_eq!(
3453            task.status,
3454            TaskStatus::Routed,
3455            "ingest must not overwrite store-authoritative status"
3456        );
3457    }
3458
3459    #[serial(cooldown_state)]
3460    #[tokio::test]
3461    async fn ingest_sync_to_project_called_only_for_new_tasks() {
3462        cooldown::reset_global_state().await;
3463
3464        // Two tasks: "10" is pre-existing, "11" is new.
3465        let backend = IngestMockBackend::with_tasks(vec![
3466            (Status::New, make_ext_task("10", "Existing task")),
3467            (Status::New, make_ext_task("11", "New task")),
3468        ]);
3469        let backend_dyn: Arc<dyn ExternalBackend> = Arc::<IngestMockBackend>::clone(&backend);
3470        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3471
3472        // Pre-insert task "10" so it already exists in the store.
3473        store
3474            .create(&NewTask {
3475                external_id: Some("10".to_string()),
3476                repo: "owner/repo".to_string(),
3477                origin: "github".to_string(),
3478                title: "Existing task".to_string(),
3479                ..Default::default()
3480            })
3481            .await
3482            .unwrap();
3483
3484        ingest_external_tasks(&backend_dyn, "owner/repo", &store)
3485            .await
3486            .unwrap();
3487
3488        let syncs = backend.project_syncs.lock().await;
3489        // Only the new task "11" should trigger sync_to_project — not the pre-existing "10".
3490        assert_eq!(
3491            syncs.len(),
3492            1,
3493            "sync_to_project must be called exactly once (for new task only), got: {syncs:?}"
3494        );
3495        assert_eq!(syncs[0].0, "11", "sync_to_project must target the new task");
3496    }
3497
3498    #[serial(cooldown_state)]
3499    #[tokio::test]
3500    async fn ingest_handles_mixed_labeled_and_unlabeled_open_issues() {
3501        // Simulate a backend where some issues have no status label (routable/unlabeled)
3502        // and others carry active status labels.  Both kinds must be ingested and the
3503        // store status must reflect the label where present.
3504        struct MixedBackend;
3505
3506        #[async_trait]
3507        impl ExternalBackend for MixedBackend {
3508            fn name(&self) -> &'static str {
3509                "mixed"
3510            }
3511            async fn create_task(
3512                &self,
3513                _: &str,
3514                _: &str,
3515                _: &[String],
3516            ) -> anyhow::Result<ExternalId> {
3517                Ok(ExternalId("x".into()))
3518            }
3519            async fn get_task(&self, _: &ExternalId) -> anyhow::Result<ExternalTask> {
3520                anyhow::bail!("not implemented")
3521            }
3522            async fn list_by_status(&self, _: Status) -> anyhow::Result<Vec<ExternalTask>> {
3523                Ok(vec![])
3524            }
3525            /// Return a mix: one unlabeled (no status) + one labeled in_progress.
3526            async fn list_active_open_issues(
3527                &self,
3528            ) -> anyhow::Result<Vec<(ExternalTask, Option<Status>)>> {
3529                Ok(vec![
3530                    (make_ext_task("100", "Unlabeled routable"), None),
3531                    (
3532                        make_ext_task("101", "In progress issue"),
3533                        Some(Status::InProgress),
3534                    ),
3535                ])
3536            }
3537            async fn post_comment(&self, _: &ExternalId, _: &str) -> anyhow::Result<()> {
3538                Ok(())
3539            }
3540            async fn set_labels(&self, _: &ExternalId, _: &[String]) -> anyhow::Result<()> {
3541                Ok(())
3542            }
3543            async fn remove_label(&self, _: &ExternalId, _: &str) -> anyhow::Result<()> {
3544                Ok(())
3545            }
3546            async fn get_sub_issues(&self, _: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
3547                Ok(vec![])
3548            }
3549            async fn health_check(&self) -> anyhow::Result<()> {
3550                Ok(())
3551            }
3552            async fn get_mentions(&self, _: &str) -> anyhow::Result<Vec<Mention>> {
3553                Ok(vec![])
3554            }
3555            async fn update_status(&self, _: &ExternalId, _: Status) -> anyhow::Result<()> {
3556                Ok(())
3557            }
3558            async fn has_open_issue_with_title(&self, _: &str, _: &str) -> anyhow::Result<bool> {
3559                Ok(false)
3560            }
3561            async fn sync_to_project(
3562                &self,
3563                _: &ExternalId,
3564                _: Status,
3565            ) -> anyhow::Result<Option<String>> {
3566                Ok(None)
3567            }
3568        }
3569
3570        cooldown::reset_global_state().await;
3571
3572        let backend: Arc<dyn ExternalBackend> = Arc::new(MixedBackend);
3573        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3574
3575        ingest_external_tasks(&backend, "owner/repo", &store)
3576            .await
3577            .unwrap();
3578
3579        let all = store.list_all("owner/repo").await.unwrap();
3580        assert_eq!(all.len(), 2, "both issues must be ingested");
3581
3582        let unlabeled = store
3583            .get_by_external_id("owner/repo", "100")
3584            .await
3585            .unwrap()
3586            .unwrap();
3587        assert_eq!(
3588            unlabeled.status,
3589            TaskStatus::New,
3590            "unlabeled issue should start as New"
3591        );
3592
3593        let in_progress = store
3594            .get_by_external_id("owner/repo", "101")
3595            .await
3596            .unwrap()
3597            .unwrap();
3598        assert_eq!(
3599            in_progress.status,
3600            TaskStatus::InProgress,
3601            "labeled in_progress issue must be synced to InProgress"
3602        );
3603    }
3604
3605    // ── kv_get_prefer_store / kv_set_prefer_store ────────────────────
3606
3607    #[serial(cooldown_state)]
3608    #[tokio::test]
3609    async fn kv_get_prefer_store_reads_from_store() {
3610        cooldown::reset_global_state().await;
3611        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3612
3613        store.kv_set("k1", "store_val").await.unwrap();
3614
3615        let opt = Some(&store);
3616        let val = kv_get_prefer_store(opt, "k1").await;
3617        assert_eq!(val.as_deref(), Some("store_val"));
3618    }
3619
3620    #[serial(cooldown_state)]
3621    #[tokio::test]
3622    async fn kv_get_prefer_store_returns_none_without_store() {
3623        cooldown::reset_global_state().await;
3624        let opt: Option<&Arc<TaskStore>> = None;
3625        let val = kv_get_prefer_store(opt, "k2").await;
3626        assert_eq!(val, None);
3627    }
3628
3629    #[serial(cooldown_state)]
3630    #[tokio::test]
3631    async fn kv_set_prefer_store_writes_to_store() {
3632        cooldown::reset_global_state().await;
3633        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3634
3635        let opt = Some(&store);
3636        kv_set_prefer_store(opt, "k3", "val3").await;
3637
3638        assert_eq!(store.kv_get("k3").await.unwrap().as_deref(), Some("val3"));
3639    }
3640
3641    #[serial(cooldown_state)]
3642    #[tokio::test]
3643    async fn kv_set_prefer_store_noop_without_store() {
3644        cooldown::reset_global_state().await;
3645        let opt: Option<&Arc<TaskStore>> = None;
3646        // Should not panic
3647        kv_set_prefer_store(opt, "k4", "val4").await;
3648    }
3649
3650    // ── try_kv_get_prefer_store / try_kv_set_prefer_store ────────────────
3651
3652    #[serial(cooldown_state)]
3653    #[tokio::test]
3654    async fn try_kv_get_prefer_store_returns_value_from_store() {
3655        cooldown::reset_global_state().await;
3656        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3657        store.kv_set("try_k1", "try_val1").await.unwrap();
3658
3659        let opt = Some(&store);
3660        let result = try_kv_get_prefer_store(opt, "try_k1").await;
3661        assert_eq!(result.unwrap().as_deref(), Some("try_val1"));
3662    }
3663
3664    #[serial(cooldown_state)]
3665    #[tokio::test]
3666    async fn try_kv_get_prefer_store_returns_none_for_missing_key() {
3667        cooldown::reset_global_state().await;
3668        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3669        let opt = Some(&store);
3670
3671        let result = try_kv_get_prefer_store(opt, "nonexistent_key").await;
3672        assert_eq!(result.unwrap(), None); // Key doesn't exist, but query succeeded
3673    }
3674
3675    #[serial(cooldown_state)]
3676    #[tokio::test]
3677    async fn try_kv_get_prefer_store_returns_none_without_store() {
3678        cooldown::reset_global_state().await;
3679        let opt: Option<&Arc<TaskStore>> = None;
3680        let result = try_kv_get_prefer_store(opt, "try_k2").await;
3681        assert_eq!(result.unwrap(), None);
3682    }
3683
3684    #[serial(cooldown_state)]
3685    #[tokio::test]
3686    async fn try_kv_set_prefer_store_writes_to_store() {
3687        cooldown::reset_global_state().await;
3688        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3689        let opt = Some(&store);
3690
3691        let result = try_kv_set_prefer_store(opt, "try_k3", "try_val3").await;
3692        assert!(result.is_ok());
3693        assert_eq!(
3694            store.kv_get("try_k3").await.unwrap().as_deref(),
3695            Some("try_val3")
3696        );
3697    }
3698
3699    #[serial(cooldown_state)]
3700    #[tokio::test]
3701    async fn try_kv_set_prefer_store_ok_without_store() {
3702        cooldown::reset_global_state().await;
3703        let opt: Option<&Arc<TaskStore>> = None;
3704        // Should not error - best effort when store unavailable
3705        let result = try_kv_set_prefer_store(opt, "try_k4", "try_val4").await;
3706        assert!(result.is_ok());
3707    }
3708
3709    // ── dispatching lock tests ──────────────────────────────────────────
3710
3711    #[serial(cooldown_state)]
3712    #[test]
3713    fn dispatching_set_blocks_duplicate_processing() {
3714        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3715
3716        let repo = "owner/repo";
3717        let task_id = "42";
3718        let dispatch_key = format!("{repo}/{task_id}");
3719
3720        // Initially not in the map
3721        assert!(!dispatching.contains_key(&dispatch_key));
3722
3723        // Insert — simulates dispatch starting
3724        dispatching.insert(dispatch_key.clone(), task_id.to_string());
3725
3726        // Now should be blocked
3727        assert!(
3728            dispatching.contains_key(&dispatch_key),
3729            "task should be locked while dispatching"
3730        );
3731
3732        // Remove — simulates review completion
3733        dispatching.remove(&dispatch_key);
3734
3735        // Now should be free again
3736        assert!(
3737            !dispatching.contains_key(&dispatch_key),
3738            "task should be unlocked after review completes"
3739        );
3740    }
3741
3742    #[serial(cooldown_state)]
3743    #[test]
3744    fn dispatching_set_does_not_block_other_tasks() {
3745        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3746
3747        let repo = "owner/repo";
3748
3749        // Lock task 42
3750        let key_42 = format!("{repo}/42");
3751        dispatching.insert(key_42.clone(), "42".to_string());
3752
3753        // Task 43 should NOT be blocked
3754        let key_43 = format!("{repo}/43");
3755        assert!(
3756            dispatching.contains_key(&key_42),
3757            "task 42 should be locked"
3758        );
3759        assert!(
3760            !dispatching.contains_key(&key_43),
3761            "task 43 should not be locked"
3762        );
3763    }
3764
3765    /// Regression test for a6d8b9a.
3766    ///
3767    /// Before the fix, `sync_tick` step 5 transitioned a task to `InReview` and then
3768    /// spawned a `tokio::spawn` review agent WITHOUT first inserting the `dispatch_key`
3769    /// into the dispatching set. This created a window where `review_open_prs` (step 4)
3770    /// in a concurrent sync tick could see the task as `InReview`, find the key absent,
3771    /// and re-dispatch it — silently discarding human `CHANGES_REQUESTED` feedback when
3772    /// the review agent later completed and marked the task `Done`.
3773    ///
3774    /// The fix: insert `dispatch_key` BEFORE `tokio::spawn`, remove it AFTER the future
3775    /// completes. This test verifies that invariant using tokio channels to synchronize
3776    /// with a simulated concurrent caller (review_open_prs).
3777    #[serial(cooldown_state)]
3778    #[tokio::test]
3779    async fn dispatch_key_held_during_review_agent_execution() {
3780        cooldown::reset_global_state().await;
3781
3782        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3783
3784        let dispatch_key = "owner/repo/42".to_string();
3785
3786        // Step 1: insert before spawning — this is the invariant added by a6d8b9a.
3787        dispatching.insert(dispatch_key.clone(), "42".to_string());
3788
3789        let (agent_started_tx, agent_started_rx) = oneshot::channel::<()>();
3790        let (check_done_tx, check_done_rx) = oneshot::channel::<()>();
3791
3792        let dispatching_agent = Arc::clone(&dispatching);
3793        let key_agent = dispatch_key.clone();
3794
3795        // Step 2: spawn review agent — key is ALREADY in the set (fix invariant holds).
3796        let review_task = tokio::spawn(async move {
3797            agent_started_tx.send(()).unwrap();
3798            // Simulate review agent work — hold until the concurrent check is done.
3799            check_done_rx.await.unwrap();
3800            // Remove from dispatching set on completion (mirror of tick.rs/sync.rs).
3801            dispatching_agent.remove(&key_agent);
3802        });
3803
3804        // Step 3: concurrent review_open_prs check — simulates step 4 of another
3805        // sync_tick invocation running while the review agent is still active.
3806        agent_started_rx.await.unwrap();
3807        assert!(
3808            dispatching.contains_key(&dispatch_key),
3809            "dispatch_key must be visible in the dispatching set while the review \
3810             agent is running — without the a6d8b9a fix, review_open_prs would see \
3811             the key absent and re-dispatch the task, silently dropping \
3812             CHANGES_REQUESTED feedback"
3813        );
3814
3815        // Step 4: review agent completes.
3816        check_done_tx.send(()).unwrap();
3817        review_task.await.unwrap();
3818
3819        // Step 5: key released — review_open_prs can now act on the task if needed.
3820        assert!(
3821            !dispatching.contains_key(&dispatch_key),
3822            "dispatch_key must be removed from the dispatching set after the \
3823             review agent completes so subsequent sync ticks can process the task"
3824        );
3825    }
3826
3827    /// Regression test for issue #833.
3828    ///
3829    /// Before the fix, `dispatch_key` was inserted into the `dispatching` HashSet
3830    /// *outside* the spawned task but removed *inside* it.  When the spawned async
3831    /// block panicked (e.g. an `unwrap()` inside `review_and_merge`), Tokio caught
3832    /// the panic and terminated the task without running the cleanup code, leaving
3833    /// the key in the set permanently.  Subsequent stuck-task recovery would reset
3834    /// the task to `NeedsReview`, but the subscriber/sync would see the key still
3835    /// present and skip it forever — a permanent review loop until service restart.
3836    ///
3837    /// The fix introduces `DispatchGuard`, a RAII wrapper whose `Drop` impl removes
3838    /// the key unconditionally, even when the task unwinds via panic.
3839    #[serial(cooldown_state)]
3840    #[tokio::test]
3841    async fn dispatch_guard_releases_key_on_panic() {
3842        cooldown::reset_global_state().await;
3843
3844        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3845        let key = "owner/repo/42".to_string();
3846
3847        // Insert key before spawn — mirrors the production invariant established
3848        // by the a6d8b9a fix (key must be visible before the spawn so concurrent
3849        // review_open_prs callers see it and skip the task).
3850        dispatching.insert(key.clone(), "42".to_string());
3851
3852        // Create the guard (takes ownership of the removal obligation).
3853        let guard = DispatchGuard::new(Arc::clone(&dispatching), key.clone());
3854
3855        // Spawn a task that panics before the normal completion path.
3856        let handle = tokio::spawn(async move {
3857            let _guard = guard; // moved in; Drop removes key whether we panic or not
3858            panic!("simulated panic inside review_and_merge");
3859        });
3860
3861        // Absorb the JoinError — the panic is expected.
3862        let _ = handle.await;
3863
3864        // Key MUST be gone even though the task panicked.
3865        assert!(
3866            !dispatching.contains_key(&key),
3867            "dispatch key must be removed from the dispatching set even when the \
3868             spawned review task panics — without DispatchGuard the key leaks and \
3869             the task gets stuck in a permanent review loop"
3870        );
3871    }
3872
3873    #[serial(cooldown_state)]
3874    #[tokio::test]
3875    async fn sync_tick_reconciles_board_without_ingesting_or_transitioning_tasks() {
3876        cooldown::reset_global_state().await;
3877        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3878        let mock = IngestMockBackend::with_tasks(vec![]);
3879        let backend: Arc<dyn ExternalBackend> = Arc::<IngestMockBackend>::clone(&mock);
3880        let task_manager = Arc::new(TaskManager::with_store(
3881            Arc::clone(&backend),
3882            Arc::clone(&store),
3883            "owner/repo".to_string(),
3884        ));
3885        let dispatching = Arc::new(DashMap::new());
3886        Box::pin(sync_tick(
3887            &backend,
3888            &Arc::new(TmuxManager::new()),
3889            "owner/repo",
3890            &EngineConfig::default(),
3891            &Arc::new(RwLock::new(Router::new_for_test(
3892                RouterConfig::default(),
3893                vec!["claude".to_string()],
3894            ))),
3895            &task_manager,
3896            &store,
3897            &dispatching,
3898            &Arc::new(DashSet::new()),
3899        ))
3900        .await
3901        .unwrap();
3902        assert_eq!(mock.project_reconciliations.load(Ordering::Relaxed), 1);
3903        assert!(mock.project_syncs.lock().await.is_empty());
3904        assert!(mock.status_updates.lock().await.is_empty());
3905        assert!(dispatching.is_empty());
3906        assert!(
3907            store
3908                .existing_external_ids("owner/repo")
3909                .await
3910                .unwrap()
3911                .is_empty()
3912        );
3913    }
3914
3915    #[serial(cooldown_state)]
3916    #[tokio::test]
3917    async fn needs_review_refire_increments_and_fires_when_old_enough() {
3918        cooldown::reset_global_state().await;
3919        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3920        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
3921        let task_manager = Arc::new(TaskManager::with_store(
3922            backend,
3923            Arc::clone(&store),
3924            "owner/repo".to_string(),
3925        ));
3926        let router = reviewer_router();
3927        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3928
3929        let id = store
3930            .upsert_external(&crate::store::UpsertExternal {
3931                repo: "owner/repo",
3932                ext_id: "201",
3933                title: "Needs review old",
3934                body: "",
3935                author: "",
3936                url: "",
3937                labels: &[],
3938                origin: "github",
3939            })
3940            .await
3941            .unwrap();
3942        store
3943            .update_status(id, TaskStatus::NeedsReview)
3944            .await
3945            .unwrap();
3946        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
3947            .bind(id)
3948            .execute(store.pool())
3949            .await
3950            .unwrap();
3951
3952        let t = store.get(id).await.unwrap();
3953        assert_eq!(t.needs_review_refires, 0);
3954
3955        refire_and_escalate_stale_needs_review_global(
3956            &task_manager,
3957            &store,
3958            &router,
3959            &dispatching,
3960            &sweep_repos(),
3961        )
3962        .await
3963        .unwrap();
3964
3965        let t2 = store.get(id).await.unwrap();
3966        assert_eq!(t2.needs_review_refires, 1);
3967        assert_eq!(t2.status, TaskStatus::NeedsReview);
3968    }
3969
3970    #[serial(cooldown_state)]
3971    #[tokio::test]
3972    async fn needs_review_backoff_delays_fire_but_increments_counter() {
3973        cooldown::reset_global_state().await;
3974        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3975        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
3976        let task_manager = Arc::new(TaskManager::with_store(
3977            backend,
3978            Arc::clone(&store),
3979            "owner/repo".to_string(),
3980        ));
3981        let router = reviewer_router();
3982        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3983
3984        let id = store
3985            .upsert_external(&crate::store::UpsertExternal {
3986                repo: "owner/repo",
3987                ext_id: "202",
3988                title: "Needs review backoff",
3989                body: "",
3990                author: "",
3991                url: "",
3992                labels: &[],
3993                origin: "github",
3994            })
3995            .await
3996            .unwrap();
3997        store
3998            .update_status(id, TaskStatus::NeedsReview)
3999            .await
4000            .unwrap();
4001
4002        store
4003            .set_fields(id, &[("needs_review_refires", serde_json::json!(1))])
4004            .await
4005            .unwrap();
4006
4007        // Age 1 minute: stale enough to consider, but backoff at refires=1 needs 2 minutes
4008        let now_minus_1 = (chrono::Utc::now() - chrono::Duration::minutes(1))
4009            .format("%Y-%m-%dT%H:%M:%SZ")
4010            .to_string();
4011        sqlx::query("UPDATE tasks SET updated_at = ? WHERE id = ?")
4012            .bind(now_minus_1.clone())
4013            .bind(id)
4014            .execute(store.pool())
4015            .await
4016            .unwrap();
4017
4018        let before = store.get(id).await.unwrap();
4019        assert_eq!(before.needs_review_refires, 1);
4020
4021        refire_and_escalate_stale_needs_review_global(
4022            &task_manager,
4023            &store,
4024            &router,
4025            &dispatching,
4026            &sweep_repos(),
4027        )
4028        .await
4029        .unwrap();
4030
4031        let after = store.get(id).await.unwrap();
4032        // Counter must not increment on a backoff delay, only on an actual re-fire
4033        assert_eq!(after.needs_review_refires, 1);
4034        assert_eq!(after.updated_at, now_minus_1);
4035        assert_eq!(after.status, TaskStatus::NeedsReview);
4036    }
4037
4038    /// Router with one claude review model, so the refire sweeps see a reviewer.
4039    fn reviewer_router() -> Router {
4040        let mut config = RouterConfig::default();
4041        config.model_map.insert(
4042            "review".to_string(),
4043            HashMap::from([("claude".to_string(), vec!["sonnet".to_string()])]),
4044        );
4045        Router::new_for_test(config, vec!["claude".to_string()])
4046    }
4047
4048    #[serial(cooldown_state)]
4049    #[tokio::test]
4050    async fn needs_review_escalates_to_blocked_after_max_refires() {
4051        cooldown::reset_global_state().await;
4052        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4053        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4054        let task_manager = Arc::new(TaskManager::with_store(
4055            backend,
4056            Arc::clone(&store),
4057            "owner/repo".to_string(),
4058        ));
4059        let router = reviewer_router();
4060        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4061
4062        let id = store
4063            .upsert_external(&crate::store::UpsertExternal {
4064                repo: "owner/repo",
4065                ext_id: "203",
4066                title: "Needs review escalate",
4067                body: "",
4068                author: "",
4069                url: "",
4070                labels: &[],
4071                origin: "github",
4072            })
4073            .await
4074            .unwrap();
4075        store
4076            .update_status(id, TaskStatus::NeedsReview)
4077            .await
4078            .unwrap();
4079
4080        // Counter at MAX (5): the next pass is attempt 6 and escalates
4081        store
4082            .set_fields(id, &[("needs_review_refires", serde_json::json!(5))])
4083            .await
4084            .unwrap();
4085        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4086            .bind(id)
4087            .execute(store.pool())
4088            .await
4089            .unwrap();
4090
4091        refire_and_escalate_stale_needs_review_global(
4092            &task_manager,
4093            &store,
4094            &router,
4095            &dispatching,
4096            &sweep_repos(),
4097        )
4098        .await
4099        .unwrap();
4100
4101        let after = store.get(id).await.unwrap();
4102        assert_eq!(after.status, TaskStatus::Blocked);
4103        assert!(after.block_reason.is_some());
4104    }
4105
4106    /// Exactly MAX_NEEDS_REVIEW_REFIRE_ATTEMPTS refires are allowed before
4107    /// escalation. Task stays NeedsReview at 5 refires and only blocks on the 6th
4108    #[serial(cooldown_state)]
4109    #[tokio::test]
4110    async fn needs_review_allows_max_refires_before_escalation() {
4111        cooldown::reset_global_state().await;
4112        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4113        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4114        let task_manager = Arc::new(TaskManager::with_store(
4115            backend,
4116            Arc::clone(&store),
4117            "owner/repo".to_string(),
4118        ));
4119        let router = reviewer_router();
4120        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4121
4122        let id = store
4123            .upsert_external(&crate::store::UpsertExternal {
4124                repo: "owner/repo",
4125                ext_id: "204",
4126                title: "Max refires test",
4127                body: "",
4128                author: "",
4129                url: "",
4130                labels: &[],
4131                origin: "github",
4132            })
4133            .await
4134            .unwrap();
4135        store
4136            .update_status(id, TaskStatus::NeedsReview)
4137            .await
4138            .unwrap();
4139
4140        // Counter at MAX-1 (4): incrementing hits MAX (5) and must not escalate
4141        store
4142            .set_fields(id, &[("needs_review_refires", serde_json::json!(4))])
4143            .await
4144            .unwrap();
4145        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4146            .bind(id)
4147            .execute(store.pool())
4148            .await
4149            .unwrap();
4150
4151        refire_and_escalate_stale_needs_review_global(
4152            &task_manager,
4153            &store,
4154            &router,
4155            &dispatching,
4156            &sweep_repos(),
4157        )
4158        .await
4159        .unwrap();
4160
4161        let after = store.get(id).await.unwrap();
4162        assert_eq!(after.status, TaskStatus::NeedsReview);
4163        assert_eq!(after.needs_review_refires, 5);
4164        assert!(after.block_reason.is_none());
4165    }
4166
4167    /// Every repo the sweep tests use, so each counts as active
4168    fn sweep_repos() -> HashSet<String> {
4169        ["owner/repo", "inactive/repo", "some/repo", "active/repo"]
4170            .into_iter()
4171            .map(String::from)
4172            .collect()
4173    }
4174
4175    /// A repo outside the active set has no review subscriber (#4024)
4176    #[serial(cooldown_state)]
4177    #[tokio::test]
4178    async fn global_sweeps_leave_tasks_of_a_removed_repo_alone() {
4179        cooldown::reset_global_state().await;
4180        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4181        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4182        let task_manager = Arc::new(TaskManager::with_store(
4183            backend,
4184            Arc::clone(&store),
4185            "active/repo".to_string(),
4186        ));
4187        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4188        let router = router_with_healthy_review_agent();
4189        let active: HashSet<String> = ["active/repo".to_string()].into_iter().collect();
4190
4191        let review_id = store
4192            .create_internal("removed/repo", "Needs review", "", "cron", "1", None)
4193            .await
4194            .unwrap();
4195        store
4196            .update_status(review_id, TaskStatus::NeedsReview)
4197            .await
4198            .unwrap();
4199        let blocked_id = store
4200            .create_internal("removed/repo", "Rebroadcast blocked", "", "cron", "2", None)
4201            .await
4202            .unwrap();
4203        store
4204            .update_status(blocked_id, TaskStatus::Blocked)
4205            .await
4206            .unwrap();
4207        store
4208            .set_block_reason(blocked_id, Some(REBROADCAST_BLOCK_REASON))
4209            .await
4210            .unwrap();
4211        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z'")
4212            .execute(store.pool())
4213            .await
4214            .unwrap();
4215
4216        refire_and_escalate_stale_needs_review_global(
4217            &task_manager,
4218            &store,
4219            &router,
4220            &dispatching,
4221            &active,
4222        )
4223        .await
4224        .unwrap();
4225        auto_recover_rebroadcast_blocked_tasks_global(&router, &task_manager, &store, &active)
4226            .await
4227            .unwrap();
4228
4229        let review = store.get(review_id).await.unwrap();
4230        assert_eq!(review.status, TaskStatus::NeedsReview);
4231        assert_eq!(review.needs_review_refires, 0);
4232        let blocked = store.get(blocked_id).await.unwrap();
4233        assert_eq!(blocked.status, TaskStatus::Blocked);
4234        assert_eq!(
4235            blocked.block_reason.as_deref(),
4236            Some(REBROADCAST_BLOCK_REASON)
4237        );
4238    }
4239
4240    // ── refire_and_escalate_stale_needs_review_global ──────────────────────
4241
4242    fn router_with_healthy_review_agent() -> Router {
4243        let mut config = RouterConfig::default();
4244        config
4245            .model_map
4246            .entry("review".to_string())
4247            .or_default()
4248            .insert("claude".to_string(), vec!["sonnet".to_string()]);
4249        let mut router = Router::new(config);
4250        router.available_agents = vec!["claude".to_string()];
4251        router
4252    }
4253
4254    /// Regression test for #3469: a task stuck in NeedsReview for a repo that
4255    /// is not the TaskManager's own repo must still be refired. The sweep
4256    /// queries via `list_all_by_status_global` and writes through the store
4257    /// directly, bypassing the repo-scoped TaskManager
4258    #[serial(cooldown_state)]
4259    #[tokio::test]
4260    async fn global_refire_sweep_refires_stale_task_from_inactive_repo() {
4261        cooldown::reset_global_state().await;
4262        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4263        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4264        // TaskManager scoped to the active repo — NOT the repo the task belongs to.
4265        let task_manager = Arc::new(TaskManager::with_store(
4266            backend,
4267            Arc::clone(&store),
4268            "active/repo".to_string(),
4269        ));
4270        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4271        let router = router_with_healthy_review_agent();
4272
4273        let id = store
4274            .upsert_external(&crate::store::UpsertExternal {
4275                repo: "inactive/repo",
4276                ext_id: "490",
4277                title: "Stale needs_review from inactive repo",
4278                body: "",
4279                author: "",
4280                url: "",
4281                labels: &[],
4282                origin: "github",
4283            })
4284            .await
4285            .unwrap();
4286        store
4287            .update_status(id, TaskStatus::NeedsReview)
4288            .await
4289            .unwrap();
4290        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4291            .bind(id)
4292            .execute(store.pool())
4293            .await
4294            .unwrap();
4295
4296        refire_and_escalate_stale_needs_review_global(
4297            &task_manager,
4298            &store,
4299            &router,
4300            &dispatching,
4301            &sweep_repos(),
4302        )
4303        .await
4304        .unwrap();
4305
4306        let task = store.get(id).await.unwrap();
4307        assert_eq!(
4308            task.needs_review_refires, 1,
4309            "cross-repo task must be refired"
4310        );
4311        assert_eq!(task.status, TaskStatus::NeedsReview);
4312    }
4313
4314    /// After `MAX_NEEDS_REVIEW_REFIRE_ATTEMPTS` refires, a cross-repo task must
4315    /// escalate to Blocked
4316    #[serial(cooldown_state)]
4317    #[tokio::test]
4318    async fn global_refire_sweep_escalates_cross_repo_task_to_blocked() {
4319        cooldown::reset_global_state().await;
4320        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4321        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4322        let task_manager = Arc::new(TaskManager::with_store(
4323            backend,
4324            Arc::clone(&store),
4325            "active/repo".to_string(),
4326        ));
4327        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4328        let router = router_with_healthy_review_agent();
4329
4330        let id = store
4331            .upsert_external(&crate::store::UpsertExternal {
4332                repo: "inactive/repo",
4333                ext_id: "493",
4334                title: "Exhausted refires from inactive repo",
4335                body: "",
4336                author: "",
4337                url: "",
4338                labels: &[],
4339                origin: "github",
4340            })
4341            .await
4342            .unwrap();
4343        store
4344            .update_status(id, TaskStatus::NeedsReview)
4345            .await
4346            .unwrap();
4347        store
4348            .set_fields(id, &[("needs_review_refires", serde_json::json!(5))])
4349            .await
4350            .unwrap();
4351        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4352            .bind(id)
4353            .execute(store.pool())
4354            .await
4355            .unwrap();
4356
4357        refire_and_escalate_stale_needs_review_global(
4358            &task_manager,
4359            &store,
4360            &router,
4361            &dispatching,
4362            &sweep_repos(),
4363        )
4364        .await
4365        .unwrap();
4366
4367        let task = store.get(id).await.unwrap();
4368        assert_eq!(task.status, TaskStatus::Blocked);
4369        assert_eq!(
4370            task.block_reason.as_deref(),
4371            Some("review agent rebroadcast escalated after repeated retries")
4372        );
4373    }
4374
4375    /// When every review agent is cooled, the sweep must skip entirely.
4376    /// Re-firing while all agents are cooled would waste the refire counter and
4377    /// could escalate a task to Blocked because agents happened to be
4378    /// temporarily unavailable
4379    #[serial(cooldown_state)]
4380    #[tokio::test]
4381    async fn global_refire_sweep_skips_when_all_review_agents_cooled() {
4382        cooldown::reset_global_state().await;
4383        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4384        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4385        let task_manager = Arc::new(TaskManager::with_store(
4386            backend,
4387            Arc::clone(&store),
4388            "active/repo".to_string(),
4389        ));
4390        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4391        // RouterConfig::default() has no "review" model map entries, so the one
4392        // available agent has no healthy model for "review" — all cooled.
4393        let router = Router::new_for_test(RouterConfig::default(), vec!["claude".to_string()]);
4394
4395        let id = store
4396            .upsert_external(&crate::store::UpsertExternal {
4397                repo: "inactive/repo",
4398                ext_id: "494",
4399                title: "Stale needs_review while agents cooled",
4400                body: "",
4401                author: "",
4402                url: "",
4403                labels: &[],
4404                origin: "github",
4405            })
4406            .await
4407            .unwrap();
4408        store
4409            .update_status(id, TaskStatus::NeedsReview)
4410            .await
4411            .unwrap();
4412        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4413            .bind(id)
4414            .execute(store.pool())
4415            .await
4416            .unwrap();
4417
4418        refire_and_escalate_stale_needs_review_global(
4419            &task_manager,
4420            &store,
4421            &router,
4422            &dispatching,
4423            &sweep_repos(),
4424        )
4425        .await
4426        .unwrap();
4427
4428        let task = store.get(id).await.unwrap();
4429        assert_eq!(
4430            task.needs_review_refires, 0,
4431            "counter must not be touched while cooled"
4432        );
4433        assert_eq!(task.status, TaskStatus::NeedsReview);
4434    }
4435
4436    /// With no available review agent at all, the sweep must not count re-fires
4437    /// or escalate to Blocked.
4438    #[serial(cooldown_state)]
4439    #[tokio::test]
4440    async fn global_refire_sweep_skips_when_no_reviewer_available() {
4441        cooldown::reset_global_state().await;
4442        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4443        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4444        let task_manager = Arc::new(TaskManager::with_store(
4445            backend,
4446            Arc::clone(&store),
4447            "active/repo".to_string(),
4448        ));
4449        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4450        let router = Router::new_for_test(RouterConfig::default(), vec![]);
4451
4452        let id = store
4453            .upsert_external(&crate::store::UpsertExternal {
4454                repo: "some/repo",
4455                ext_id: "495",
4456                title: "needs_review with no reviewer",
4457                body: "",
4458                author: "",
4459                url: "",
4460                labels: &[],
4461                origin: "github",
4462            })
4463            .await
4464            .unwrap();
4465        store
4466            .update_status(id, TaskStatus::NeedsReview)
4467            .await
4468            .unwrap();
4469        store
4470            .set_fields(id, &[("needs_review_refires", serde_json::json!(5))])
4471            .await
4472            .unwrap();
4473        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4474            .bind(id)
4475            .execute(store.pool())
4476            .await
4477            .unwrap();
4478
4479        refire_and_escalate_stale_needs_review_global(
4480            &task_manager,
4481            &store,
4482            &router,
4483            &dispatching,
4484            &sweep_repos(),
4485        )
4486        .await
4487        .unwrap();
4488
4489        let task = store.get(id).await.unwrap();
4490        assert_eq!(task.status, TaskStatus::NeedsReview);
4491        assert_eq!(task.needs_review_refires, 5);
4492    }
4493
4494    /// Regression test for #3499: a task blocked with the rebroadcast-escalation
4495    /// reason after surviving one review round (`review_cycles > 0`) must be
4496    /// recovered back to NeedsReview once review agents are routable again.
4497    /// Previously the `review_cycles != 0` filter excluded these tasks, stranding
4498    /// them permanently even when agents recovered.
4499    #[serial(cooldown_state)]
4500    #[tokio::test]
4501    async fn auto_recover_rebroadcast_blocked_task_after_review_cycle() {
4502        cooldown::reset_global_state().await;
4503        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4504        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4505        let task_manager = Arc::new(TaskManager::with_store(
4506            backend,
4507            Arc::clone(&store),
4508            "owner/repo".to_string(),
4509        ));
4510        let router = router_with_healthy_review_agent();
4511
4512        let id = store
4513            .upsert_external(&crate::store::UpsertExternal {
4514                repo: "owner/repo",
4515                ext_id: "511",
4516                title: "Rebroadcast blocked after review",
4517                body: "",
4518                author: "",
4519                url: "",
4520                labels: &[],
4521                origin: "github",
4522            })
4523            .await
4524            .unwrap();
4525        store.update_status(id, TaskStatus::Blocked).await.unwrap();
4526        store
4527            .set_block_reason(
4528                id,
4529                Some("review agent rebroadcast escalated after repeated retries"),
4530            )
4531            .await
4532            .unwrap();
4533        store
4534            .set_fields(id, &[("review_cycles", serde_json::json!(1))])
4535            .await
4536            .unwrap();
4537        store
4538            .set_fields(id, &[("needs_review_refires", serde_json::json!(6))])
4539            .await
4540            .unwrap();
4541        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4542            .bind(id)
4543            .execute(store.pool())
4544            .await
4545            .unwrap();
4546
4547        auto_recover_rebroadcast_blocked_tasks_global(
4548            &router,
4549            &task_manager,
4550            &store,
4551            &sweep_repos(),
4552        )
4553        .await
4554        .unwrap();
4555
4556        let task = store.get(id).await.unwrap();
4557        assert_eq!(
4558            task.status,
4559            TaskStatus::NeedsReview,
4560            "blocked rebroadcast task must be recovered to needs_review"
4561        );
4562        assert!(
4563            task.block_reason.is_none(),
4564            "block_reason must be cleared on recovery"
4565        );
4566        assert_eq!(
4567            task.needs_review_refires, 0,
4568            "needs_review_refires must be reset to 0 on recovery"
4569        );
4570    }
4571
4572    /// A fresh InReview task (<1 min old) should NOT be reset — the review
4573    /// agent may still be starting up.
4574    #[serial(cooldown_state)]
4575    #[tokio::test]
4576    async fn stale_in_review_recovery_skips_fresh_tasks() {
4577        cooldown::reset_global_state().await;
4578        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4579        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4580        let task_manager = Arc::new(TaskManager::with_store(
4581            Arc::clone(&backend),
4582            Arc::clone(&store),
4583            "owner/repo".to_string(),
4584        ));
4585        let tmux = Arc::new(TmuxManager::new());
4586        let router = Arc::new(RwLock::new(Router::new_for_test(
4587            RouterConfig::default(),
4588            vec!["claude".to_string()],
4589        )));
4590        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4591        let auto_merge_in_flight: Arc<DashSet<String>> = Arc::new(DashSet::new());
4592
4593        let id = store
4594            .upsert_external(&crate::store::UpsertExternal {
4595                repo: "owner/repo",
4596                ext_id: "66",
4597                title: "Fresh review",
4598                body: "",
4599                author: "",
4600                url: "",
4601                labels: &[],
4602                origin: "github",
4603            })
4604            .await
4605            .unwrap();
4606        store.update_status(id, TaskStatus::InReview).await.unwrap();
4607        store
4608            .set_fields(id, &[("branch", serde_json::json!("feature/66"))])
4609            .await
4610            .unwrap();
4611        // Don't backdate — task is fresh (just created)
4612
4613        Box::pin(sync_tick(
4614            &backend,
4615            &tmux,
4616            "owner/repo",
4617            &EngineConfig::default(),
4618            &router,
4619            &task_manager,
4620            &store,
4621            &dispatching,
4622            &auto_merge_in_flight,
4623        ))
4624        .await
4625        .unwrap();
4626
4627        // Should stay InReview — too young to be considered orphaned
4628        let task = store.get(id).await.unwrap();
4629        assert_eq!(task.status, TaskStatus::InReview);
4630    }
4631
4632    #[serial(cooldown_state)]
4633    #[tokio::test]
4634    async fn auto_unblock_routes_recoverable_failure() {
4635        cooldown::reset_global_state().await;
4636        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4637        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4638        let task_manager = Arc::new(TaskManager::with_store(
4639            backend,
4640            Arc::clone(&store),
4641            "owner/repo".to_string(),
4642        ));
4643        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4644
4645        let id = store
4646            .upsert_external(&crate::store::UpsertExternal {
4647                repo: "owner/repo",
4648                ext_id: "77",
4649                title: "Blocked task",
4650                body: "",
4651                author: "",
4652                url: "",
4653                labels: &[],
4654                origin: "github",
4655            })
4656            .await
4657            .unwrap();
4658        store.update_status(id, TaskStatus::Blocked).await.unwrap();
4659        store
4660            .set_fields(
4661                id,
4662                &[
4663                    ("agent", serde_json::json!("codex")),
4664                    ("model", serde_json::json!("gpt-5")),
4665                ],
4666            )
4667            .await
4668            .unwrap();
4669
4670        let run_id = store
4671            .start_run(&crate::store::StartRun {
4672                task_id: id,
4673                attempt: 1,
4674                run_type: "agent",
4675                agent: "codex",
4676                model: "gpt-5",
4677                command: "codex --model gpt-5",
4678                prompt: "do thing",
4679            })
4680            .await
4681            .unwrap();
4682        store
4683            .complete_run(&crate::store::CompleteRun {
4684                run_id,
4685                exit_code: Some(1),
4686                stdout: "",
4687                stderr: "",
4688                parsed: "",
4689                outcome: "rate_limit",
4690                error: "rate limit exceeded",
4691                tokens: RunTokenUsage::default(),
4692            })
4693            .await
4694            .unwrap();
4695
4696        auto_unblock_blocked_tasks(
4697            "owner/repo",
4698            &task_manager,
4699            &store,
4700            &dispatching,
4701            &Arc::new(DashSet::new()),
4702        )
4703        .await
4704        .unwrap();
4705
4706        let task = store.get(id).await.unwrap();
4707        assert_eq!(task.status, TaskStatus::New);
4708        assert!(task.agent.is_none());
4709        assert!(task.model.is_none());
4710        assert_eq!(task.auto_unblock_count, 1);
4711        assert_ne!(task.auto_unblock_last_at, "");
4712        assert_eq!(task.auto_unblock_last_reason, "RateLimit");
4713    }
4714
4715    #[serial(cooldown_state)]
4716    #[tokio::test]
4717    async fn auto_unblock_skips_manual_block() {
4718        cooldown::reset_global_state().await;
4719        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4720        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4721        let task_manager = Arc::new(TaskManager::with_store(
4722            backend,
4723            Arc::clone(&store),
4724            "owner/repo".to_string(),
4725        ));
4726        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4727
4728        let id = store
4729            .upsert_external(&crate::store::UpsertExternal {
4730                repo: "owner/repo",
4731                ext_id: "88",
4732                title: "Manually blocked",
4733                body: "",
4734                author: "",
4735                url: "",
4736                labels: &[],
4737                origin: "github",
4738            })
4739            .await
4740            .unwrap();
4741        store.update_status(id, TaskStatus::Blocked).await.unwrap();
4742        store
4743            .set_block_reason(id, Some("waiting on input"))
4744            .await
4745            .unwrap();
4746
4747        let run_id = store
4748            .start_run(&crate::store::StartRun {
4749                task_id: id,
4750                attempt: 1,
4751                run_type: "agent",
4752                agent: "codex",
4753                model: "gpt-5",
4754                command: "codex --model gpt-5",
4755                prompt: "do thing",
4756            })
4757            .await
4758            .unwrap();
4759        store
4760            .complete_run(&crate::store::CompleteRun {
4761                run_id,
4762                exit_code: Some(1),
4763                stdout: "",
4764                stderr: "",
4765                parsed: "",
4766                outcome: "rate_limit",
4767                error: "rate limit exceeded",
4768                tokens: RunTokenUsage::default(),
4769            })
4770            .await
4771            .unwrap();
4772
4773        auto_unblock_blocked_tasks(
4774            "owner/repo",
4775            &task_manager,
4776            &store,
4777            &dispatching,
4778            &Arc::new(DashSet::new()),
4779        )
4780        .await
4781        .unwrap();
4782
4783        let task = store.get(id).await.unwrap();
4784        assert_eq!(task.status, TaskStatus::Blocked);
4785        assert_eq!(task.auto_unblock_count, 0);
4786    }
4787
4788    // ── FailureCategory::as_str ─────────────────────────────────────────
4789
4790    #[serial(cooldown_state)]
4791    #[test]
4792    fn failure_category_as_str() {
4793        // as_str() must return a stable, human-readable string used as the
4794        // reason key in auto_unblock tracking (stored in auto_unblock_last_reason).
4795        assert_eq!(FailureCategory::RateLimit.as_str(), "RateLimit");
4796        assert_eq!(FailureCategory::Timeout.as_str(), "Timeout");
4797        assert_eq!(
4798            FailureCategory::ModelUnavailable.as_str(),
4799            "ModelUnavailable"
4800        );
4801        assert_eq!(FailureCategory::MaxAttempts.as_str(), "MaxAttempts");
4802        assert_eq!(FailureCategory::Unknown.as_str(), "Unknown");
4803        assert_eq!(FailureCategory::SilentExit0.as_str(), "SilentExit0");
4804        assert_eq!(FailureCategory::ConnectionError.as_str(), "ConnectionError");
4805        assert_eq!(FailureCategory::CliFlagError.as_str(), "CliFlagError");
4806        assert_eq!(
4807            FailureCategory::AllAgentsExhausted.as_str(),
4808            "AllAgentsExhausted"
4809        );
4810        assert_eq!(FailureCategory::ParseError.as_str(), "ParseError");
4811        assert_eq!(FailureCategory::FalseFailure.as_str(), "FalseFailure");
4812        assert_eq!(FailureCategory::PushFailed.as_str(), "PushFailed");
4813        assert_eq!(FailureCategory::PrCreateFailed.as_str(), "PrCreateFailed");
4814    }
4815
4816    // ── classify_failure: ModelUnavailable ──────────────────────────────
4817
4818    #[serial(cooldown_state)]
4819    #[test]
4820    fn classify_failure_model_unavailable_phrase_is_recoverable() {
4821        let category = classify_failure(
4822            "model unavailable (anthropic/claude-sonnet-4-6): Model not found: anthropic/claude-sonnet-4-6",
4823            "error",
4824        );
4825        assert_eq!(
4826            category,
4827            FailureCategory::ModelUnavailable,
4828            "\"model unavailable\" error must classify as ModelUnavailable"
4829        );
4830        assert!(
4831            category.is_recoverable(),
4832            "ModelUnavailable must be recoverable so auto-unblock can re-route"
4833        );
4834    }
4835
4836    // ── classify_failure: CreditExhausted (billing cycle) ──────────────
4837    #[serial(cooldown_state)]
4838    #[test]
4839    fn classify_failure_kimi_billing_cycle_is_credit_exhausted_not_rate_limit() {
4840        // Regression test for issue #3529: Kimi's billing-cycle exhaustion message
4841        // contains "usage limit", which also matches the generic rate-limit content
4842        // check. Credit exhaustion must be checked first so the failure is
4843        // categorized as CreditExhausted, not RateLimit, even when the stored
4844        // outcome is still "rate_limit" (e.g. from a task_run recorded before this
4845        // fix landed).
4846        let error = "kimi rate limit: Failed to authenticate. API Error: 403 You've reached \
4847                      your usage limit for this billing cycle. Your quota will be refreshed soon.";
4848        let category = classify_failure(error, "rate_limit");
4849        assert_eq!(
4850            category,
4851            FailureCategory::CreditExhausted,
4852            "billing-cycle exhaustion must classify as CreditExhausted, not RateLimit"
4853        );
4854        assert!(category.is_recoverable());
4855    }
4856
4857    #[serial(cooldown_state)]
4858    #[test]
4859    fn classify_failure_billing_cycle_exhausted_outcome_is_credit_exhausted() {
4860        // New task_runs store the specific "billing_cycle_exhausted" outcome
4861        // (post-fix). classify_failure must recognize it directly too.
4862        let category = classify_failure("some billing message", "billing_cycle_exhausted");
4863        assert_eq!(category, FailureCategory::CreditExhausted);
4864    }
4865
4866    #[serial(cooldown_state)]
4867    #[test]
4868    fn classify_failure_generic_rate_limit_is_still_rate_limit() {
4869        // Genuine rate limits (no billing-cycle/credit-exhaustion markers) must
4870        // remain RateLimit, not be swallowed by the reordered credit-exhaustion check.
4871        let category = classify_failure("429 Too Many Requests", "rate_limit");
4872        assert_eq!(category, FailureCategory::RateLimit);
4873    }
4874
4875    // ── classify_failure: PrCreateFailed ──────────────────────────────
4876    #[serial(cooldown_state)]
4877    #[test]
4878    fn classify_failure_pr_create_failed_specific_patterns() {
4879        // These should be classified as PrCreateFailed
4880        assert_eq!(
4881            classify_failure("create pr failed", ""),
4882            FailureCategory::PrCreateFailed
4883        );
4884        assert_eq!(
4885            classify_failure("failed to create pull request", ""),
4886            FailureCategory::PrCreateFailed
4887        );
4888        assert_eq!(
4889            classify_failure("pull request creation failed", ""),
4890            FailureCategory::PrCreateFailed
4891        );
4892
4893        // These should NOT be classified as PrCreateFailed (they were before the fix)
4894        assert_ne!(
4895            classify_failure("network failed while fetching pull request status", ""),
4896            FailureCategory::PrCreateFailed
4897        );
4898        assert_ne!(
4899            classify_failure("connection failed, retrying pull request check", ""),
4900            FailureCategory::PrCreateFailed
4901        );
4902        assert_ne!(
4903            classify_failure("pull request created but failed to merge", ""),
4904            FailureCategory::PrCreateFailed
4905        );
4906    }
4907
4908    #[serial(cooldown_state)]
4909    #[test]
4910    fn classify_failure_model_not_found_is_recoverable() {
4911        let category = classify_failure("Model not found: gpt-5-ultra", "error");
4912        assert_eq!(
4913            category,
4914            FailureCategory::ModelUnavailable,
4915            "\"model not found\" error must classify as ModelUnavailable"
4916        );
4917        assert!(category.is_recoverable());
4918    }
4919
4920    #[serial(cooldown_state)]
4921    #[test]
4922    fn classify_failure_model_does_not_exist_is_recoverable() {
4923        let category = classify_failure("The model `claude-opus-99` does not exist", "error");
4924        assert_eq!(category, FailureCategory::ModelUnavailable);
4925        assert!(category.is_recoverable());
4926    }
4927
4928    #[serial(cooldown_state)]
4929    #[tokio::test]
4930    async fn auto_unblock_routes_task_blocked_by_model_unavailable() {
4931        cooldown::reset_global_state().await;
4932        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4933        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4934        let task_manager = Arc::new(TaskManager::with_store(
4935            backend,
4936            Arc::clone(&store),
4937            "owner/repo".to_string(),
4938        ));
4939        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4940
4941        let id = store
4942            .upsert_external(&crate::store::UpsertExternal {
4943                repo: "owner/repo",
4944                ext_id: "1202",
4945                title: "Task blocked by model unavailable",
4946                body: "",
4947                author: "",
4948                url: "",
4949                labels: &[],
4950                origin: "github",
4951            })
4952            .await
4953            .unwrap();
4954        store.update_status(id, TaskStatus::Blocked).await.unwrap();
4955
4956        let run_id = store
4957            .start_run(&crate::store::StartRun {
4958                task_id: id,
4959                attempt: 1,
4960                run_type: "agent",
4961                agent: "opencode",
4962                model: "anthropic/claude-sonnet-4-6",
4963                command: "opencode",
4964                prompt: "fix the bug",
4965            })
4966            .await
4967            .unwrap();
4968        store
4969            .complete_run(&crate::store::CompleteRun {
4970                run_id,
4971                exit_code: Some(1),
4972                stdout: "",
4973                stderr: "",
4974                parsed: "",
4975                outcome: "error",
4976                error: "model unavailable (anthropic/claude-sonnet-4-6): Model not found: anthropic/claude-sonnet-4-6",
4977                tokens: RunTokenUsage::default(),
4978            })
4979            .await
4980            .unwrap();
4981
4982        auto_unblock_blocked_tasks(
4983            "owner/repo",
4984            &task_manager,
4985            &store,
4986            &dispatching,
4987            &Arc::new(DashSet::new()),
4988        )
4989        .await
4990        .unwrap();
4991
4992        let task = store.get(id).await.unwrap();
4993        assert_eq!(
4994            task.status,
4995            TaskStatus::New,
4996            "task blocked by model unavailable must be auto-unblocked and re-routed"
4997        );
4998        assert_eq!(task.auto_unblock_count, 1);
4999        assert_eq!(task.auto_unblock_last_reason, "ModelUnavailable");
5000    }
5001
5002    // ── auto_unblock: reason-reset (regression for #1227) ─────────────────
5003
5004    #[serial(cooldown_state)]
5005    #[tokio::test]
5006    async fn auto_unblock_resets_count_when_reason_changes() {
5007        cooldown::reset_global_state().await;
5008        // Regression test for #1227: when a task is blocked for a different failure
5009        // reason than the last auto-unblock, the counter should reset to 0 (immediate
5010        // retry) instead of accumulating across unrelated failures.
5011        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5012        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
5013        let task_manager = Arc::new(TaskManager::with_store(
5014            backend,
5015            Arc::clone(&store),
5016            "owner/repo".to_string(),
5017        ));
5018        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
5019
5020        let id = store
5021            .upsert_external(&crate::store::UpsertExternal {
5022                repo: "owner/repo",
5023                ext_id: "1227",
5024                title: "Task with different failure reason",
5025                body: "",
5026                author: "",
5027                url: "",
5028                labels: &[],
5029                origin: "github",
5030            })
5031            .await
5032            .unwrap();
5033        store.update_status(id, TaskStatus::Blocked).await.unwrap();
5034        // Simulate: task was previously auto-unblocked for ModelUnavailable (count=1)
5035        store
5036            .set_fields(
5037                id,
5038                &[
5039                    ("auto_unblock_count", serde_json::json!(1)),
5040                    (
5041                        "auto_unblock_last_reason",
5042                        serde_json::json!("ModelUnavailable"),
5043                    ),
5044                ],
5045            )
5046            .await
5047            .unwrap();
5048
5049        // Add a run with a DIFFERENT failure: RateLimit
5050        let run_id = store
5051            .start_run(&crate::store::StartRun {
5052                task_id: id,
5053                attempt: 2,
5054                run_type: "agent",
5055                agent: "claude",
5056                model: "opus",
5057                command: "claude",
5058                prompt: "fix the bug",
5059            })
5060            .await
5061            .unwrap();
5062        store
5063            .complete_run(&crate::store::CompleteRun {
5064                run_id,
5065                exit_code: Some(1),
5066                stdout: "",
5067                stderr: "",
5068                parsed: "",
5069                outcome: "error",
5070                error: "rate limit reached for model opus",
5071                tokens: RunTokenUsage::default(),
5072            })
5073            .await
5074            .unwrap();
5075
5076        auto_unblock_blocked_tasks(
5077            "owner/repo",
5078            &task_manager,
5079            &store,
5080            &dispatching,
5081            &Arc::new(DashSet::new()),
5082        )
5083        .await
5084        .unwrap();
5085
5086        let task = store.get(id).await.unwrap();
5087        assert_eq!(
5088            task.status,
5089            TaskStatus::New,
5090            "task must be auto-unblocked even with count=1 from a different reason"
5091        );
5092        // Count is reset to 0, then incremented to 1 for the new reason
5093        assert_eq!(task.auto_unblock_count, 1);
5094        assert_eq!(
5095            task.auto_unblock_last_reason, "RateLimit",
5096            "reason must be updated to the new failure category"
5097        );
5098    }
5099
5100    /**
5101     * A task blocked for "exceeded max attempts" has already run the configured number
5102     * of times. Auto-unblock must leave it alone — flipping it back to new/needs_review
5103     * on the next tick restarted the very loop that produced the block (#3783)
5104     **/
5105    #[serial(cooldown_state)]
5106    #[tokio::test]
5107    async fn auto_unblock_leaves_max_attempts_blocked_task_alone() {
5108        cooldown::reset_global_state().await;
5109        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5110        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
5111        let task_manager = Arc::new(TaskManager::with_store(
5112            backend,
5113            Arc::clone(&store),
5114            "owner/repo".to_string(),
5115        ));
5116        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
5117
5118        let id = store
5119            .upsert_external(&crate::store::UpsertExternal {
5120                repo: "owner/repo",
5121                ext_id: "3783",
5122                title: "Approved work sent back to the agent",
5123                body: "",
5124                author: "",
5125                url: "",
5126                labels: &[],
5127                origin: "github",
5128            })
5129            .await
5130            .unwrap();
5131        store.update_status(id, TaskStatus::Blocked).await.unwrap();
5132
5133        let run_id = store
5134            .start_run(&crate::store::StartRun {
5135                task_id: id,
5136                attempt: 11,
5137                run_type: "agent",
5138                agent: "claude",
5139                model: "opus",
5140                command: "claude",
5141                prompt: "fix the bug",
5142            })
5143            .await
5144            .unwrap();
5145        store
5146            .complete_run(&crate::store::CompleteRun {
5147                run_id,
5148                exit_code: Some(1),
5149                stdout: "",
5150                stderr: "",
5151                parsed: "",
5152                outcome: "error",
5153                error: "exceeded max attempts (11/10). Use `/retry` to reset.",
5154                tokens: RunTokenUsage::default(),
5155            })
5156            .await
5157            .unwrap();
5158
5159        auto_unblock_blocked_tasks(
5160            "owner/repo",
5161            &task_manager,
5162            &store,
5163            &dispatching,
5164            &Arc::new(DashSet::new()),
5165        )
5166        .await
5167        .unwrap();
5168
5169        let task = store.get(id).await.unwrap();
5170        assert_eq!(
5171            task.status,
5172            TaskStatus::Blocked,
5173            "max-attempts block must survive the next tick until a person acts"
5174        );
5175        assert_eq!(
5176            task.auto_unblock_count, 0,
5177            "no auto-unblock attempt should be recorded"
5178        );
5179    }
5180
5181    #[serial(cooldown_state)]
5182    #[tokio::test]
5183    async fn auto_unblock_stores_reason_on_first_unblock() {
5184        cooldown::reset_global_state().await;
5185        // When a task is auto-unblocked for the first time, the failure reason
5186        // should be stored so subsequent blocks can be compared.
5187        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5188        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
5189        let task_manager = Arc::new(TaskManager::with_store(
5190            backend,
5191            Arc::clone(&store),
5192            "owner/repo".to_string(),
5193        ));
5194        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
5195
5196        let id = store
5197            .upsert_external(&crate::store::UpsertExternal {
5198                repo: "owner/repo",
5199                ext_id: "1227b",
5200                title: "Task for first-time reason storage",
5201                body: "",
5202                author: "",
5203                url: "",
5204                labels: &[],
5205                origin: "github",
5206            })
5207            .await
5208            .unwrap();
5209        store.update_status(id, TaskStatus::Blocked).await.unwrap();
5210
5211        let run_id = store
5212            .start_run(&crate::store::StartRun {
5213                task_id: id,
5214                attempt: 1,
5215                run_type: "agent",
5216                agent: "claude",
5217                model: "opus",
5218                command: "claude",
5219                prompt: "fix the bug",
5220            })
5221            .await
5222            .unwrap();
5223        store
5224            .complete_run(&crate::store::CompleteRun {
5225                run_id,
5226                exit_code: Some(1),
5227                stdout: "",
5228                stderr: "",
5229                parsed: "",
5230                outcome: "timeout",
5231                error: "task timed out",
5232                tokens: RunTokenUsage::default(),
5233            })
5234            .await
5235            .unwrap();
5236
5237        auto_unblock_blocked_tasks(
5238            "owner/repo",
5239            &task_manager,
5240            &store,
5241            &dispatching,
5242            &Arc::new(DashSet::new()),
5243        )
5244        .await
5245        .unwrap();
5246
5247        let task = store.get(id).await.unwrap();
5248        assert_eq!(task.status, TaskStatus::New);
5249        assert_eq!(task.auto_unblock_count, 1);
5250        assert_eq!(
5251            task.auto_unblock_last_reason, "Timeout",
5252            "reason must be stored on first auto-unblock"
5253        );
5254    }
5255
5256    // ── classify_failure: sparse checkout (regression for substring false positive) ──
5257
5258    #[serial(cooldown_state)]
5259    #[test]
5260    fn classify_failure_sparse_checkout_not_parse_error() {
5261        // Regression test for #1204: "sparse" substring contains "parse",
5262        // so overly broad substring match would misclassify git sparse-checkout errors
5263        let category = classify_failure("error: cannot reset paths in a sparse checkout", "error");
5264        assert_ne!(
5265            category,
5266            FailureCategory::ParseError,
5267            "sparse checkout errors must NOT be classified as ParseError"
5268        );
5269        assert_eq!(
5270            category,
5271            FailureCategory::Unknown,
5272            "sparse checkout errors should be Unknown (not recoverable auto-unblock)"
5273        );
5274    }
5275
5276    #[serial(cooldown_state)]
5277    #[test]
5278    fn classify_failure_sparse_index_not_parse_error() {
5279        // Another sparse-checkout variant that contains "parse"
5280        let category = classify_failure("failed due to sparse index configuration", "error");
5281        assert_ne!(category, FailureCategory::ParseError);
5282        assert_eq!(category, FailureCategory::Unknown);
5283    }
5284
5285    // ── classify_failure: truncated (issue #3473) ───────────────────────
5286
5287    /// A review output truncated by the model's token budget (opencode
5288    /// step_finish reason=length) must classify as recoverable, so a task
5289    /// blocked after repeated truncations can be auto-unblocked instead of
5290    /// requiring manual intervention.
5291    #[serial(cooldown_state)]
5292    #[test]
5293    fn classify_failure_truncated_is_recoverable() {
5294        let category = classify_failure(
5295            "truncated: review output was cut off before completion \
5296             (output/reasoning token budget exceeded)",
5297            "truncated",
5298        );
5299        assert_eq!(
5300            category,
5301            FailureCategory::ParseError,
5302            "truncated outcome must classify as a recoverable parse-style failure"
5303        );
5304        assert!(
5305            category.is_recoverable(),
5306            "truncated failures must be recoverable so auto-unblock can retry"
5307        );
5308    }
5309
5310    #[serial(cooldown_state)]
5311    #[test]
5312    fn dispatching_key_includes_repo_for_cross_project_isolation() {
5313        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
5314
5315        // Lock task 42 in repo A
5316        let key_a = "owner/repo-a/42".to_string();
5317        dispatching.insert(key_a.clone(), "42".to_string());
5318
5319        // Same task ID in repo B should NOT be blocked
5320        let key_b = "owner/repo-b/42".to_string();
5321        assert!(dispatching.contains_key(&key_a));
5322        assert!(
5323            !dispatching.contains_key(&key_b),
5324            "same task ID in different repo should not be blocked"
5325        );
5326    }
5327
5328    #[serial(cooldown_state)]
5329    #[tokio::test]
5330    async fn emit_degraded_agents_writes_metric_when_three_or_more_agents_degraded() {
5331        cooldown::reset_global_state().await;
5332
5333        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5334
5335        // Create a router and set deterministic available agents
5336        let mut config = RouterConfig::default();
5337        config
5338            .model_map
5339            .entry("medium".to_string())
5340            .or_default()
5341            .insert(
5342                "test-agent-3a".to_string(),
5343                vec!["test-model-a".to_string()],
5344            );
5345        config
5346            .model_map
5347            .entry("medium".to_string())
5348            .or_default()
5349            .insert(
5350                "test-agent-3b".to_string(),
5351                vec!["test-model-b".to_string()],
5352            );
5353        config
5354            .model_map
5355            .entry("medium".to_string())
5356            .or_default()
5357            .insert(
5358                "test-agent-3c".to_string(),
5359                vec!["test-model-c".to_string()],
5360            );
5361        config
5362            .model_map
5363            .entry("medium".to_string())
5364            .or_default()
5365            .insert(
5366                "test-agent-3d".to_string(),
5367                vec!["test-model-d".to_string()],
5368            );
5369        let mut router = Router::new(config);
5370        router.available_agents = vec![
5371            "test-agent-3a".to_string(),
5372            "test-agent-3b".to_string(),
5373            "test-agent-3c".to_string(),
5374            "test-agent-3d".to_string(),
5375        ];
5376
5377        // Place three agents into cooldown
5378        set_agent_cooldown("test-agent-3a", 3600).await;
5379        set_agent_cooldown("test-agent-3b", 3600).await;
5380        set_agent_cooldown("test-agent-3c", 3600).await;
5381
5382        // Call the helper and assert KV metrics written
5383        emit_degraded_agents_if_needed(&router.available_agents, &router.config, Some(&store))
5384            .await;
5385
5386        // Count metric always written
5387        let count_val = store
5388            .kv_get("metrics:orch.agents_degraded.count")
5389            .await
5390            .unwrap();
5391        assert_eq!(count_val.as_deref(), Some("3"));
5392
5393        // Alert metric set to "1" when threshold crossed
5394        let alert_val = store
5395            .kv_get("metrics:orch.agents_degraded.alert")
5396            .await
5397            .unwrap();
5398        assert_eq!(alert_val.as_deref(), Some("1"));
5399    }
5400
5401    #[serial(cooldown_state)]
5402    #[tokio::test]
5403    async fn emit_degraded_agents_clears_alert_when_below_threshold() {
5404        cooldown::reset_global_state().await;
5405
5406        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5407
5408        let mut config = RouterConfig::default();
5409        config
5410            .model_map
5411            .entry("medium".to_string())
5412            .or_default()
5413            .insert(
5414                "test-agent-2a".to_string(),
5415                vec!["test-model-a".to_string()],
5416            );
5417        config
5418            .model_map
5419            .entry("medium".to_string())
5420            .or_default()
5421            .insert(
5422                "test-agent-2b".to_string(),
5423                vec!["test-model-b".to_string()],
5424            );
5425        let mut router = Router::new(config);
5426        router.available_agents = vec!["test-agent-2a".to_string(), "test-agent-2b".to_string()];
5427
5428        // Place only two agents into cooldown — below the threshold of 3
5429        set_agent_cooldown("test-agent-2a", 3600).await;
5430        set_agent_cooldown("test-agent-2b", 3600).await;
5431
5432        emit_degraded_agents_if_needed(&router.available_agents, &router.config, Some(&store))
5433            .await;
5434
5435        // Count metric still written (always-emit)
5436        let count_val = store
5437            .kv_get("metrics:orch.agents_degraded.count")
5438            .await
5439            .unwrap();
5440        assert_eq!(count_val.as_deref(), Some("2"));
5441
5442        // Alert metric cleared (set to "0") — no WARN should fire
5443        let alert_val = store
5444            .kv_get("metrics:orch.agents_degraded.alert")
5445            .await
5446            .unwrap();
5447        assert_eq!(alert_val.as_deref(), Some("0"));
5448
5449        let stale_pool_alert = store
5450            .kv_get("metrics:orch.model_pool_stale_persistent.alert")
5451            .await
5452            .unwrap();
5453        assert_eq!(stale_pool_alert.as_deref(), Some("0"));
5454    }
5455
5456    #[serial(cooldown_state)]
5457    #[tokio::test]
5458    async fn emit_degraded_agents_sets_stale_pool_alert_for_persistent_model_failures() {
5459        cooldown::reset_global_state().await;
5460
5461        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5462
5463        let mut config = RouterConfig::default();
5464        config
5465            .model_map
5466            .entry("medium".to_string())
5467            .or_default()
5468            .insert(
5469                "test-agent-stale".to_string(),
5470                vec!["model-a".to_string(), "model-b".to_string()],
5471            );
5472        let mut router = Router::new(config);
5473        router.available_agents = vec!["test-agent-stale".to_string()];
5474
5475        // 50% cooled + persistent reason should trigger stale-pool alert.
5476        record_persistent_model_failure("test-agent-stale", "model-a").await;
5477
5478        emit_degraded_agents_if_needed(&router.available_agents, &router.config, Some(&store))
5479            .await;
5480
5481        let stale_pool_alert = store
5482            .kv_get("metrics:orch.model_pool_stale_persistent.alert")
5483            .await
5484            .unwrap();
5485        assert_eq!(stale_pool_alert.as_deref(), Some("1"));
5486    }
5487
5488    // ── classify_comment tests ──────────────────────────────────────────
5489
5490    #[serial(cooldown_state)]
5491    #[test]
5492    fn classify_skip_already_processed() {
5493        let action = classify_comment(
5494            "@orch /retry",
5495            Some("https://api.github.com/repos/o/r/issues/1"),
5496            "@bot",
5497            true,
5498        );
5499        assert_eq!(action, CommentAction::Skip);
5500    }
5501
5502    #[serial(cooldown_state)]
5503    #[test]
5504    fn classify_skip_irrelevant_comment() {
5505        let action = classify_comment("just a regular comment", None, "@bot", false);
5506        assert_eq!(action, CommentAction::Skip);
5507    }
5508
5509    #[serial(cooldown_state)]
5510    #[test]
5511    fn classify_command_without_mention() {
5512        let action = classify_comment(
5513            "/retry",
5514            Some("https://api.github.com/repos/o/r/issues/42"),
5515            "@bot",
5516            false,
5517        );
5518        assert_eq!(
5519            action,
5520            CommentAction::ExecuteCommand {
5521                command: OwnerCommand::Retry,
5522                issue_num: "42".to_string(),
5523            }
5524        );
5525    }
5526
5527    #[serial(cooldown_state)]
5528    #[test]
5529    fn classify_command_with_mention() {
5530        let action = classify_comment(
5531            "@orch /close",
5532            Some("https://api.github.com/repos/o/r/issues/7"),
5533            "@bot",
5534            false,
5535        );
5536        assert_eq!(
5537            action,
5538            CommentAction::ExecuteCommandForMention {
5539                command: OwnerCommand::Close,
5540                issue_num: "7".to_string(),
5541            }
5542        );
5543    }
5544
5545    #[serial(cooldown_state)]
5546    #[test]
5547    fn classify_mention_without_command() {
5548        let action = classify_comment(
5549            "@orch can you look at this?",
5550            Some("https://api.github.com/repos/o/r/issues/99"),
5551            "@bot",
5552            false,
5553        );
5554        assert_eq!(
5555            action,
5556            CommentAction::CreateMentionTask {
5557                issue_num: Some("99".to_string()),
5558            }
5559        );
5560    }
5561
5562    #[serial(cooldown_state)]
5563    #[test]
5564    fn classify_mention_without_issue_url() {
5565        let action = classify_comment("@orch help me", None, "@bot", false);
5566        assert_eq!(action, CommentAction::CreateMentionTask { issue_num: None });
5567    }
5568
5569    #[serial(cooldown_state)]
5570    #[test]
5571    fn classify_command_no_mention_no_url_skips() {
5572        let action = classify_comment("/retry", None, "@bot", false);
5573        assert_eq!(action, CommentAction::Skip);
5574    }
5575
5576    #[serial(cooldown_state)]
5577    #[test]
5578    fn classify_mention_via_current_user() {
5579        let action = classify_comment(
5580            "@bot please help",
5581            Some("https://api.github.com/repos/o/r/issues/5"),
5582            "@bot",
5583            false,
5584        );
5585        assert_eq!(
5586            action,
5587            CommentAction::CreateMentionTask {
5588                issue_num: Some("5".to_string()),
5589            }
5590        );
5591    }
5592
5593    #[serial(cooldown_state)]
5594    #[test]
5595    fn classify_command_with_mention_no_url_creates_task() {
5596        // @mention + command but no issue URL → can't run command, create mention task
5597        let action = classify_comment("@orch /retry", None, "@bot", false);
5598        assert_eq!(action, CommentAction::CreateMentionTask { issue_num: None });
5599    }
5600
5601    #[serial(cooldown_state)]
5602    #[test]
5603    fn mention_cursor_stops_advancing_after_gap() {
5604        let mut cursor = MentionCursor::default();
5605        cursor.advance("2026-01-01T00:00:00Z");
5606        cursor.block_on_gap();
5607        cursor.advance("2026-01-02T00:00:00Z");
5608
5609        assert_eq!(
5610            cursor.into_last_safe_ts().as_deref(),
5611            Some("2026-01-01T00:00:00Z")
5612        );
5613    }
5614
5615    #[serial(cooldown_state)]
5616    #[test]
5617    fn mention_cursor_tracks_max_timestamp_until_gap() {
5618        let mut cursor = MentionCursor::default();
5619        cursor.advance("2026-01-01T00:00:00Z");
5620        cursor.advance("2026-01-01T00:05:00Z");
5621        cursor.block_on_gap();
5622        cursor.advance("2026-01-01T00:10:00Z");
5623
5624        assert_eq!(
5625            cursor.into_last_safe_ts().as_deref(),
5626            Some("2026-01-01T00:05:00Z")
5627        );
5628    }
5629
5630    // ── scan_comments: acknowledge_mention failure does not advance cursor ────
5631
5632    /// Backend that tracks acknowledge_mention calls and can be configured to fail them.
5633    struct AckTrackingBackend {
5634        ack_calls: Mutex<Vec<String>>,
5635        posted: Mutex<Vec<String>>,
5636        ack_failure: bool,
5637        inner: NoopBackend,
5638    }
5639
5640    impl AckTrackingBackend {
5641        fn new(ack_failure: bool) -> Arc<Self> {
5642            Arc::new(Self {
5643                ack_calls: Mutex::new(Vec::new()),
5644                posted: Mutex::new(Vec::new()),
5645                ack_failure,
5646                inner: NoopBackend,
5647            })
5648        }
5649
5650        fn fail_on_ack(&self) -> bool {
5651            self.ack_failure
5652        }
5653    }
5654
5655    #[async_trait]
5656    impl ExternalBackend for AckTrackingBackend {
5657        fn name(&self) -> &'static str {
5658            "ack-tracking"
5659        }
5660
5661        async fn acknowledge_mention(&self, comment_id: &str) -> anyhow::Result<()> {
5662            self.ack_calls.lock().await.push(comment_id.to_string());
5663            if self.fail_on_ack() {
5664                // Return a transient error to simulate network glitch.
5665                anyhow::bail!("network error: temporary failure posting reaction");
5666            }
5667            Ok(())
5668        }
5669
5670        // Override get_authenticated_user to ensure scan proceeds.
5671        async fn get_authenticated_user(&self) -> anyhow::Result<Option<String>> {
5672            Ok(Some("bot".into()))
5673        }
5674
5675        // Override get_mentions to return empty (we use prefetched comments).
5676        async fn get_mentions(&self, _since: &str) -> anyhow::Result<Vec<Mention>> {
5677            Ok(vec![])
5678        }
5679
5680        // Delegate everything else to NoopBackend.
5681        async fn create_task(&self, t: &str, b: &str, l: &[String]) -> anyhow::Result<ExternalId> {
5682            self.inner.create_task(t, b, l).await
5683        }
5684        async fn get_task(&self, id: &ExternalId) -> anyhow::Result<ExternalTask> {
5685            self.inner.get_task(id).await
5686        }
5687        async fn list_by_status(&self, s: Status) -> anyhow::Result<Vec<ExternalTask>> {
5688            self.inner.list_by_status(s).await
5689        }
5690        async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
5691            self.inner.list_routable().await
5692        }
5693        async fn post_comment(&self, id: &ExternalId, b: &str) -> anyhow::Result<()> {
5694            self.posted.lock().await.push(b.to_string());
5695            self.inner.post_comment(id, b).await
5696        }
5697        async fn set_labels(&self, id: &ExternalId, l: &[String]) -> anyhow::Result<()> {
5698            self.inner.set_labels(id, l).await
5699        }
5700        async fn remove_label(&self, id: &ExternalId, l: &str) -> anyhow::Result<()> {
5701            self.inner.remove_label(id, l).await
5702        }
5703        async fn get_sub_issues(&self, id: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
5704            self.inner.get_sub_issues(id).await
5705        }
5706        async fn create_sub_task(
5707            &self,
5708            p: &ExternalId,
5709            t: &str,
5710            b: &str,
5711            l: &[String],
5712        ) -> anyhow::Result<ExternalId> {
5713            self.inner.create_sub_task(p, t, b, l).await
5714        }
5715        async fn ensure_status_label(&self, l: &str) -> anyhow::Result<()> {
5716            self.inner.ensure_status_label(l).await
5717        }
5718        async fn has_open_issue_with_title(&self, t: &str, l: &str) -> anyhow::Result<bool> {
5719            self.inner.has_open_issue_with_title(t, l).await
5720        }
5721        async fn health_check(&self) -> anyhow::Result<()> {
5722            self.inner.health_check().await
5723        }
5724        async fn is_pr_merged(&self, b: &str) -> anyhow::Result<bool> {
5725            self.inner.is_pr_merged(b).await
5726        }
5727        async fn update_status(&self, id: &ExternalId, s: Status) -> anyhow::Result<()> {
5728            self.inner.update_status(id, s).await
5729        }
5730    }
5731
5732    /// When acknowledge_mention fails for CreateMentionTask, the cursor must NOT
5733    /// advance and no mention task should be created. The next tick will retry.
5734    #[serial(cooldown_state)]
5735    #[tokio::test]
5736    async fn create_mention_task_skips_on_ack_failure() {
5737        cooldown::reset_global_state().await;
5738        let backend: Arc<AckTrackingBackend> = AckTrackingBackend::new(true);
5739        let store: Arc<TaskStore> = Arc::new(TaskStore::open_memory().await.unwrap());
5740
5741        // Set an initial cursor so we can verify it wasn't advanced.
5742        let initial_ts = "2026-01-01T00:00:00Z";
5743        store
5744            .kv_set("mentions_last_checked", initial_ts)
5745            .await
5746            .unwrap();
5747
5748        let mention = Mention {
5749            id: "mention-abc".into(),
5750            body: "@bot can you help?".into(),
5751            author: "alice".into(),
5752            created_at: "2026-01-02T00:00:00Z".into(),
5753            issue_url: Some("https://api.github.com/repos/owner/repo/issues/42".into()),
5754        };
5755
5756        let backend_trait: Arc<dyn ExternalBackend> = Arc::<AckTrackingBackend>::clone(&backend);
5757        let task_manager = Arc::new(TaskManager::new(Arc::clone(&backend_trait)));
5758
5759        scan_comments(
5760            &backend_trait,
5761            Some(&store),
5762            "owner/repo",
5763            &task_manager,
5764            Some(&[mention]),
5765        )
5766        .await
5767        .unwrap();
5768
5769        // acknowledge_mention was called with the mention ID.
5770        let calls = backend.ack_calls.lock().await;
5771        assert_eq!(
5772            &**calls,
5773            &["mention-abc"],
5774            "acknowledge_mention must be called"
5775        );
5776
5777        // No mention task was created.
5778        let all = store.list_all("owner/repo").await.unwrap();
5779        assert!(
5780            all.is_empty(),
5781            "no task should be created when acknowledge_mention fails"
5782        );
5783
5784        // Cursor was NOT advanced.
5785        let cursor = store.kv_get("mentions_last_checked").await.unwrap();
5786        assert_eq!(
5787            cursor.as_deref(),
5788            Some(initial_ts),
5789            "cursor must not advance when acknowledge_mention fails"
5790        );
5791    }
5792
5793    /// When acknowledge_mention fails for ExecuteCommandForMention, handle_slash_command
5794    /// must NOT be called and no mention task should be created.
5795    #[serial(cooldown_state)]
5796    #[tokio::test]
5797    async fn execute_command_for_mention_skips_on_ack_failure() {
5798        cooldown::reset_global_state().await;
5799        let backend: Arc<AckTrackingBackend> = AckTrackingBackend::new(true);
5800        let store: Arc<TaskStore> = Arc::new(TaskStore::open_memory().await.unwrap());
5801
5802        let initial_ts = "2026-01-01T00:00:00Z";
5803        store
5804            .kv_set("mentions_last_checked", initial_ts)
5805            .await
5806            .unwrap();
5807
5808        // Mention with a slash command and issue URL → ExecuteCommandForMention.
5809        let mention = Mention {
5810            id: "mention-def".into(),
5811            body: "@bot /close".into(),
5812            author: "alice".into(),
5813            created_at: "2026-01-02T00:00:00Z".into(),
5814            issue_url: Some("https://api.github.com/repos/owner/repo/issues/42".into()),
5815        };
5816
5817        let backend_trait: Arc<dyn ExternalBackend> = Arc::<AckTrackingBackend>::clone(&backend);
5818        let task_manager = Arc::new(TaskManager::new(Arc::clone(&backend_trait)));
5819
5820        scan_comments(
5821            &backend_trait,
5822            Some(&store),
5823            "owner/repo",
5824            &task_manager,
5825            Some(&[mention]),
5826        )
5827        .await
5828        .unwrap();
5829
5830        // acknowledge_mention was called.
5831        let calls = backend.ack_calls.lock().await;
5832        assert_eq!(&**calls, &["mention-def"]);
5833
5834        // No mention task was created (handle_slash_command was not called).
5835        let all = store.list_all("owner/repo").await.unwrap();
5836        assert!(
5837            all.is_empty(),
5838            "no task should be created when acknowledge_mention fails for ExecuteCommandForMention"
5839        );
5840
5841        // Cursor was NOT advanced.
5842        let cursor = store.kv_get("mentions_last_checked").await.unwrap();
5843        assert_eq!(
5844            cursor.as_deref(),
5845            Some(initial_ts),
5846            "cursor must not advance when acknowledge_mention fails for ExecuteCommandForMention"
5847        );
5848    }
5849
5850    // ── slash commands retry after a transient GitHub error ──────────────────
5851
5852    /// GitHub stub whose `get_issue` fails once, then succeeds.
5853    #[derive(Default)]
5854    struct FlakyGh {
5855        get_issue_calls: AtomicUsize,
5856    }
5857
5858    #[async_trait]
5859    impl CommandGh for FlakyGh {
5860        async fn get_issue(&self, _repo: &str, number: &str) -> anyhow::Result<GitHubIssue> {
5861            if self.get_issue_calls.fetch_add(1, Ordering::SeqCst) == 0 {
5862                anyhow::bail!("network error: temporary failure fetching issue");
5863            }
5864            Ok(GitHubIssue {
5865                number: number.parse().unwrap_or(0),
5866                id: 1,
5867                title: "t".into(),
5868                body: None,
5869                state: "open".into(),
5870                labels: Vec::new(),
5871                user: GitHubUser {
5872                    login: "alice".into(),
5873                },
5874                created_at: String::new(),
5875                updated_at: String::new(),
5876                html_url: String::new(),
5877                node_id: None,
5878                pull_request: None,
5879                author_association: Some("OWNER".into()),
5880                state_reason: None,
5881            })
5882        }
5883
5884        async fn is_collaborator(&self, _repo: &str, _username: &str) -> anyhow::Result<bool> {
5885            Ok(true)
5886        }
5887
5888        async fn close_issue(&self, _repo: &str, _number: &str) -> anyhow::Result<()> {
5889            Ok(())
5890        }
5891    }
5892
5893    /// A transient `get_issue` failure must leave the cursor before the comment so
5894    /// the next scan retries it — and the command must then run exactly once.
5895    #[serial(cooldown_state)]
5896    #[tokio::test]
5897    async fn slash_command_retries_after_transient_fetch_failure() {
5898        cooldown::reset_global_state().await;
5899        let backend: Arc<AckTrackingBackend> = AckTrackingBackend::new(false);
5900        let store: Arc<TaskStore> = Arc::new(TaskStore::open_memory().await.unwrap());
5901        let initial_ts = "2026-01-01T00:00:00Z";
5902
5903        let mention = Mention {
5904            id: "mention-ghi".into(),
5905            body: "@bot /unblock".into(),
5906            author: "alice".into(),
5907            created_at: "2026-01-02T00:00:00Z".into(),
5908            issue_url: Some("https://api.github.com/repos/owner/repo/issues/42".into()),
5909        };
5910
5911        let backend_trait: Arc<dyn ExternalBackend> = Arc::<AckTrackingBackend>::clone(&backend);
5912        let task_manager = Arc::new(TaskManager::new(Arc::clone(&backend_trait)));
5913        let gh = FlakyGh::default();
5914        let mut cursor = MentionCursor::default();
5915
5916        // First scan: get_issue fails.
5917        let advanced_safely = handle_slash_command(
5918            &backend_trait,
5919            Some(&store),
5920            "owner/repo",
5921            &task_manager,
5922            &gh,
5923            &mention,
5924            &OwnerCommand::Unblock,
5925            &mut cursor,
5926        )
5927        .await
5928        .unwrap();
5929
5930        assert!(
5931            !advanced_safely,
5932            "a transient fetch failure must stop cursor advancement"
5933        );
5934        assert!(
5935            cursor.last_safe_ts.is_none(),
5936            "cursor must stay before the comment so the next tick retries it"
5937        );
5938        assert!(
5939            store.list_all("owner/repo").await.unwrap().is_empty(),
5940            "no mention task on a transient failure"
5941        );
5942        assert!(
5943            backend.posted.lock().await.is_empty(),
5944            "nothing is posted to GitHub on a transient failure"
5945        );
5946        assert_eq!(
5947            filter_mentions_by_since(slice::from_ref(&mention), initial_ts).len(),
5948            1,
5949            "the comment is still in range for the next scan"
5950        );
5951
5952        // Second scan: get_issue succeeds and the command runs.
5953        let mut cursor = MentionCursor::default();
5954        let advanced_safely = handle_slash_command(
5955            &backend_trait,
5956            Some(&store),
5957            "owner/repo",
5958            &task_manager,
5959            &gh,
5960            &mention,
5961            &OwnerCommand::Unblock,
5962            &mut cursor,
5963        )
5964        .await
5965        .unwrap();
5966
5967        assert!(advanced_safely);
5968        assert_eq!(
5969            cursor.into_last_safe_ts().as_deref(),
5970            Some("2026-01-02T00:00:00Z"),
5971            "cursor advances once the command actually runs"
5972        );
5973        assert_eq!(
5974            backend.posted.lock().await.len(),
5975            1,
5976            "the command runs once, posting a single confirmation"
5977        );
5978        assert_eq!(
5979            store.list_all("owner/repo").await.unwrap().len(),
5980            1,
5981            "one sentinel mention task for the executed command"
5982        );
5983    }
5984
5985    /// In the plain `ExecuteCommand` arm the GitHub client points at a dead port
5986    /// under `cfg(test)`, so `get_issue` always fails — the cursor must stay put.
5987    #[serial(cooldown_state)]
5988    #[tokio::test]
5989    async fn execute_command_keeps_cursor_on_fetch_failure() {
5990        cooldown::reset_global_state().await;
5991        let backend: Arc<AckTrackingBackend> = AckTrackingBackend::new(false);
5992        let store: Arc<TaskStore> = Arc::new(TaskStore::open_memory().await.unwrap());
5993
5994        let initial_ts = "2026-01-01T00:00:00Z";
5995        store
5996            .kv_set("mentions_last_checked", initial_ts)
5997            .await
5998            .unwrap();
5999
6000        // Slash command without a mention → ExecuteCommand.
6001        let comment = Mention {
6002            id: "comment-jkl".into(),
6003            body: "/unblock".into(),
6004            author: "alice".into(),
6005            created_at: "2026-01-02T00:00:00Z".into(),
6006            issue_url: Some("https://api.github.com/repos/owner/repo/issues/42".into()),
6007        };
6008
6009        let backend_trait: Arc<dyn ExternalBackend> = Arc::<AckTrackingBackend>::clone(&backend);
6010        let task_manager = Arc::new(TaskManager::new(Arc::clone(&backend_trait)));
6011
6012        scan_comments(
6013            &backend_trait,
6014            Some(&store),
6015            "owner/repo",
6016            &task_manager,
6017            Some(&[comment]),
6018        )
6019        .await
6020        .unwrap();
6021
6022        assert!(
6023            backend.posted.lock().await.is_empty(),
6024            "nothing is posted when the issue fetch fails"
6025        );
6026        assert_eq!(
6027            store
6028                .kv_get("mentions_last_checked")
6029                .await
6030                .unwrap()
6031                .as_deref(),
6032            Some(initial_ts),
6033            "cursor must not advance past a command that never ran"
6034        );
6035    }
6036
6037    // ── is_ci_failure_block ────────────────────────────────────────────────
6038
6039    fn make_task(block_reason: Option<String>) -> Task {
6040        Task {
6041            id: 1,
6042            external_id: None,
6043            repo: "owner/repo".to_string(),
6044            origin: String::new(),
6045            title: String::new(),
6046            body: String::new(),
6047            status: TaskStatus::Blocked,
6048            source: String::new(),
6049            source_id: String::new(),
6050            author: String::new(),
6051            url: String::new(),
6052            labels: vec![],
6053            agent: None,
6054            model: None,
6055            complexity: String::new(),
6056            estimate: 0,
6057            route_reason: String::new(),
6058            agent_profile: String::new(),
6059            selected_skills: String::new(),
6060            route_attempts: 0,
6061            attempts: 0,
6062            branch: String::new(),
6063            worktree: String::new(),
6064            worktree_cleaned: false,
6065            owner_node: None,
6066            summary: String::new(),
6067            last_error: String::new(),
6068            parent_id: None,
6069            block_reason,
6070            pr_number: Some(42),
6071            pr_review_context: String::new(),
6072            last_review_ts: String::new(),
6073            review_ts_map: String::new(),
6074            last_comment_review_ts: String::new(),
6075            merge_conflict_retries: 0,
6076            ci_merge_failures: 0,
6077            pr_create_failures: 0,
6078            push_failures: 0,
6079            network_retries: 0,
6080            timeout_retries: 0,
6081            review_agent_failures: 0,
6082            review_resets: 0,
6083            review_cycles: 0,
6084            review_invocations: 0,
6085            needs_review_refires: 0,
6086            input_tokens: 0,
6087            output_tokens: 0,
6088            total_cost_usd: 0.0,
6089            model_reroute_chain: String::new(),
6090            limit_reroute_chain: String::new(),
6091            memory: vec![],
6092            delegations: vec![],
6093            auto_unblock_count: 0,
6094            auto_unblock_last_at: String::new(),
6095            auto_unblock_last_reason: String::new(),
6096            ci_recovery_count: 0,
6097            no_code_reroutes: 0,
6098            no_code_last_agent: String::new(),
6099            created_at: String::new(),
6100            updated_at: String::new(),
6101        }
6102    }
6103
6104    #[serial(cooldown_state)]
6105    #[test]
6106    fn is_ci_failure_block_matches_ci_failure_limit() {
6107        let task = make_task(Some(
6108            "CI failure limit (3) reached during auto-merge".to_string(),
6109        ));
6110        assert!(is_ci_failure_block(&task));
6111    }
6112
6113    #[serial(cooldown_state)]
6114    #[test]
6115    fn is_ci_failure_block_matches_ci_checks_timeout() {
6116        let task = make_task(Some(
6117            "CI checks timed out after 3 auto-merge failures".to_string(),
6118        ));
6119        assert!(is_ci_failure_block(&task));
6120    }
6121
6122    #[serial(cooldown_state)]
6123    #[test]
6124    fn is_ci_failure_block_false_for_regular_block() {
6125        let task = make_task(Some("waiting on input".to_string()));
6126        assert!(!is_ci_failure_block(&task));
6127    }
6128
6129    #[serial(cooldown_state)]
6130    #[test]
6131    fn is_ci_failure_block_false_for_none_reason() {
6132        let task = make_task(None);
6133        assert!(!is_ci_failure_block(&task));
6134    }
6135
6136    // ── ci_failure_unblock_cooldown_elapsed ────────────────────────────────
6137
6138    #[serial(cooldown_state)]
6139    #[test]
6140    fn ci_failure_cooldown_zero_count_immediate() {
6141        // count=0 means never attempted — always allow immediate attempt
6142        let task = make_task(None);
6143        assert!(ci_failure_unblock_cooldown_elapsed(&task));
6144    }
6145
6146    #[serial(cooldown_state)]
6147    #[test]
6148    fn ci_failure_cooldown_empty_last_at_immediate() {
6149        // count=1 but empty last_at means it was never recorded — allow immediate
6150        let mut task = make_task(None);
6151        task.auto_unblock_count = 1;
6152        assert!(ci_failure_unblock_cooldown_elapsed(&task));
6153    }
6154
6155    #[serial(cooldown_state)]
6156    #[test]
6157    fn ci_failure_cooldown_invalid_timestamp_immediate() {
6158        // invalid timestamp should not block
6159        let mut task = make_task(None);
6160        task.auto_unblock_count = 1;
6161        task.auto_unblock_last_at = "not-a-timestamp".to_string();
6162        assert!(ci_failure_unblock_cooldown_elapsed(&task));
6163    }
6164
6165    #[serial(cooldown_state)]
6166    #[test]
6167    fn ci_failure_cooldown_high_count_still_eventually_elapses() {
6168        // count >= 3 must NOT be a permanent block — verify a sufficiently old
6169        // timestamp eventually elapses the cooldown.
6170        let long_ago = (chrono::Utc::now() - chrono::Duration::days(30)).to_rfc3339();
6171        for count in [3i32, 5, 99] {
6172            let mut task = make_task(None);
6173            task.auto_unblock_count = count;
6174            task.auto_unblock_last_at = long_ago.clone();
6175            assert!(
6176                ci_failure_unblock_cooldown_elapsed(&task),
6177                "count={count} should eventually elapse"
6178            );
6179        }
6180    }
6181
6182    #[serial(cooldown_state)]
6183    #[test]
6184    fn ci_failure_cooldown_recent_timestamp_still_blocks() {
6185        // A recent timestamp should still gate the check at every count.
6186        let just_now = chrono::Utc::now().to_rfc3339();
6187        for count in [1i32, 2, 3, 99] {
6188            let mut task = make_task(None);
6189            task.auto_unblock_count = count;
6190            task.auto_unblock_last_at = just_now.clone();
6191            assert!(
6192                !ci_failure_unblock_cooldown_elapsed(&task),
6193                "count={count} should still be cooling down"
6194            );
6195        }
6196    }
6197
6198    // ── auto_unblock_ci_failure_blocked_tasks_global ───────────────────────
6199
6200    #[serial(cooldown_state)]
6201    #[tokio::test]
6202    async fn global_ci_failure_sweep_returns_ok_with_no_blocked_tasks() {
6203        cooldown::reset_global_state().await;
6204        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6205        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6206        let task_manager = Arc::new(TaskManager::with_store(
6207            backend,
6208            Arc::clone(&store),
6209            "owner/repo".to_string(),
6210        ));
6211        auto_unblock_ci_failure_blocked_tasks_global(
6212            &task_manager,
6213            &store,
6214            &Arc::new(DashSet::new()),
6215        )
6216        .await
6217        .unwrap();
6218    }
6219
6220    #[serial(cooldown_state)]
6221    #[tokio::test]
6222    async fn global_ci_failure_sweep_skips_tasks_in_cooldown() {
6223        // Tasks with a fresh auto_unblock_last_at are skipped before reaching GhHttp,
6224        // so this test works without a real GitHub token.
6225        cooldown::reset_global_state().await;
6226        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6227        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6228        let task_manager = Arc::new(TaskManager::with_store(
6229            backend,
6230            Arc::clone(&store),
6231            "active/repo".to_string(),
6232        ));
6233
6234        // Insert a CI-failure task from an inactive repo with a very recent last_at.
6235        let id = store
6236            .upsert_external(&crate::store::UpsertExternal {
6237                repo: "inactive/repo",
6238                ext_id: "10",
6239                title: "Old blocked task",
6240                body: "",
6241                author: "",
6242                url: "",
6243                labels: &[],
6244                origin: "github",
6245            })
6246            .await
6247            .unwrap();
6248        store.update_status(id, TaskStatus::Blocked).await.unwrap();
6249        let just_now = chrono::Utc::now().to_rfc3339();
6250        store
6251            .set_fields(
6252                id,
6253                &[
6254                    (
6255                        "block_reason",
6256                        serde_json::json!("CI failure limit reached during auto-merge"),
6257                    ),
6258                    ("pr_number", serde_json::json!(99)),
6259                    ("auto_unblock_count", serde_json::json!(1)),
6260                    ("auto_unblock_last_at", serde_json::json!(just_now)),
6261                ],
6262            )
6263            .await
6264            .unwrap();
6265
6266        auto_unblock_ci_failure_blocked_tasks_global(
6267            &task_manager,
6268            &store,
6269            &Arc::new(DashSet::new()),
6270        )
6271        .await
6272        .unwrap();
6273
6274        // Cooldown not elapsed — task must remain untouched.
6275        let task = store.get(id).await.unwrap();
6276        assert_eq!(
6277            task.auto_unblock_count, 1,
6278            "task in cooldown must not be processed"
6279        );
6280    }
6281
6282    #[serial(cooldown_state)]
6283    #[tokio::test]
6284    async fn global_ci_failure_sweep_ignores_non_ci_failure_blocks() {
6285        cooldown::reset_global_state().await;
6286        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6287        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6288        let task_manager = Arc::new(TaskManager::with_store(
6289            backend,
6290            Arc::clone(&store),
6291            "active/repo".to_string(),
6292        ));
6293
6294        // Insert a non-CI-failure blocked task — should be ignored entirely.
6295        let id = store
6296            .upsert_external(&crate::store::UpsertExternal {
6297                repo: "inactive/repo",
6298                ext_id: "20",
6299                title: "Manual block",
6300                body: "",
6301                author: "",
6302                url: "",
6303                labels: &[],
6304                origin: "github",
6305            })
6306            .await
6307            .unwrap();
6308        store.update_status(id, TaskStatus::Blocked).await.unwrap();
6309        store
6310            .set_fields(
6311                id,
6312                &[("block_reason", serde_json::json!("waiting for human input"))],
6313            )
6314            .await
6315            .unwrap();
6316
6317        auto_unblock_ci_failure_blocked_tasks_global(
6318            &task_manager,
6319            &store,
6320            &Arc::new(DashSet::new()),
6321        )
6322        .await
6323        .unwrap();
6324
6325        // Non-CI-failure tasks must remain blocked.
6326        let task = store.get(id).await.unwrap();
6327        assert_eq!(task.status, TaskStatus::Blocked);
6328    }
6329
6330    // ── auto_recover_rebroadcast_blocked_tasks_global ───────────────────────
6331
6332    #[serial(cooldown_state)]
6333    #[tokio::test]
6334    async fn global_rebroadcast_sweep_returns_ok_with_no_blocked_tasks() {
6335        cooldown::reset_global_state().await;
6336        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6337        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6338        let task_manager = Arc::new(TaskManager::with_store(
6339            backend,
6340            Arc::clone(&store),
6341            "owner/repo".to_string(),
6342        ));
6343        let router = router_with_healthy_review_agent();
6344        auto_recover_rebroadcast_blocked_tasks_global(
6345            &router,
6346            &task_manager,
6347            &store,
6348            &sweep_repos(),
6349        )
6350        .await
6351        .unwrap();
6352    }
6353
6354    #[serial(cooldown_state)]
6355    #[tokio::test]
6356    async fn global_rebroadcast_sweep_ignores_non_rebroadcast_blocks() {
6357        cooldown::reset_global_state().await;
6358        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6359        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6360        let task_manager = Arc::new(TaskManager::with_store(
6361            backend,
6362            Arc::clone(&store),
6363            "active/repo".to_string(),
6364        ));
6365        let router = router_with_healthy_review_agent();
6366
6367        let id = store
6368            .upsert_external(&crate::store::UpsertExternal {
6369                repo: "inactive/repo",
6370                ext_id: "30",
6371                title: "Manual block",
6372                body: "",
6373                author: "",
6374                url: "",
6375                labels: &[],
6376                origin: "github",
6377            })
6378            .await
6379            .unwrap();
6380        store.update_status(id, TaskStatus::Blocked).await.unwrap();
6381        store
6382            .set_fields(
6383                id,
6384                &[("block_reason", serde_json::json!("waiting for human input"))],
6385            )
6386            .await
6387            .unwrap();
6388        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
6389            .bind(id)
6390            .execute(store.pool())
6391            .await
6392            .unwrap();
6393
6394        auto_recover_rebroadcast_blocked_tasks_global(
6395            &router,
6396            &task_manager,
6397            &store,
6398            &sweep_repos(),
6399        )
6400        .await
6401        .unwrap();
6402
6403        let task = store.get(id).await.unwrap();
6404        assert_eq!(task.status, TaskStatus::Blocked);
6405        assert_eq!(
6406            task.block_reason.as_deref(),
6407            Some("waiting for human input")
6408        );
6409    }
6410
6411    #[serial(cooldown_state)]
6412    #[tokio::test]
6413    async fn global_rebroadcast_sweep_skips_freshly_blocked_tasks() {
6414        cooldown::reset_global_state().await;
6415        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6416        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6417        let task_manager = Arc::new(TaskManager::with_store(
6418            backend,
6419            Arc::clone(&store),
6420            "active/repo".to_string(),
6421        ));
6422        let router = router_with_healthy_review_agent();
6423
6424        let id = store
6425            .upsert_external(&crate::store::UpsertExternal {
6426                repo: "inactive/repo",
6427                ext_id: "40",
6428                title: "Just escalated",
6429                body: "",
6430                author: "",
6431                url: "",
6432                labels: &[],
6433                origin: "github",
6434            })
6435            .await
6436            .unwrap();
6437        store.update_status(id, TaskStatus::Blocked).await.unwrap();
6438        store
6439            .set_fields(
6440                id,
6441                &[("block_reason", serde_json::json!(REBROADCAST_BLOCK_REASON))],
6442            )
6443            .await
6444            .unwrap();
6445        // No backdating — escalated "just now" in this same tick, must not be
6446        // immediately un-blocked by the same pass.
6447
6448        auto_recover_rebroadcast_blocked_tasks_global(
6449            &router,
6450            &task_manager,
6451            &store,
6452            &sweep_repos(),
6453        )
6454        .await
6455        .unwrap();
6456
6457        let task = store.get(id).await.unwrap();
6458        assert_eq!(task.status, TaskStatus::Blocked);
6459    }
6460
6461    /// Regression test for #3532: a rebroadcast-blocked task whose repo is not
6462    /// the TaskManager's own repo must still be recovered. The sweep queries
6463    /// across all repos and writes through the store directly, bypassing the
6464    /// repo-scoped TaskManager
6465    #[serial(cooldown_state)]
6466    #[tokio::test]
6467    async fn global_rebroadcast_sweep_recovers_task_from_inactive_repo() {
6468        cooldown::reset_global_state().await;
6469        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6470        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6471        let task_manager = Arc::new(TaskManager::with_store(
6472            backend,
6473            Arc::clone(&store),
6474            "active/repo".to_string(),
6475        ));
6476        let router = router_with_healthy_review_agent();
6477
6478        let id = store
6479            .upsert_external(&crate::store::UpsertExternal {
6480                repo: "inactive/repo",
6481                ext_id: "458",
6482                title: "Rebroadcast blocked from inactive repo",
6483                body: "",
6484                author: "",
6485                url: "",
6486                labels: &[],
6487                origin: "github",
6488            })
6489            .await
6490            .unwrap();
6491        store.update_status(id, TaskStatus::Blocked).await.unwrap();
6492        store
6493            .set_fields(
6494                id,
6495                &[
6496                    ("block_reason", serde_json::json!(REBROADCAST_BLOCK_REASON)),
6497                    ("needs_review_refires", serde_json::json!(6)),
6498                ],
6499            )
6500            .await
6501            .unwrap();
6502        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
6503            .bind(id)
6504            .execute(store.pool())
6505            .await
6506            .unwrap();
6507
6508        auto_recover_rebroadcast_blocked_tasks_global(
6509            &router,
6510            &task_manager,
6511            &store,
6512            &sweep_repos(),
6513        )
6514        .await
6515        .unwrap();
6516
6517        let task = store.get(id).await.unwrap();
6518        assert_eq!(
6519            task.status,
6520            TaskStatus::NeedsReview,
6521            "cross-repo rebroadcast-blocked task must be recovered to needs_review"
6522        );
6523        assert!(
6524            task.block_reason.is_none(),
6525            "block_reason must be cleared on recovery"
6526        );
6527        assert_eq!(
6528            task.needs_review_refires, 0,
6529            "needs_review_refires must be reset to 0 on recovery"
6530        );
6531    }
6532
6533    /// With no routable review agent, the sweep must not reset the task, that
6534    /// would immediately re-exhaust the refire counter
6535    #[serial(cooldown_state)]
6536    #[tokio::test]
6537    async fn global_rebroadcast_sweep_skips_reset_without_routable_review_agent() {
6538        cooldown::reset_global_state().await;
6539        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6540        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6541        let task_manager = Arc::new(TaskManager::with_store(
6542            backend,
6543            Arc::clone(&store),
6544            "active/repo".to_string(),
6545        ));
6546        // RouterConfig::default() has no "review" model map entries, so the one
6547        // available agent has no healthy model for "review" — nothing is routable.
6548        let router = Router::new_for_test(RouterConfig::default(), vec!["claude".to_string()]);
6549
6550        let id = store
6551            .upsert_external(&crate::store::UpsertExternal {
6552                repo: "inactive/repo",
6553                ext_id: "459",
6554                title: "Rebroadcast blocked, no routable agents",
6555                body: "",
6556                author: "",
6557                url: "",
6558                labels: &[],
6559                origin: "github",
6560            })
6561            .await
6562            .unwrap();
6563        store.update_status(id, TaskStatus::Blocked).await.unwrap();
6564        store
6565            .set_fields(
6566                id,
6567                &[("block_reason", serde_json::json!(REBROADCAST_BLOCK_REASON))],
6568            )
6569            .await
6570            .unwrap();
6571        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
6572            .bind(id)
6573            .execute(store.pool())
6574            .await
6575            .unwrap();
6576
6577        auto_recover_rebroadcast_blocked_tasks_global(
6578            &router,
6579            &task_manager,
6580            &store,
6581            &sweep_repos(),
6582        )
6583        .await
6584        .unwrap();
6585
6586        let task = store.get(id).await.unwrap();
6587        assert_eq!(task.status, TaskStatus::Blocked);
6588        assert_eq!(task.block_reason.as_deref(), Some(REBROADCAST_BLOCK_REASON));
6589    }
6590}