Skip to main content

orch/engine/
sync.rs

1//! Sync tick — periodic operations that run less frequently than the core tick.
2//!
3//! The sync tick runs every ~45 seconds and handles:
4//! - Worktree cleanup for done tasks
5//! - Merged-PR detection
6//! - @mention scanning and internal task creation
7//! - PR review processing and follow-up dispatch
8//! - Stale InReview detection
9//! - Owner /slash command scanning
10//! - Skill repository syncing
11
12use super::EngineConfig;
13use super::ci_bootstrap;
14use super::cleanup::{check_merged_prs, cleanup_done_worktrees, prune_old_run_output};
15use super::commands::{CommandOutcome, CommandStoreOps, parse_command, validate_and_run_command};
16use super::review_poll::review_open_prs;
17use crate::backends::ExternalId;
18use crate::backends::ExternalTask;
19use crate::backends::{ExternalBackend, Mention, Status};
20use crate::cmd::CommandErrorContext;
21use crate::config;
22use crate::engine;
23use crate::engine::auto_merge::{is_pr_behind, required_checks_state};
24use crate::engine::cleanup::cleanup_task_worktree;
25use crate::engine::commands::CommandGh;
26use crate::engine::commands::OwnerCommand;
27use crate::engine::cooldown;
28use crate::engine::cooldown::{
29    cooldown_reason, github_circuit_remaining_secs, is_agent_in_cooldown, is_github_circuit_open,
30    is_model_in_cooldown, refresh_degraded_agents,
31};
32use crate::engine::dependencies;
33use crate::engine::router::Router;
34use crate::engine::router::RouterConfig;
35use crate::engine::router::config as router_config;
36use crate::engine::runner::agents::patterns;
37use crate::engine::runner::git_ops;
38use crate::engine::tasks;
39use crate::engine::tasks::{TaskManager, status_to_task_status};
40use crate::github::http::GhHttp;
41use crate::github::types::extract_issue_number_from_url;
42use crate::home;
43use crate::store;
44use crate::store::Task;
45use crate::store::TaskRun;
46use crate::store::TaskStatus;
47use crate::store::TaskStore;
48use crate::tmux::TmuxManager;
49use dashmap::{DashMap, DashSet};
50use futures::future;
51use std::collections::HashMap;
52use std::collections::HashSet;
53use std::path::PathBuf;
54use std::sync::Arc;
55use std::time::Duration;
56use std::time::Instant;
57use tokio::fs;
58use tokio::sync::OnceCell;
59use tokio::sync::RwLock;
60use tokio::time;
61
62/// Read a KV value from the store.
63///
64/// Returns `None` if the store is unavailable, the key doesn't exist, OR if
65/// the store query fails. Use [`try_kv_get_prefer_store`] if you need to
66/// distinguish between "key not found" and actual errors.
67async fn kv_get_prefer_store(store: Option<&Arc<TaskStore>>, key: &str) -> Option<String> {
68    match try_kv_get_prefer_store(store, key).await {
69        Ok(opt) => opt,
70        Err(e) => {
71            tracing::error!(key, err = %e, "kv_get failed");
72            None
73        }
74    }
75}
76
77/// Try to read a KV value from the store, returning errors instead of
78/// silently discarding them.
79///
80/// Returns `Ok(None)` if the store is unavailable or the key doesn't exist.
81/// Returns `Err` if the store query fails.
82async fn try_kv_get_prefer_store(
83    store: Option<&Arc<TaskStore>>,
84    key: &str,
85) -> anyhow::Result<Option<String>> {
86    if let Some(s) = store {
87        Ok(s.kv_get(key).await?)
88    } else {
89        Ok(None)
90    }
91}
92
93/// Write a KV value to the store.
94///
95/// Logs warnings on failure but provides no way for callers to know if the
96/// write succeeded. Use [`try_kv_set_prefer_store`] if you need to handle
97/// errors or guarantee persistence (e.g., for circuit breaker state).
98async fn kv_set_prefer_store(store: Option<&Arc<TaskStore>>, key: &str, value: &str) {
99    if let Err(e) = try_kv_set_prefer_store(store, key, value).await {
100        tracing::error!(key, err = %e, "kv_set failed");
101    }
102}
103
104/// Try to write a KV value to the store, returning errors instead of
105/// silently logging them.
106///
107/// Returns `Ok(())` if the store is unavailable (best-effort) or if the
108/// write succeeds. Returns `Err` if the store query fails.
109async fn try_kv_set_prefer_store(
110    store: Option<&Arc<TaskStore>>,
111    key: &str,
112    value: &str,
113) -> anyhow::Result<()> {
114    if let Some(s) = store {
115        s.kv_set(key, value).await?;
116    }
117    Ok(())
118}
119
120#[derive(Debug, Clone)]
121pub(crate) struct ReviewTaskSnapshot {
122    pub external: ExternalTask,
123    pub stored: Task,
124}
125
126async fn prefetch_review_tasks(
127    store: &Arc<TaskStore>,
128    repo: &str,
129) -> anyhow::Result<(
130    Vec<ReviewTaskSnapshot>,
131    Vec<ReviewTaskSnapshot>,
132    Vec<ReviewTaskSnapshot>,
133)> {
134    // Approved tasks wait on the same merge path as a review in progress
135    let mut in_review = store.list_by_status(repo, TaskStatus::InReview).await?;
136    in_review.extend(store.list_by_status(repo, TaskStatus::Approved).await?);
137    let in_review = in_review
138        .into_iter()
139        .map(|stored| ReviewTaskSnapshot {
140            external: tasks::store_task_to_external(&stored),
141            stored,
142        })
143        .collect();
144    let needs_review = store
145        .list_by_status(repo, TaskStatus::NeedsReview)
146        .await?
147        .into_iter()
148        .map(|stored| ReviewTaskSnapshot {
149            external: tasks::store_task_to_external(&stored),
150            stored,
151        })
152        .collect();
153
154    // Also prefetch blocked tasks that have a branch or PR number recorded.
155    // These can be merged externally (or manually) but previously were invisible
156    // to the merged-PR checker, leaving tasks stuck in Blocked after merges.
157    let blocked_candidates = store
158        .list_by_status(repo, TaskStatus::Blocked)
159        .await?
160        .into_iter()
161        .filter(|stored| stored.pr_number.is_some() || !stored.branch.is_empty())
162        .map(|stored| ReviewTaskSnapshot {
163            external: tasks::store_task_to_external(&stored),
164            stored,
165        })
166        .collect();
167
168    Ok((in_review, needs_review, blocked_candidates))
169}
170
171async fn fetch_comments_since(
172    backend: &Arc<dyn ExternalBackend>,
173    since: &str,
174) -> anyhow::Result<Vec<Mention>> {
175    backend.get_mentions(since).await
176}
177
178fn filter_mentions_by_since(comments: &[Mention], since: &str) -> Vec<Mention> {
179    comments
180        .iter()
181        .filter(|comment| comment.created_at.as_str() > since)
182        .cloned()
183        .collect()
184}
185
186#[derive(Debug, Clone, Copy, PartialEq, Eq)]
187enum FailureCategory {
188    FalseFailure,
189    RateLimit,
190    CreditExhausted,
191    SilentExit0,
192    Timeout,
193    ConnectionError,
194    CliFlagError,
195    AllAgentsExhausted,
196    ParseError,
197    MaxAttempts,
198    PushFailed,
199    PrCreateFailed,
200    ModelUnavailable,
201    Unknown,
202}
203
204impl FailureCategory {
205    /// Whether auto-unblock may put the task back in the queue on its own.
206    ///
207    /// `MaxAttempts` is deliberately absent: the task already ran the configured
208    /// number of times, so an automatic retry just restarts the loop that produced
209    /// the block. It stays blocked until `orch task unblock` or `orch task retry`
210    /// (#3783)
211    fn is_recoverable(self) -> bool {
212        matches!(
213            self,
214            FailureCategory::FalseFailure
215                | FailureCategory::RateLimit
216                | FailureCategory::CreditExhausted
217                | FailureCategory::SilentExit0
218                | FailureCategory::Timeout
219                | FailureCategory::ConnectionError
220                | FailureCategory::CliFlagError
221                | FailureCategory::AllAgentsExhausted
222                | FailureCategory::ParseError
223                | FailureCategory::ModelUnavailable
224        )
225    }
226
227    fn as_str(self) -> &'static str {
228        match self {
229            FailureCategory::FalseFailure => "FalseFailure",
230            FailureCategory::RateLimit => "RateLimit",
231            FailureCategory::CreditExhausted => "CreditExhausted",
232            FailureCategory::SilentExit0 => "SilentExit0",
233            FailureCategory::Timeout => "Timeout",
234            FailureCategory::ConnectionError => "ConnectionError",
235            FailureCategory::CliFlagError => "CliFlagError",
236            FailureCategory::AllAgentsExhausted => "AllAgentsExhausted",
237            FailureCategory::ParseError => "ParseError",
238            FailureCategory::MaxAttempts => "MaxAttempts",
239            FailureCategory::PushFailed => "PushFailed",
240            FailureCategory::PrCreateFailed => "PrCreateFailed",
241            FailureCategory::ModelUnavailable => "ModelUnavailable",
242            FailureCategory::Unknown => "Unknown",
243        }
244    }
245}
246
247fn classify_failure(error: &str, outcome: &str) -> FailureCategory {
248    let lower = error.to_lowercase();
249
250    if outcome == "timeout" || lower.contains("timeout") {
251        return FailureCategory::Timeout;
252    }
253
254    // Credit exhaustion is a transient condition (credits replenish, billing cycles
255    // reset). Classifying it as recoverable allows auto-unblock to retry once the
256    // credit-related cooldown expires, instead of permanently blocking the task.
257    //
258    // Checked before the generic rate-limit branch below: billing-cycle exhaustion
259    // messages (e.g. Kimi's "usage limit for this billing cycle") also contain
260    // "usage limit"/"rate limit" substrings, so checking outcome/content-based
261    // rate-limit patterns first would misclassify them as plain RateLimit (issue
262    // #3529). `outcome` also carries the specific string from `classify_run_outcome`
263    // when the task_run already recorded it.
264    if outcome == "billing_cycle_exhausted"
265        || outcome == "credit_exhausted"
266        || outcome == "org_level_disabled"
267        || cooldown::detect_credit_exhaustion(&lower).is_some()
268    {
269        return FailureCategory::CreditExhausted;
270    }
271
272    if outcome == "rate_limit" || lower.contains("rate limit") || lower.contains("usage limit") {
273        return FailureCategory::RateLimit;
274    }
275
276    if error.trim().is_empty() {
277        return FailureCategory::FalseFailure;
278    }
279
280    if lower.contains("all agents exhausted") {
281        return FailureCategory::AllAgentsExhausted;
282    }
283
284    if lower.contains("max attempts") || lower.contains("exceeded max attempts") {
285        return FailureCategory::MaxAttempts;
286    }
287
288    if lower.contains("output-format=stream-json") && lower.contains("requires --verbose") {
289        return FailureCategory::CliFlagError;
290    }
291
292    if lower.contains("model unavailable")
293        || lower.contains("model not found")
294        || (lower.contains("does not exist") && lower.contains("model"))
295    {
296        return FailureCategory::ModelUnavailable;
297    }
298
299    if lower.contains("push failed") {
300        return FailureCategory::PushFailed;
301    }
302
303    if lower.contains("create pr failed")
304        || lower.contains("failed to create pull request")
305        || lower.contains("pull request creation failed")
306    {
307        return FailureCategory::PrCreateFailed;
308    }
309
310    if lower.contains("exit 0") {
311        return FailureCategory::SilentExit0;
312    }
313
314    if outcome == "parse_error"
315        || outcome == "truncated"
316        || lower.contains("invalid response")
317        || lower.contains("parse error")
318        || lower.contains("truncated")
319    {
320        return FailureCategory::ParseError;
321    }
322
323    if patterns::detect_network_error(&lower).is_some() {
324        return FailureCategory::ConnectionError;
325    }
326
327    FailureCategory::Unknown
328}
329
330fn classify_failure_from_run(run: &TaskRun) -> Option<FailureCategory> {
331    // Completed successfully — no failure to classify.
332    if run.outcome == "success" && run.error.trim().is_empty() {
333        return None;
334    }
335
336    // Skip runs with no meaningful data. Some runs are created then never
337    // populated (aborted/stale work). Treat these as non-failures so they
338    // don't block auto-unblock logic which considers recent failures.
339    // Historically an all-NULL/empty run would be promoted to Unknown which
340    // prevented auto-unblock. Ensure we early-return None for truly empty
341    // runs (no outcome, no error, no exit_code, no output).
342    if run.outcome.trim().is_empty()
343        && run.error.trim().is_empty()
344        && run.exit_code.is_none()
345        && run.stdout.trim().is_empty()
346        && run.stderr.trim().is_empty()
347    {
348        return None;
349    }
350
351    let has_output = !run.stdout.trim().is_empty() || !run.stderr.trim().is_empty();
352    if run.exit_code == Some(0) && !has_output && run.outcome != "success" {
353        return Some(FailureCategory::SilentExit0);
354    }
355
356    let mut category = classify_failure(&run.error, &run.outcome);
357
358    // Promote a bare "false failure" to Unknown only when we have a
359    // concrete non-zero exit code or other signs the run truly failed.
360    // Do NOT promote when exit_code is None (incomplete run) — those are
361    // handled above by skipping. This prevents NULL/aborted runs from
362    // turning recoverable FalseFailure into non-recoverable Unknown.
363    // Only promote when we have explicit evidence: a non-zero exit code
364    // or non-empty stdout/stderr. If exit_code is None we keep FalseFailure
365    // so the run remains recoverable.
366    if category == FailureCategory::FalseFailure {
367        let has_nonzero_exit = run.exit_code.is_some() && run.exit_code != Some(0);
368        let has_output = !run.stdout.trim().is_empty() || !run.stderr.trim().is_empty();
369        if has_nonzero_exit || has_output {
370            category = FailureCategory::Unknown;
371        }
372    }
373
374    Some(category)
375}
376
377/// Resolve the parent task ID for a mention.
378///
379/// - If the mention is on a PR, look up the task that owns that PR via `pr_number`.
380/// - If the mention is on an issue, look up the task by external_id (issue number).
381///
382/// Returns:
383/// - `Ok(Some(id))` — parent found
384/// - `Ok(None)` — no matching parent (genuine miss, e.g. unknown issue/PR)
385/// - `Err(_)` — lookup failed (DB error); caller should defer/retry
386async fn resolve_mention_parent_id(
387    store: &TaskStore,
388    repo: &str,
389    issue_num: &str,
390    is_pr: bool,
391) -> anyhow::Result<Option<i64>> {
392    if is_pr {
393        if let Ok(pr_num) = issue_num.parse::<i32>() {
394            store.resolve_id_by_pr_number(repo, pr_num).await
395        } else {
396            Ok(None)
397        }
398    } else {
399        store.resolve_id_by_external(repo, issue_num).await
400    }
401}
402
403/// Create a sentinel mention task in the store and advance the cursor.
404async fn record_mention_task(
405    store: &TaskStore,
406    repo: &str,
407    mention: &Mention,
408    title: &str,
409    body: &str,
410    parent_id: Option<i64>,
411    cursor: &mut MentionCursor,
412) -> bool {
413    match store
414        .create_internal(repo, title, body, "mention", &mention.id, parent_id)
415        .await
416    {
417        Ok(task_id) => {
418            tracing::info!(task_id, mention_id = %mention.id, "created mention task");
419            cursor.advance(&mention.created_at);
420            true
421        }
422        Err(e) => {
423            tracing::warn!(mention_id = %mention.id, err = %e, "failed to create mention task");
424            false
425        }
426    }
427}
428
429/// Advance the mention cursor if the timestamp is newer.
430fn advance_cursor(last_safe_ts: &mut Option<String>, ts: &str) {
431    if last_safe_ts.as_deref() < Some(ts) {
432        *last_safe_ts = Some(ts.to_string());
433    }
434}
435
436#[derive(Debug, Default)]
437struct MentionCursor {
438    last_safe_ts: Option<String>,
439    blocked_by_gap: bool,
440}
441
442impl MentionCursor {
443    fn advance(&mut self, ts: &str) {
444        if self.blocked_by_gap {
445            return;
446        }
447        advance_cursor(&mut self.last_safe_ts, ts);
448    }
449
450    fn block_on_gap(&mut self) {
451        self.blocked_by_gap = true;
452    }
453
454    fn into_last_safe_ts(self) -> Option<String> {
455        self.last_safe_ts
456    }
457}
458
459/// What to do with a comment in the unified scan loop.
460#[derive(Debug, PartialEq)]
461pub(crate) enum CommentAction {
462    /// Already processed or not relevant — skip.
463    Skip,
464    /// Slash command without @mention — execute the command.
465    ExecuteCommand {
466        command: OwnerCommand,
467        issue_num: String,
468    },
469    /// @mention with a slash command — execute and record sentinel task.
470    ExecuteCommandForMention {
471        command: OwnerCommand,
472        issue_num: String,
473    },
474    /// @mention without command — create internal task for agent to respond.
475    CreateMentionTask { issue_num: Option<String> },
476}
477
478/// Classify a comment into the appropriate action.
479///
480/// Pure function — no I/O, no async. Determines what the scan loop should do
481/// with each comment based on its content and whether it was already processed.
482pub(crate) fn classify_comment(
483    body: &str,
484    issue_url: Option<&str>,
485    current_user: &str,
486    already_processed: bool,
487) -> CommentAction {
488    if already_processed {
489        return CommentAction::Skip;
490    }
491
492    let is_mention =
493        body.contains(current_user) || body.contains("@orch") || body.contains("@orchestrator");
494    let command = parse_command(body);
495    let issue_num = issue_url.and_then(extract_issue_number_from_url);
496
497    match (command, is_mention, issue_num) {
498        (Some(cmd), true, Some(num)) => CommentAction::ExecuteCommandForMention {
499            command: cmd,
500            issue_num: num,
501        },
502        (Some(cmd), false, Some(num)) => CommentAction::ExecuteCommand {
503            command: cmd,
504            issue_num: num,
505        },
506        // Command but no issue URL — if it's a mention, create task; otherwise skip
507        (Some(_), true, None) => CommentAction::CreateMentionTask { issue_num: None },
508        // Mention without command — create task for agent
509        (None, true, issue_num) => CommentAction::CreateMentionTask { issue_num },
510        // Neither mention nor command — skip
511        (Some(_), false, None) | (None, false, _) => CommentAction::Skip,
512    }
513}
514
515/// Handle a slash command from a mention. Records a sentinel task and advances
516/// the cursor on success; logs a warning and returns early if the mention has
517/// no valid issue URL.
518///
519/// Execute a slash command mentioned by `@orch` on a comment.
520///
521/// Returns:
522/// - `Ok(true)` — command processed and cursor may continue advancing
523/// - `Ok(false)` — mention task insert failed, or validation hit a transient
524///   GitHub error; caller must stop cursor advancement so the next tick retries
525/// - `Err(_)` — DB lookup failed for parent_id; caller must NOT advance cursor
526///   so this mention is retried on the next sync
527#[expect(
528    clippy::too_many_arguments,
529    reason = "the sync tick hands over its shared state piece by piece"
530)]
531async fn handle_slash_command(
532    backend: &Arc<dyn ExternalBackend>,
533    store: Option<&Arc<TaskStore>>,
534    repo: &str,
535    task_manager: &Arc<TaskManager>,
536    gh: &dyn CommandGh,
537    mention: &Mention,
538    command: &OwnerCommand,
539    cursor: &mut MentionCursor,
540) -> anyhow::Result<bool> {
541    let Some(issue_num) = mention
542        .issue_url
543        .as_deref()
544        .and_then(extract_issue_number_from_url)
545    else {
546        tracing::warn!(comment_id = %mention.id, "slash command without valid issue number");
547        return Ok(true);
548    };
549
550    let store_opt: Option<Arc<dyn CommandStoreOps>> =
551        store.map(|s| Arc::clone(s) as Arc<dyn CommandStoreOps>);
552    let outcome = validate_and_run_command(
553        backend,
554        gh,
555        repo,
556        &issue_num,
557        command,
558        &mention.author,
559        &store_opt,
560        task_manager,
561    )
562    .await;
563
564    // Extract is_pr from the outcome (avoids a second get_issue call)
565    let is_pr = match &outcome {
566        CommandOutcome::Executed { is_pr }
567        | CommandOutcome::NotOpen { is_pr }
568        | CommandOutcome::NotCollaborator { is_pr } => *is_pr,
569        CommandOutcome::FetchFailed | CommandOutcome::CollaboratorCheckFailed => {
570            // Nothing was executed and nothing was posted to GitHub. Leave the cursor
571            // before this comment so the next tick retries the command.
572            tracing::warn!(
573                comment_id = %mention.id,
574                command = %command,
575                "deferring @orch slash command after a transient GitHub error"
576            );
577            return Ok(false);
578        }
579    };
580
581    // Record a sentinel task for the mention (with correct parent_id).
582    // On DB lookup failure, defer so the mention can be retried on next sync.
583    if let Some(s) = store {
584        let parent_id = match resolve_mention_parent_id(s, repo, &issue_num, is_pr).await {
585            Ok(parent_id) => parent_id,
586            Err(e) => {
587                tracing::warn!(
588                    comment_id = %mention.id,
589                    err = %e,
590                    "deferring mention task due to parent lookup failure"
591                );
592                return Err(e);
593            }
594        };
595
596        let (title, body) = match &outcome {
597            CommandOutcome::NotOpen { .. } => (
598                format!(
599                    "Skipped @orch command on closed #{issue_num} from @{}",
600                    mention.author
601                ),
602                format!(
603                    "Mention by @{}:\n\n{}\n\n**Skipped:** target is closed",
604                    mention.author, mention.body
605                ),
606            ),
607            CommandOutcome::NotCollaborator { .. } => (
608                format!(
609                    "Skipped @orch command on #{issue_num} from non-collaborator @{}",
610                    mention.author
611                ),
612                format!(
613                    "Mention by @{}:\n\n{}\n\n**Skipped:** author is not a collaborator",
614                    mention.author, mention.body
615                ),
616            ),
617            CommandOutcome::Executed { .. } => (
618                format!(
619                    "Respond to mention by @{} on task #{issue_num} — command executed",
620                    mention.author
621                ),
622                format!(
623                    "Mention by @{} on #{issue_num}:\n\n{}\n\n**Status:** Command executed",
624                    mention.author, mention.body
625                ),
626            ),
627            CommandOutcome::FetchFailed | CommandOutcome::CollaboratorCheckFailed => {
628                tracing::debug!("skipping mention task record due to earlier command failure");
629                return Ok(true);
630            }
631        };
632
633        let created = record_mention_task(s, repo, mention, &title, &body, parent_id, cursor).await;
634        if !created {
635            return Ok(false);
636        }
637    }
638    Ok(true)
639}
640
641fn auto_unblock_cooldown_elapsed(count: i32, last_at: &str) -> bool {
642    if count == 0 {
643        return true;
644    }
645
646    let required = match count {
647        1 => chrono::Duration::minutes(30),
648        2 => chrono::Duration::hours(2),
649        _ => return false,
650    };
651
652    if last_at.trim().is_empty() {
653        return true;
654    }
655
656    match chrono::DateTime::parse_from_rfc3339(last_at) {
657        Ok(ts) => chrono::Utc::now() - ts.with_timezone(&chrono::Utc) >= required,
658        Err(_) => true,
659    }
660}
661
662fn is_ci_failure_block(task: &Task) -> bool {
663    let reason = task.block_reason.as_deref().unwrap_or("");
664    reason.contains("CI failure limit") || reason.contains("CI checks timed out")
665}
666
667fn ci_failure_unblock_cooldown_elapsed(task: &Task) -> bool {
668    if task.auto_unblock_count == 0 {
669        return true;
670    }
671
672    // Short cooldowns — the PR may close quickly after the first check.
673    // We never permanently block (no `_ => return false`): eventually the PR
674    // will close or a human will intervene.
675    let required = match task.auto_unblock_count {
676        1 => chrono::Duration::minutes(10),
677        2 => chrono::Duration::hours(1),
678        3 => chrono::Duration::hours(6),
679        _ => chrono::Duration::hours(24),
680    };
681
682    let last_at = &task.auto_unblock_last_at;
683    if last_at.trim().is_empty() {
684        return true;
685    }
686
687    match chrono::DateTime::parse_from_rfc3339(last_at) {
688        Ok(ts) => chrono::Utc::now() - ts.with_timezone(&chrono::Utc) >= required,
689        Err(_) => true,
690    }
691}
692
693async fn try_unblock_ci_failure_task(
694    gh: &GhHttp,
695    repo: &str,
696    task_manager: &Arc<TaskManager>,
697    store: &Arc<TaskStore>,
698    task: &Task,
699    auto_merge_in_flight: &Arc<DashSet<String>>,
700) -> anyhow::Result<bool> {
701    let Some(pr_number) = task.pr_number.and_then(|n| u64::try_from(n).ok()) else {
702        tracing::debug!(
703            task_id = task.id,
704            "CI failure blocked task has no PR number"
705        );
706        return Ok(false);
707    };
708
709    let pr = match gh.get_pr(repo, pr_number).await {
710        Ok(p) => p,
711        Err(e) => {
712            tracing::warn!(task_id = task.id, pr_number, err = %e, "failed to get PR status for CI failure block");
713            return Ok(false);
714        }
715    };
716
717    tracing::debug!(
718        task_id = task.id,
719        pr_number,
720        merged = pr.merged,
721        state = ?pr.state,
722        "verifying PR state for CI failure blocked task"
723    );
724
725    if pr.merged.unwrap_or(false) || pr.state.eq_ignore_ascii_case("closed") {
726        tracing::info!(
727            task_id = task.id,
728            pr_number,
729            merged = pr.merged,
730            state = %pr.state,
731            "PR merged/closed — unblocking CI failure blocked task"
732        );
733        let ext_id = task
734            .external_id
735            .clone()
736            .unwrap_or_else(|| format!("internal:{}", task.id));
737        // Update the store directly by the already-resolved numeric id instead of going
738        // through `task_manager.update_task_status`, which resolves by `self.repo` — this
739        // sweep runs cross-repo, so `task.repo` may not match the caller's own repo scope.
740        task_manager
741            .mark_task_done_by_store_id(task.id, &task.repo, &ExternalId(ext_id))
742            .await?;
743        let fields = vec![
744            ("block_reason", serde_json::Value::Null),
745            ("auto_unblock_count", serde_json::json!(0)),
746            ("auto_unblock_last_at", serde_json::json!("")),
747            ("auto_unblock_last_reason", serde_json::json!("")),
748        ];
749        store.set_fields(task.id, &fields).await?;
750        return Ok(true);
751    }
752
753    // Skip if a recovery merge is already in flight for this task. The poll loop
754    // can take up to 10 minutes, and the same task may be seen by both the per-repo
755    // sync_tick and the global sweep in the same tick window.
756    let task_id_str = task
757        .external_id
758        .clone()
759        .unwrap_or_else(|| format!("internal:{}", task.id));
760    if auto_merge_in_flight.contains(&task_id_str) {
761        tracing::debug!(
762            task_id = task.id,
763            pr_number,
764            "CI-failure recovery merge already in flight — skipping"
765        );
766        return Ok(false);
767    }
768
769    // Give the PR an active chance to recover: merge the base branch into the PR
770    // branch to re-trigger CI, then re-enable GitHub auto-merge. GitHub's queue
771    // can silently stall even when checks are green and the branch is up to date,
772    // so after arming auto-merge we poll for completion and fall back to a direct
773    // merge once the PR is mergeable and CI is passing.
774    let branch_update_result = gh.update_pr_branch(repo, pr_number).await;
775    match branch_update_result {
776        Ok(()) => {
777            tracing::info!(
778                task_id = task.id,
779                pr_number,
780                "updated CI-failure-blocked PR branch against base to retrigger CI"
781            );
782        }
783        Err(ref e) => {
784            // warn, not debug, so a silently-broken corrective retry doesn't look like a working one (#3561)
785            tracing::warn!(
786                task_id = task.id,
787                pr_number,
788                err = %e,
789                outcome = "branch_update_failed",
790                "failed to update CI-failure-blocked PR branch (may already be up to date)"
791            );
792        }
793    }
794
795    if let Err(e) = gh.enable_auto_merge(repo, pr_number).await {
796        tracing::warn!(task_id = task.id, pr_number, err = %e, "failed to re-enable auto-merge after branch update");
797    }
798
799    // Record this attempt so the cooldown advances even though the actual
800    // poll/merge now runs in the background.
801    let pr_state = pr.state.as_str();
802    let new_reason = if let Err(ref e) = branch_update_result {
803        format!(
804            "CI failure limit reached during auto-merge (PR #{pr_number} still open, state: {pr_state}, branch update failed: {e})"
805        )
806    } else {
807        format!(
808            "CI failure limit reached during auto-merge (PR #{pr_number} still open, state: {pr_state})"
809        )
810    };
811    let now = chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string();
812    // Increment auto_unblock_count so the cooldown backoff (24h, 3d, never) kicks in
813    // and we don't hammer the GitHub API every tick for a PR that stays open.
814    let new_count = task.auto_unblock_count.saturating_add(1);
815    let fields = vec![
816        ("block_reason", serde_json::json!(&new_reason)),
817        ("auto_unblock_count", serde_json::json!(new_count)),
818        ("auto_unblock_last_at", serde_json::json!(&now)),
819    ];
820    // Use set_fields_silent so the probe write does not bump updated_at on stale
821    // blocked tasks — triage queries keyed off updated_at must only see real
822    // status/result changes, not periodic re-checks of an open PR.
823    store.set_fields_silent(task.id, &fields).await?;
824
825    // Spawn the poll and direct-merge fallback in the background so the
826    // recovery sweep does not block the main tick loop (or sync_tick) while
827    // CI runs. The in-flight guard prevents double-spawns across ticks.
828    auto_merge_in_flight.insert(task_id_str.clone());
829    let gh_clone = gh.clone();
830    let repo_string = repo.to_string();
831    let task_manager_clone = Arc::clone(task_manager);
832    let store_clone = Arc::clone(store);
833    let task_clone = task.clone();
834    let in_flight_clone = Arc::clone(auto_merge_in_flight);
835
836    tokio::spawn(async move {
837        let _guard = scopeguard::guard((), |()| {
838            in_flight_clone.remove(&task_id_str);
839        });
840
841        match poll_and_merge_recovered_pr(&gh_clone, &repo_string, pr_number, task_clone.id).await {
842            Ok(true) => {
843                tracing::info!(
844                    task_id = task_clone.id,
845                    pr_number,
846                    "CI-failure-blocked PR merged during recovery sweep"
847                );
848                let ext_id = task_clone
849                    .external_id
850                    .clone()
851                    .unwrap_or_else(|| format!("internal:{}", task_clone.id));
852                let fields = [
853                    ("block_reason", serde_json::Value::Null),
854                    ("auto_unblock_count", serde_json::json!(0)),
855                    ("auto_unblock_last_at", serde_json::json!("")),
856                    ("auto_unblock_last_reason", serde_json::json!("")),
857                ];
858                if let Err(e) = task_manager_clone
859                    .update_task_status_and_result_by_store_id(
860                        task_clone.id,
861                        &task_clone.repo,
862                        &ExternalId(ext_id),
863                        Status::Done,
864                        &fields,
865                    )
866                    .await
867                {
868                    tracing::warn!(
869                        task_id = task_clone.id,
870                        pr_number,
871                        err = %e,
872                        "failed to mark CI-failure-blocked task done after recovery merge"
873                    );
874                }
875                if let Err(e) =
876                    cleanup_task_worktree(&format!("{}", task_clone.id), &repo_string, &store_clone)
877                        .await
878                {
879                    tracing::warn!(
880                        task_id = task_clone.id,
881                        pr_number,
882                        err = %e,
883                        "cleanup after CI-failure recovery merge failed"
884                    );
885                }
886            }
887            Ok(false) => {
888                tracing::debug!(
889                    task_id = task_clone.id,
890                    pr_number,
891                    "CI-failure-blocked PR not merged this recovery tick — poll completed without merge"
892                );
893            }
894            Err(e) => {
895                tracing::warn!(
896                    task_id = task_clone.id,
897                    pr_number,
898                    err = %e,
899                    "error while polling CI-failure-blocked PR for recovery"
900                );
901            }
902        }
903    });
904
905    Ok(false)
906}
907
908/// Poll a recovered CI-failure PR until it merges or its checks settle, then
909/// attempt a direct merge if GitHub's auto-merge queue has not completed it.
910///
911/// GitHub's `enablePullRequestAutoMerge` hands the PR off to GitHub's queue and
912/// returns immediately; the queue can stall silently. This helper mirrors the
913/// primary `auto_merge_pr` polling pattern but is scoped to the recovery sweep:
914/// it waits for required checks to pass, verifies the PR is not behind the base
915/// branch, and calls `gh.merge_pr()` directly when the PR is ready but still
916/// open. Returns `Ok(true)` when the PR is merged/closed, `Ok(false)` when the
917/// PR needs to wait for a future tick.
918async fn poll_and_merge_recovered_pr(
919    gh: &GhHttp,
920    repo: &str,
921    pr_number: u64,
922    task_id: i64,
923) -> anyhow::Result<bool> {
924    // The branch can fall behind while CI runs. Retry the update-branch call a
925    // few times inside the poll loop instead of exiting and waiting for the next
926    // 24-hour recovery cooldown.
927    const MAX_BEHIND_RETRIES: u32 = 3;
928
929    let max_wait_secs: u64 = config::get("workflow.ci_poll_max_wait_secs")
930        .ok()
931        .and_then(|s| s.parse().ok())
932        .unwrap_or(600);
933    let base_interval_secs: u64 = config::get("workflow.ci_poll_interval_secs")
934        .ok()
935        .and_then(|s| s.parse().ok())
936        .unwrap_or(15);
937    let max_wait = Duration::from_secs(max_wait_secs);
938    let start = Instant::now();
939    let mut poll_count: u32 = 0;
940    let mut behind_retries: u32 = 0;
941
942    let initial_pr = gh.get_pr(repo, pr_number).await?;
943    let base_branch = initial_pr.base.ref_.clone();
944    let required_contexts = gh
945        .get_required_status_check_contexts(repo, &base_branch)
946        .await?;
947    let repo_has_workflows = if required_contexts.is_empty() {
948        gh.has_workflows(repo).await?
949    } else {
950        true
951    };
952
953    loop {
954        let pr = gh.get_pr(repo, pr_number).await?;
955        if pr.merged.unwrap_or(false) || pr.state.eq_ignore_ascii_case("closed") {
956            tracing::info!(
957                task_id = task_id,
958                pr_number,
959                "PR merged/closed during recovery poll"
960            );
961            return Ok(true);
962        }
963
964        if pr.mergeable == Some(false) {
965            tracing::info!(
966                task_id = task_id,
967                pr_number,
968                outcome = "poll_exited_without_merge",
969                reason = "merge_conflicts",
970                "PR has merge conflicts after recovery update — leaving for next recovery cycle"
971            );
972            return Ok(false);
973        }
974
975        if is_pr_behind(&pr) {
976            if behind_retries >= MAX_BEHIND_RETRIES {
977                tracing::info!(
978                    task_id = task_id,
979                    pr_number,
980                    behind_retries,
981                    max_behind_retries = MAX_BEHIND_RETRIES,
982                    outcome = "poll_exited_without_merge",
983                    reason = "behind_retry_exhausted",
984                    "PR still behind base after recovery retries — leaving for next recovery cycle"
985                );
986                return Ok(false);
987            }
988            behind_retries += 1;
989            match gh.update_pr_branch(repo, pr_number).await {
990                Ok(()) => {
991                    tracing::info!(
992                        task_id = task_id,
993                        pr_number,
994                        behind_retries,
995                        "PR was behind base — updated branch and continuing recovery poll"
996                    );
997                    continue;
998                }
999                Err(e) => {
1000                    tracing::warn!(
1001                        task_id = task_id,
1002                        pr_number,
1003                        behind_retries,
1004                        err = %e,
1005                        outcome = "branch_update_failed",
1006                        "failed to update behind PR branch during recovery poll"
1007                    );
1008                    return Ok(false);
1009                }
1010            }
1011        }
1012
1013        let head_sha = pr.head.sha.clone();
1014        let (state, total, passing, failing, pending) = if required_contexts.is_empty() {
1015            gh.get_combined_status(
1016                repo,
1017                &head_sha,
1018                repo_has_workflows,
1019                pr.mergeable_state.as_deref(),
1020            )
1021            .await?
1022        } else {
1023            let check_runs = gh.get_check_runs(repo, &head_sha).await?;
1024            let statuses = gh.get_commit_status_contexts(repo, &head_sha).await?;
1025            required_checks_state(&required_contexts, &check_runs, &statuses)
1026        };
1027
1028        tracing::info!(
1029            task_id = task_id,
1030            pr_number,
1031            state = %state,
1032            total,
1033            passing,
1034            failing,
1035            pending,
1036            "CI recovery status check"
1037        );
1038
1039        match state.as_str() {
1040            "success" => {
1041                tracing::info!(
1042                    task_id = task_id,
1043                    pr_number,
1044                    "CI green after recovery — attempting direct merge"
1045                );
1046                match gh.merge_pr(repo, pr_number, true, &head_sha).await {
1047                    Ok(()) => {
1048                        tracing::info!(
1049                            task_id = task_id,
1050                            pr_number,
1051                            "direct merge succeeded after recovery"
1052                        );
1053                        return Ok(true);
1054                    }
1055                    Err(e) => {
1056                        let err = e.to_string().to_ascii_lowercase();
1057                        tracing::warn!(
1058                            task_id = task_id,
1059                            pr_number,
1060                            err = %e,
1061                            "direct merge failed after recovery"
1062                        );
1063                        // If the PR became merged between the get_pr call and the
1064                        // merge attempt, treat it as success rather than a failed tick.
1065                        if err.contains("already merged")
1066                            || err.contains("no merge commit")
1067                            || err.contains("pull request already merged")
1068                        {
1069                            return Ok(true);
1070                        }
1071                        // The head moved after we read it, and updating the branch would skip CI on the new commit
1072                        if err.contains("head branch was modified")
1073                            || err.contains("sha does not match")
1074                        {
1075                            tracing::info!(
1076                                task_id = task_id,
1077                                pr_number,
1078                                outcome = "poll_exited_without_merge",
1079                                reason = "head_moved",
1080                                "merge rejected because the head sha changed"
1081                            );
1082                            return Ok(false);
1083                        }
1084                        // GitHub may reject the merge because the branch fell behind
1085                        // again while CI was finishing. Retry the branch update inside
1086                        // the poll window instead of waiting for the next sweep.
1087                        if (err.contains("head branch is out of date")
1088                            || err.contains("behind")
1089                            || err.contains("409"))
1090                            && behind_retries < MAX_BEHIND_RETRIES
1091                        {
1092                            behind_retries += 1;
1093                            match gh.update_pr_branch(repo, pr_number).await {
1094                                Ok(()) => {
1095                                    tracing::info!(
1096                                        task_id = task_id,
1097                                        pr_number,
1098                                        behind_retries,
1099                                        "merge rejected due to out-of-date branch — updated and continuing recovery poll"
1100                                    );
1101                                    continue;
1102                                }
1103                                Err(update_err) => {
1104                                    tracing::warn!(
1105                                        task_id = task_id,
1106                                        pr_number,
1107                                        behind_retries,
1108                                        err = %update_err,
1109                                        outcome = "branch_update_failed",
1110                                        "failed to update out-of-date PR branch during recovery poll"
1111                                    );
1112                                }
1113                            }
1114                        }
1115                        tracing::info!(
1116                            task_id = task_id,
1117                            pr_number,
1118                            outcome = "poll_exited_without_merge",
1119                            reason = "direct_merge_failed",
1120                            "direct merge did not succeed — leaving for next recovery cycle"
1121                        );
1122                        return Ok(false);
1123                    }
1124                }
1125            }
1126            "failure" => {
1127                tracing::info!(
1128                    task_id = task_id,
1129                    pr_number,
1130                    outcome = "poll_exited_without_merge",
1131                    reason = "ci_failure",
1132                    "CI still failing after recovery — will retry later"
1133                );
1134                return Ok(false);
1135            }
1136            _ => {}
1137        }
1138
1139        if start.elapsed() >= max_wait {
1140            tracing::info!(
1141                task_id = task_id,
1142                pr_number,
1143                outcome = "poll_exited_without_merge",
1144                reason = "poll_timeout",
1145                "CI still pending after recovery poll timeout — will retry later"
1146            );
1147            return Ok(false);
1148        }
1149
1150        poll_count += 1;
1151        let multiplier = (2_u64).pow(poll_count.saturating_sub(1)).min(4);
1152        let sleep_secs = base_interval_secs.saturating_mul(multiplier);
1153        time::sleep(Duration::from_secs(sleep_secs)).await;
1154    }
1155}
1156
1157/// Exact `block_reason` written when a task's NeedsReview refire counter exhausts
1158/// during a rebroadcast escalation (`refire_and_escalate_stale_needs_review_global`).
1159/// Shared with `auto_recover_rebroadcast_blocked_tasks_global` so escalation and
1160/// recovery never drift apart on the literal string
1161const REBROADCAST_BLOCK_REASON: &str = "review agent rebroadcast escalated after repeated retries";
1162
1163/// Global sweep: auto-recover rebroadcast-blocked tasks across all repos.
1164///
1165/// When a NeedsReview refire counter exhausts, the task is blocked with
1166/// `REBROADCAST_BLOCK_REASON`. Once any review agent is routable again, this
1167/// sweep resets matching tasks to `NeedsReview` so the subscriber can pick them up.
1168/// Writes through the store via `*_by_store_id` helpers, not a repo-scoped
1169/// `TaskManager`. Called once per main tick from `tick_global_sweeps`.
1170///
1171/// Before resetting, the attached PR's live state is checked (mirroring
1172/// `try_unblock_ci_failure_task`): merged means the task is marked done, closed
1173/// unmerged means it stays blocked with an annotated reason, and only an open PR
1174/// (or no PR) falls through to `NeedsReview`. A 5-minute block age prevents
1175/// same-tick escalate/recover flapping. Reset-to-NeedsReview still requires at
1176/// least one routable review agent
1177pub(crate) async fn auto_recover_rebroadcast_blocked_tasks_global(
1178    router: &Router,
1179    task_manager: &Arc<TaskManager>,
1180    store: &Arc<TaskStore>,
1181    active_repos: &HashSet<String>,
1182) -> anyhow::Result<()> {
1183    const MIN_BLOCK_AGE_MINUTES: i64 = 5;
1184
1185    let all_blocked = match store.list_all_by_status_global(TaskStatus::Blocked).await {
1186        Ok(tasks) => tasks,
1187        Err(e) => {
1188            tracing::warn!(err = %e, "global rebroadcast-blocked sweep: failed to list blocked tasks");
1189            return Ok(());
1190        }
1191    };
1192
1193    let now = chrono::Utc::now();
1194    let candidates: Vec<_> = all_blocked
1195        .into_iter()
1196        .filter(|t| {
1197            if t.block_reason.as_deref() != Some(REBROADCAST_BLOCK_REASON) {
1198                return false;
1199            }
1200            // A repo outside the active set has no review subscriber, so a reset would loop
1201            if !active_repos.contains(&t.repo) {
1202                return false;
1203            }
1204            // Only recover tasks blocked long enough to rule out same-tick escalations.
1205            match chrono::DateTime::parse_from_rfc3339(&t.updated_at) {
1206                Ok(updated_at) => {
1207                    let age = now - updated_at.with_timezone(&chrono::Utc);
1208                    age.num_minutes() >= MIN_BLOCK_AGE_MINUTES
1209                }
1210                Err(_) => true,
1211            }
1212        })
1213        .collect();
1214
1215    if candidates.is_empty() {
1216        return Ok(());
1217    }
1218
1219    let gh = match GhHttp::new() {
1220        Ok(g) => Some(g),
1221        Err(e) => {
1222            tracing::warn!(err = %e, "global rebroadcast-blocked sweep: failed to create GhHttp — skipping PR-state checks this tick");
1223            None
1224        }
1225    };
1226
1227    // Skip reset-to-NeedsReview when no review agent is routable, re-firing
1228    // while cooled would immediately re-exhaust the counter
1229    let any_routable =
1230        !router.available_agents.is_empty() && router.healthy_agent_count("review") > 0;
1231
1232    for task in candidates {
1233        let ext_id = task
1234            .external_id
1235            .clone()
1236            .unwrap_or_else(|| format!("internal:{}", task.id));
1237
1238        if let (Some(gh), Some(pr_number)) =
1239            (&gh, task.pr_number.and_then(|n| u64::try_from(n).ok()))
1240        {
1241            match gh.get_pr(&task.repo, pr_number).await {
1242                Ok(pr) => {
1243                    if pr.merged.unwrap_or(false) {
1244                        tracing::info!(
1245                            task_id = task.id,
1246                            repo = %task.repo,
1247                            pr_number,
1248                            "global rebroadcast-blocked sweep: PR already merged — marking task done"
1249                        );
1250                        if let Err(e) = task_manager
1251                            .mark_task_done_by_store_id(
1252                                task.id,
1253                                &task.repo,
1254                                &ExternalId(ext_id.clone()),
1255                            )
1256                            .await
1257                        {
1258                            tracing::warn!(task_id = task.id, repo = %task.repo, err = %e, "global rebroadcast-blocked sweep: failed to mark merged task done");
1259                            continue;
1260                        }
1261                        let fields: &[(&str, serde_json::Value)] = &[
1262                            ("block_reason", serde_json::Value::Null),
1263                            ("needs_review_refires", serde_json::json!(0)),
1264                        ];
1265                        if let Err(e) = store.set_fields(task.id, fields).await {
1266                            tracing::warn!(task_id = task.id, err = %e, "global rebroadcast-blocked sweep: failed to clear fields after marking done");
1267                        }
1268                        continue;
1269                    }
1270                    if pr.state.eq_ignore_ascii_case("closed") {
1271                        tracing::info!(
1272                            task_id = task.id,
1273                            repo = %task.repo,
1274                            pr_number,
1275                            "global rebroadcast-blocked sweep: PR closed unmerged — leaving task blocked, not re-dispatching"
1276                        );
1277                        let new_reason = format!(
1278                            "{REBROADCAST_BLOCK_REASON} (PR #{pr_number} closed unmerged — no further auto-recovery)"
1279                        );
1280                        let fields: &[(&str, serde_json::Value)] =
1281                            &[("block_reason", serde_json::json!(new_reason))];
1282                        // set_fields_silent: this is a terminal annotation, not a real
1283                        // status change — don't bump updated_at so triage queries keyed
1284                        // off it aren't confused by a task that stays blocked forever.
1285                        if let Err(e) = store.set_fields_silent(task.id, fields).await {
1286                            tracing::warn!(task_id = task.id, err = %e, "global rebroadcast-blocked sweep: failed to annotate closed-unmerged block reason");
1287                        }
1288                        continue;
1289                    }
1290                    // PR still open — fall through to the reset-to-NeedsReview path.
1291                }
1292                Err(e) => {
1293                    tracing::warn!(task_id = task.id, repo = %task.repo, pr_number, err = %e, "global rebroadcast-blocked sweep: failed to get PR status — skipping this tick");
1294                    continue;
1295                }
1296            }
1297        }
1298
1299        if !any_routable {
1300            continue;
1301        }
1302
1303        if let Err(e) = task_manager
1304            .update_task_status_by_store_id(
1305                task.id,
1306                &task.repo,
1307                &ExternalId(ext_id.clone()),
1308                Status::NeedsReview,
1309            )
1310            .await
1311        {
1312            tracing::warn!(task_id = task.id, repo = %task.repo, err = %e, "global rebroadcast-blocked sweep: failed to reset task to needs_review — skipping");
1313            continue;
1314        }
1315
1316        let fields: &[(&str, serde_json::Value)] = &[
1317            ("block_reason", serde_json::Value::Null),
1318            ("needs_review_refires", serde_json::json!(0)),
1319        ];
1320        if let Err(e) = store.set_fields(task.id, fields).await {
1321            tracing::warn!(task_id = task.id, err = %e, "global rebroadcast-blocked sweep: failed to clear block_reason/needs_review_refires after status reset");
1322        }
1323
1324        tracing::info!(
1325            task_id = task.id,
1326            repo = %task.repo,
1327            ext_id = %ext_id,
1328            "global rebroadcast-blocked sweep: auto-recovered rebroadcast-blocked task — review agents now available"
1329        );
1330    }
1331
1332    Ok(())
1333}
1334
1335async fn auto_unblock_blocked_tasks(
1336    repo: &str,
1337    task_manager: &Arc<TaskManager>,
1338    store: &Arc<TaskStore>,
1339    dispatching: &Arc<DashMap<String, String>>,
1340    auto_merge_in_flight: &Arc<DashSet<String>>,
1341) -> anyhow::Result<()> {
1342    let blocked = match store.list_by_status(repo, TaskStatus::Blocked).await {
1343        Ok(tasks) => tasks,
1344        Err(e) => {
1345            tracing::warn!(err = %e, "failed to list blocked tasks — skipping auto-unblock this tick");
1346            return Ok(());
1347        }
1348    };
1349
1350    if blocked.is_empty() {
1351        return Ok(());
1352    }
1353
1354    // Batch-load all runs for blocked tasks in a single query instead of N individual queries.
1355    let task_ids: Vec<i64> = blocked.iter().map(|t| t.id).collect();
1356    let mut runs_by_task = match store.get_runs_batch(&task_ids).await {
1357        Ok(map) => map,
1358        Err(e) => {
1359            tracing::warn!(err = %e, "get_runs_batch failed — skipping auto-unblock this tick");
1360            return Ok(());
1361        }
1362    };
1363
1364    // Separate CI-failure blocked tasks from other blocked tasks.
1365    let (ci_failure_tasks, other_blocked): (Vec<_>, Vec<_>) =
1366        blocked.into_iter().partition(is_ci_failure_block);
1367
1368    // Handle CI-failure blocked tasks by verifying PR state.
1369    if !ci_failure_tasks.is_empty() {
1370        let gh = match GhHttp::new() {
1371            Ok(g) => Some(g),
1372            Err(e) => {
1373                tracing::warn!(err = %e, "failed to create GhHttp — skipping CI failure unblock this tick");
1374                None
1375            }
1376        };
1377        for task in ci_failure_tasks {
1378            if ci_failure_unblock_cooldown_elapsed(&task)
1379                && let Some(ref gh) = gh
1380                && let Err(e) = try_unblock_ci_failure_task(
1381                    gh,
1382                    repo,
1383                    task_manager,
1384                    store,
1385                    &task,
1386                    auto_merge_in_flight,
1387                )
1388                .await
1389            {
1390                tracing::warn!(task_id = task.id, err = %e, "CI failure unblock check failed");
1391            }
1392        }
1393    }
1394
1395    // Process regular blocked tasks (those without a block_reason).
1396    for task in other_blocked {
1397        if task.block_reason.is_some() {
1398            continue;
1399        }
1400
1401        let task_id_str = task
1402            .external_id
1403            .clone()
1404            .unwrap_or_else(|| format!("internal:{}", task.id));
1405        let dispatch_key = format!("{repo}/{task_id_str}");
1406        if dispatching.contains_key(&dispatch_key) {
1407            continue;
1408        }
1409
1410        let runs = runs_by_task.remove(&task.id).unwrap_or_default();
1411        let relevant_runs: Vec<_> = runs
1412            .into_iter()
1413            .filter(|r| r.run_type == "agent" || r.run_type == "review")
1414            .collect();
1415        let mut failures = Vec::new();
1416        let mut has_review_failure = false;
1417        for run in relevant_runs.iter().rev() {
1418            if let Some(category) = classify_failure_from_run(run) {
1419                if run.run_type == "review" {
1420                    has_review_failure = true;
1421                }
1422                failures.push(category);
1423            }
1424            if failures.len() >= 3 {
1425                break;
1426            }
1427        }
1428
1429        if failures.is_empty() || !failures.iter().all(|f| f.is_recoverable()) {
1430            continue;
1431        }
1432
1433        // Determine the most recent failure category as the "reason key" for this unblock.
1434        let reason_key = failures.first().map_or("", |f| f.as_str());
1435
1436        // Detect reason changes to reset exponential backoff.
1437        // New reasons bypass old cooldown windows and start fresh attempt counters.
1438        let current_reason = task.auto_unblock_last_reason.clone();
1439        let reason_changed = reason_key != current_reason;
1440
1441        // Compute effective counter: when the reason changes, start from 0;
1442        // otherwise use the accumulated count for exponential backoff.
1443        let cooldown_count = if reason_changed {
1444            0
1445        } else {
1446            task.auto_unblock_count
1447        };
1448
1449        // Gate on attempt count using the effective count — if the reason changed, the
1450        // counter was reset above so cooldown_count is 0 and a new reason always gets a
1451        // fresh attempt regardless of how many times the old reason fired.
1452        if cooldown_count >= 3 {
1453            continue;
1454        }
1455
1456        if !auto_unblock_cooldown_elapsed(cooldown_count, &task.auto_unblock_last_at) {
1457            continue;
1458        }
1459
1460        // Compute the new counter value: when reason changed, cooldown_count is 0,
1461        // so incrementing gives 1 (first attempt for the new reason). When the reason is the
1462        // same, advance from current count for exponential backoff.
1463        let new_count = cooldown_count + 1;
1464        let now = chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string();
1465
1466        let ext_id = task
1467            .external_id
1468            .clone()
1469            .unwrap_or_else(|| format!("internal:{}", task.id));
1470        let new_status = if has_review_failure {
1471            Status::NeedsReview
1472        } else {
1473            Status::New
1474        };
1475        let new_status_str = if has_review_failure {
1476            "needs_review"
1477        } else {
1478            "new"
1479        };
1480
1481        // First update the status; only increment the counter if this succeeds.
1482        // This prevents transient GitHub API errors from exhausting the retry quota.
1483        if let Err(e) = task_manager
1484            .update_task_status(&ExternalId(ext_id), new_status)
1485            .await
1486        {
1487            tracing::warn!(
1488                task_id = task.id,
1489                err = %e,
1490                "auto-unblock failed to update status — counter not incremented"
1491            );
1492            continue;
1493        }
1494
1495        // Status update succeeded; now record the counter increment and clear related fields.
1496        let mut fields: Vec<(&str, serde_json::Value)> = vec![
1497            ("auto_unblock_count", serde_json::json!(new_count)),
1498            ("auto_unblock_last_at", serde_json::json!(now)),
1499            ("auto_unblock_last_reason", serde_json::json!(reason_key)),
1500            ("agent", serde_json::Value::Null),
1501            ("model", serde_json::Value::Null),
1502        ];
1503        fields.push(("no_code_reroutes", serde_json::json!(0)));
1504        fields.push(("no_code_last_agent", serde_json::json!("")));
1505        if has_review_failure {
1506            fields.push(("review_agent_failures", serde_json::json!(0)));
1507            fields.push(("review_invocations", serde_json::json!(0)));
1508            fields.push(("review_resets", serde_json::json!(0)));
1509        }
1510        if let Err(e) = store.set_fields(task.id, &fields).await {
1511            tracing::warn!(task_id = task.id, err = %e, "failed to set auto_unblock fields after successful status update");
1512            // Task is unblocked (correct behavior) but counter hasn't advanced.
1513            // Slightly over-eager retries are better than permanent task stalls.
1514        }
1515
1516        if let Err(e) = store
1517            .append_activity(
1518                task.id,
1519                "auto_unblock",
1520                Some("blocked"),
1521                Some(new_status_str),
1522                None,
1523                None,
1524                Some(&serde_json::json!({
1525                    "reason": reason_key,
1526                    "count": if reason_changed { 1 } else { task.auto_unblock_count + 1 },
1527                    "failures": failures.iter().map(|f| format!("{f:?}")).collect::<Vec<_>>(),
1528                })),
1529            )
1530            .await
1531        {
1532            tracing::warn!(task_id = task.id, err = %e, "failed to log auto_unblock activity");
1533        }
1534
1535        tracing::info!(
1536            task_id = task.id,
1537            failures = ?failures,
1538            review_failure = has_review_failure,
1539            "auto-unblocked task with recoverable failures"
1540        );
1541    }
1542
1543    Ok(())
1544}
1545
1546/// Global sweep for CI-failure blocked tasks from inactive or removed projects.
1547///
1548/// `auto_unblock_blocked_tasks` is called from `sync_tick` and only runs for the
1549/// currently-active repo. Tasks that were blocked for a project that is later removed
1550/// from orch config never get processed and accumulate in the DB indefinitely.
1551///
1552/// This function queries all blocked tasks across all repos and handles the CI-failure
1553/// cases (block_reason LIKE '%CI failure%' or '%CI checks timed out%') so they are
1554/// eventually cleaned up regardless of project activity. Called once per main tick from
1555/// `tick_global_sweeps`.
1556pub(crate) async fn auto_unblock_ci_failure_blocked_tasks_global(
1557    task_manager: &Arc<TaskManager>,
1558    store: &Arc<TaskStore>,
1559    auto_merge_in_flight: &Arc<DashSet<String>>,
1560) -> anyhow::Result<()> {
1561    let all_blocked = match store.list_all_by_status_global(TaskStatus::Blocked).await {
1562        Ok(tasks) => tasks,
1563        Err(e) => {
1564            tracing::warn!(err = %e, "global CI-failure blocked sweep: failed to list blocked tasks");
1565            return Ok(());
1566        }
1567    };
1568
1569    let ci_failure_tasks: Vec<_> = all_blocked
1570        .into_iter()
1571        .filter(is_ci_failure_block)
1572        .collect();
1573    if ci_failure_tasks.is_empty() {
1574        return Ok(());
1575    }
1576
1577    let gh = match GhHttp::new() {
1578        Ok(g) => g,
1579        Err(e) => {
1580            tracing::warn!(err = %e, "global CI-failure blocked sweep: failed to create GhHttp");
1581            return Ok(());
1582        }
1583    };
1584
1585    for task in ci_failure_tasks {
1586        if !ci_failure_unblock_cooldown_elapsed(&task) {
1587            continue;
1588        }
1589        if let Err(e) = try_unblock_ci_failure_task(
1590            &gh,
1591            &task.repo,
1592            task_manager,
1593            store,
1594            &task,
1595            auto_merge_in_flight,
1596        )
1597        .await
1598        {
1599            tracing::warn!(
1600                task_id = task.id,
1601                repo = %task.repo,
1602                err = %e,
1603                "global CI-failure blocked sweep: unblock check failed"
1604            );
1605        }
1606    }
1607
1608    Ok(())
1609}
1610
1611/// Global sweep: re-fire stale NeedsReview tasks and escalate persistently-stuck ones
1612/// to Blocked, across all repos.
1613///
1614/// Two cases where the event-driven subscriber in `subscribers/review.rs` can miss
1615/// a NeedsReview task: all review slots are busy when the event fires, or the
1616/// broadcast receiver lagged and dropped it. Re-publish the NeedsReview status
1617/// after a short idle window so the subscriber can pick it up when a slot is free.
1618/// Using a status re-fire rather than spawning the review agent directly keeps this
1619/// path stateless. The subscriber is still the sole spawner and uses the InReview
1620/// transition as its atomic guard (issue #857)
1621///
1622/// Queries across all repos and writes through the store via `*_by_store_id`
1623/// TaskManager helpers. Called once per main tick from `tick_global_sweeps`
1624pub(crate) async fn refire_and_escalate_stale_needs_review_global(
1625    task_manager: &Arc<TaskManager>,
1626    store: &Arc<TaskStore>,
1627    router: &Router,
1628    dispatching: &Arc<DashMap<String, String>>,
1629    active_repos: &HashSet<String>,
1630) -> anyhow::Result<()> {
1631    const MIN_STALE_NEEDS_REVIEW_MINUTES: i64 = 1;
1632    const MAX_NEEDS_REVIEW_REFIRE_ATTEMPTS: u64 = 5;
1633
1634    let needs_review_tasks = match store
1635        .list_all_by_status_global(TaskStatus::NeedsReview)
1636        .await
1637    {
1638        Ok(tasks) => tasks,
1639        Err(e) => {
1640            tracing::warn!(err = %e, "global NeedsReview refire sweep: failed to list needs_review tasks");
1641            return Ok(());
1642        }
1643    };
1644
1645    if needs_review_tasks.is_empty() {
1646        return Ok(());
1647    }
1648
1649    // Re-firing while every review agent is cooled wastes the counter and can
1650    // escalate a task to Blocked while the subscriber would only bail again
1651    let all_review_agents_cooled = router.healthy_agent_count("review") == 0;
1652    if all_review_agents_cooled {
1653        tracing::info!(
1654            count = needs_review_tasks.len(),
1655            "global NeedsReview refire sweep: no review agent available — skipping this pass"
1656        );
1657        return Ok(());
1658    }
1659
1660    for stored in needs_review_tasks {
1661        // No review subscriber listens for a repo outside the active set
1662        if !active_repos.contains(&stored.repo) {
1663            continue;
1664        }
1665
1666        let external = tasks::store_task_to_external(&stored);
1667        let ext_id = external.id.clone();
1668        let repo = stored.repo.clone();
1669
1670        let age_minutes = match chrono::DateTime::parse_from_rfc3339(&external.updated_at) {
1671            Ok(updated_at) => {
1672                let age = chrono::Utc::now() - updated_at.with_timezone(&chrono::Utc);
1673                if age.num_minutes() < MIN_STALE_NEEDS_REVIEW_MINUTES {
1674                    continue;
1675                }
1676                Some(age.num_minutes())
1677            }
1678            Err(_) => None,
1679        };
1680
1681        // Skip tasks actively being dispatched by their owning repo's engine loop.
1682        let dispatch_key = format!("{}/{}", repo, ext_id.0);
1683        if dispatching.contains_key(&dispatch_key) {
1684            continue;
1685        }
1686
1687        let current_refires = u64::try_from(stored.needs_review_refires).unwrap_or(0);
1688        let new_refires = current_refires + 1;
1689
1690        if new_refires > MAX_NEEDS_REVIEW_REFIRE_ATTEMPTS {
1691            tracing::warn!(
1692                task_id = stored.id,
1693                repo = %repo,
1694                new_refires,
1695                "global NeedsReview refire sweep: escalating to Blocked after repeated refires"
1696            );
1697            let fields = [
1698                ("block_reason", serde_json::json!(REBROADCAST_BLOCK_REASON)),
1699                (
1700                    "last_error",
1701                    serde_json::json!(format!("escalated after {} retries", new_refires)),
1702                ),
1703            ];
1704            if let Err(e) = task_manager
1705                .update_task_status_and_result_by_store_id(
1706                    stored.id,
1707                    &repo,
1708                    &ext_id,
1709                    Status::Blocked,
1710                    &fields,
1711                )
1712                .await
1713            {
1714                tracing::error!(task_id = stored.id, repo = %repo, err = %e, "global NeedsReview refire sweep: escalation to Blocked failed — skipping to avoid silent auto-unblock loop");
1715                continue;
1716            }
1717            if let Err(e) = store::store_increment_by_id(
1718                &Some(Arc::clone(store)),
1719                stored.id,
1720                "needs_review_refires",
1721            )
1722            .await
1723            {
1724                tracing::warn!(
1725                    task_id = stored.id,
1726                    repo = %repo,
1727                    err = %e,
1728                    "global NeedsReview refire sweep: failed to increment needs_review_refires after escalation"
1729                );
1730            }
1731            continue;
1732        }
1733
1734        // Exponential backoff: MIN * 2^(current_refires)
1735        let required_minutes =
1736            MIN_STALE_NEEDS_REVIEW_MINUTES * (1i64 << u32::try_from(current_refires).unwrap_or(0));
1737
1738        let should_fire = match age_minutes {
1739            Some(age) => age >= required_minutes,
1740            None => true,
1741        };
1742
1743        if !should_fire {
1744            continue;
1745        }
1746
1747        if let Err(e) = task_manager
1748            .update_task_status_by_store_id(stored.id, &repo, &ext_id, Status::NeedsReview)
1749            .await
1750        {
1751            tracing::warn!(
1752                task_id = stored.id,
1753                repo = %repo,
1754                err = %e,
1755                "global NeedsReview refire sweep: failed to re-fire NeedsReview event — not incrementing counter"
1756            );
1757            continue;
1758        }
1759
1760        let fired_refires = match store::store_increment_by_id(
1761            &Some(Arc::clone(store)),
1762            stored.id,
1763            "needs_review_refires",
1764        )
1765        .await
1766        {
1767            Ok(v) => v,
1768            Err(e) => {
1769                tracing::warn!(
1770                    task_id = stored.id,
1771                    repo = %repo,
1772                    err = %e,
1773                    "global NeedsReview refire sweep: failed to increment needs_review_refires after re-fire"
1774                );
1775                current_refires + 1
1776            }
1777        };
1778
1779        tracing::info!(
1780            task_id = stored.id,
1781            repo = %repo,
1782            age_minutes,
1783            refires = fired_refires,
1784            required_minutes,
1785            "global NeedsReview refire sweep: re-firing NeedsReview event for stale task"
1786        );
1787    }
1788
1789    Ok(())
1790}
1791
1792/// Sync tick — runs every 45s.
1793///
1794/// Handles less-frequent operations:
1795/// - Cleanup finished worktrees
1796/// - Check for merged PRs → mark tasks done
1797/// - Scan for @mentions
1798#[expect(
1799    clippy::too_many_arguments,
1800    reason = "the engine passes its shared handles one by one"
1801)]
1802pub(crate) async fn sync_tick(
1803    backend: &Arc<dyn ExternalBackend>,
1804    tmux: &Arc<TmuxManager>,
1805    repo: &str,
1806    config: &EngineConfig,
1807    router: &Arc<RwLock<Router>>,
1808    task_manager: &Arc<TaskManager>,
1809    store: &Arc<TaskStore>,
1810    dispatching: &Arc<DashMap<String, String>>,
1811    auto_merge_in_flight: &Arc<DashSet<String>>,
1812) -> anyhow::Result<()> {
1813    // One node polls GitHub and decides for the whole cluster
1814    if !store.is_leader() {
1815        return Ok(());
1816    }
1817
1818    tracing::debug!("sync tick");
1819
1820    let mut phases: Vec<(&str, u128)> = Vec::new();
1821    let mut at = Instant::now();
1822
1823    // Global GitHub 5xx circuit breaker — skip non-critical background work
1824    // during sustained GitHub outages to reduce churn and wasted API calls.
1825    let gh_circuit_open = is_github_circuit_open();
1826
1827    if gh_circuit_open {
1828        let remaining = github_circuit_remaining_secs();
1829        tracing::debug!(
1830            remaining_secs = remaining,
1831            "GitHub 5xx circuit breaker open — skipping non-critical sync work"
1832        );
1833        // Persist circuit breaker state as a metric for operators.
1834        // This is critical for avoiding thundering herd after restart during outages.
1835        if let Err(e) =
1836            try_kv_set_prefer_store(Some(store), "metrics:orch.github_5xx_circuit.open", "1").await
1837        {
1838            tracing::error!(err = %e, "failed to persist circuit breaker state — may cause thundering herd after restart");
1839        }
1840    } else if let Err(e) =
1841        try_kv_set_prefer_store(Some(store), "metrics:orch.github_5xx_circuit.open", "0").await
1842    {
1843        tracing::error!(err = %e, "failed to clear circuit breaker state");
1844    }
1845
1846    // 0. Ingest all active external tasks into the unified store.
1847    //    This ensures the store has data for tasks created before dual-write was added.
1848    if !gh_circuit_open {
1849        if let Err(e) = ingest_external_tasks(backend, repo, store).await {
1850            tracing::debug!(err = %e, "external task ingest failed");
1851        }
1852        phases.push(("ingest", at.elapsed().as_millis()));
1853        at = Instant::now();
1854
1855        // Reconcile existing board members too: labels can change without a
1856        // local task transition, and a human/automation may move a card. The
1857        // backend only writes drifted fields through the shared GitHub client.
1858        if let Err(e) = backend.reconcile_project_statuses().await {
1859            tracing::warn!(err = %e, "project status reconciliation failed");
1860        }
1861        phases.push(("board_reconcile", at.elapsed().as_millis()));
1862    }
1863    at = Instant::now();
1864
1865    // 1. Cleanup worktrees for done tasks (background — must not block routing/dispatch)
1866    if !gh_circuit_open {
1867        let backend = Arc::clone(backend);
1868        let repo = repo.to_string();
1869        let task_manager = Arc::clone(task_manager);
1870        let store = Arc::clone(store);
1871        tokio::spawn(async move {
1872            if let Err(e) = cleanup_done_worktrees(&backend, &repo, &task_manager, &store).await {
1873                tracing::warn!(err = %e, "worktree cleanup failed");
1874            }
1875            if let Err(e) = prune_old_run_output(&store).await {
1876                tracing::warn!(err = %e, "task run pruning failed");
1877            }
1878        });
1879    }
1880
1881    let (in_review_tasks, needs_review_tasks, blocked_review_candidates) =
1882        match prefetch_review_tasks(store, repo).await {
1883            Ok(tasks) => tasks,
1884            Err(e) => {
1885                tracing::warn!(err = %e, "failed to prefetch review tasks");
1886                (vec![], vec![], vec![])
1887            }
1888        };
1889    phases.push(("prefetch_review", at.elapsed().as_millis()));
1890    at = Instant::now();
1891
1892    if !gh_circuit_open {
1893        let mention_fallback = chrono::Utc::now() - chrono::Duration::hours(24);
1894        let mention_since = kv_get_prefer_store(Some(store), "mentions_last_checked")
1895            .await
1896            .unwrap_or_else(|| mention_fallback.format("%Y-%m-%dT%H:%M:%SZ").to_string());
1897        let shared_since = mention_since.clone();
1898
1899        let cached_router_config = router.read().await.config.clone();
1900        // Include blocked tasks that have branch/pr info when checking for merged PRs.
1901        let mut combined_needs = needs_review_tasks.clone();
1902        combined_needs.extend(blocked_review_candidates.clone());
1903
1904        let (merge_result, comments_result, review_result) = tokio::join!(
1905            check_merged_prs(
1906                backend,
1907                repo,
1908                store,
1909                task_manager,
1910                &in_review_tasks,
1911                &combined_needs,
1912            ),
1913            fetch_comments_since(backend, &shared_since),
1914            async {
1915                let gh = match GhHttp::new() {
1916                    Ok(g) => g,
1917                    Err(e) => {
1918                        tracing::warn!(err = %e, "failed to create GitHub HTTP client for review poll");
1919                        return Ok(());
1920                    }
1921                };
1922                Box::pin(review_open_prs(
1923                    backend,
1924                    repo,
1925                    config,
1926                    task_manager,
1927                    store,
1928                    dispatching,
1929                    auto_merge_in_flight,
1930                    &in_review_tasks,
1931                    &gh,
1932                    &cached_router_config,
1933                ))
1934                .await
1935            }
1936        );
1937
1938        if let Err(e) = merge_result {
1939            tracing::warn!(err = %e, "PR merge check failed");
1940        }
1941
1942        match comments_result {
1943            Ok(comments) => {
1944                let filtered = filter_mentions_by_since(&comments, &mention_since);
1945
1946                if let Err(e) =
1947                    scan_comments(backend, Some(store), repo, task_manager, Some(&filtered)).await
1948                {
1949                    tracing::warn!(err = %e, "comment scan failed");
1950                }
1951            }
1952            Err(e) => {
1953                tracing::warn!(err = %e, "comment fetch failed");
1954            }
1955        }
1956
1957        if let Err(e) = review_result {
1958            tracing::warn!(err = %e, "PR review failed");
1959        }
1960
1961        if let Err(e) = ci_bootstrap::apply_pending(store, task_manager, repo).await {
1962            tracing::warn!(err = %e, "CI bootstrap ruleset apply failed");
1963        }
1964
1965        phases.push(("merge_comments_review", at.elapsed().as_millis()));
1966    }
1967    at = Instant::now();
1968
1969    // 5. Stale InReview recovery lives in `tick_recover_stuck_tasks`, the single owner of an
1970    // in_review task whose review session is gone or hung. It ran here too, after one minute,
1971    // which beat a finished review's result to the engine and dropped it as stale
1972
1973    // 5c. Detect stale InProgress tasks and recover them.
1974    //
1975    // Similar to stale InReview detection, we check for InProgress tasks whose
1976    // agent tmux session has died (crash, OOM, etc.) and reset them to Routed
1977    // so they can be re-dispatched.
1978    let in_progress_tasks: Vec<_> = match store.list_by_status(repo, TaskStatus::InProgress).await {
1979        Ok(tasks) => tasks
1980            .into_iter()
1981            .map(|stored| {
1982                let external = tasks::store_task_to_external(&stored);
1983                (stored, external)
1984            })
1985            .collect(),
1986        Err(e) => {
1987            tracing::warn!(err = %e, "failed to list in_progress tasks — skipping stale InProgress check");
1988            vec![]
1989        }
1990    };
1991
1992    if !in_progress_tasks.is_empty() {
1993        // Fetch all live tmux sessions once instead of one subprocess call per task.
1994        let live_sessions: HashSet<String> = tmux
1995            .list_sessions()
1996            .await
1997            .unwrap_or_default()
1998            .into_iter()
1999            .map(|s| s.name)
2000            .collect();
2001
2002        for (_stored, external) in &in_progress_tasks {
2003            // A task is only considered stale if it has been in InProgress for > 5 minutes.
2004            const MIN_STALE_MINUTES: i64 = 5;
2005
2006            let task_id = &external.id.0;
2007
2008            // Skip tasks currently being processed by the main tick (dispatch + agent flow).
2009            let dispatch_key = format!("{repo}/{task_id}");
2010            {
2011                if dispatching.contains_key(&dispatch_key) {
2012                    tracing::debug!(
2013                        task_id = %task_id,
2014                        "task locked by dispatch flow, skipping stale check"
2015                    );
2016                    continue;
2017                }
2018            }
2019
2020            // Skip tasks that just transitioned to InProgress — allow time for the
2021            // agent to start its tmux session before treating it as stale.
2022            match chrono::DateTime::parse_from_rfc3339(&external.updated_at) {
2023                Ok(updated_at) => {
2024                    let age = chrono::Utc::now() - updated_at.with_timezone(&chrono::Utc);
2025                    if age.num_minutes() < MIN_STALE_MINUTES {
2026                        tracing::debug!(
2027                            task_id = %task_id,
2028                            age_seconds = age.num_seconds(),
2029                            "InProgress task is too young to be considered stale, skipping"
2030                        );
2031                        continue;
2032                    }
2033                }
2034                Err(e) => {
2035                    tracing::warn!(
2036                        task_id = %task_id,
2037                        ts = %external.updated_at,
2038                        err = %e,
2039                        "invalid updated_at timestamp — treating task as potentially stale"
2040                    );
2041                }
2042            }
2043
2044            let agent_session = tmux.session_name(repo, task_id);
2045            if live_sessions.contains(&agent_session) {
2046                // Agent is still alive — skip.
2047                continue;
2048            }
2049
2050            // No agent session exists. The agent is dead regardless of
2051            // session tracking — hard restart, crash, or flag not persisted.
2052            // Reset to Routed so the dispatcher can re-spawn it.
2053            {
2054                tracing::warn!(
2055                    task_id = %task_id,
2056                    session = %agent_session,
2057                    "InProgress task has no active agent session — resetting to Routed"
2058                );
2059                match task_manager
2060                    .update_task_status_if(&external.id, Status::Routed, Status::InProgress)
2061                    .await
2062                {
2063                    Err(e) => {
2064                        tracing::error!(task_id = %task_id, err = %e, "failed to reset stale InProgress task — task may be stuck in InProgress indefinitely");
2065                    }
2066                    Ok(false) => {
2067                        tracing::debug!(
2068                            task_id = %task_id,
2069                            "stale InProgress reset skipped — task already transitioned (concurrent Done/Blocked/NeedsReview)"
2070                        );
2071                    }
2072                    Ok(true) => {
2073                        // Successfully reset to Routed
2074                    }
2075                }
2076            }
2077        }
2078    }
2079
2080    phases.push(("stale_in_progress", at.elapsed().as_millis()));
2081    at = Instant::now();
2082    // 5c. Auto-unblock tasks with recoverable failures.
2083    if let Err(e) =
2084        auto_unblock_blocked_tasks(repo, task_manager, store, dispatching, auto_merge_in_flight)
2085            .await
2086    {
2087        tracing::warn!(err = %e, "auto-unblock failed");
2088    }
2089
2090    phases.push(("auto_unblock", at.elapsed().as_millis()));
2091
2092    engine::log_phase_timings("sync", repo, &phases);
2093    Ok(())
2094}
2095
2096/**
2097 * Skill repos and agent health are the same whichever project asks, so they run once per
2098 * sync pass rather than once per project. Called after every project's `sync_tick`
2099 **/
2100pub(crate) async fn sync_shared_tail(router: &Arc<RwLock<Router>>, store: Option<&Arc<TaskStore>>) {
2101    let Some(store) = store else {
2102        return;
2103    };
2104    if !store.is_leader() {
2105        return;
2106    }
2107    match skills_sync().await {
2108        Ok(()) => {
2109            // Clone the Arc handle before releasing the read guard so the read lock is not
2110            // held across the async invalidation call
2111            let llm = router.read().await.llm_router_handle();
2112            llm.invalidate_skills_catalog().await;
2113        }
2114        Err(e) => {
2115            tracing::warn!(err = %e, "skills sync failed");
2116        }
2117    }
2118
2119    // Pre-emptive health check: refresh degraded-agent flags from the rate_limits table.
2120    // Clone what is needed out of the router first, so the write lock is not starved for the
2121    // duration of the DB query
2122    let (available_agents, config) = {
2123        let r = router.read().await;
2124        (r.available_agents.clone(), r.config.clone())
2125    };
2126    let model_checker = |agent: &str| -> bool {
2127        for comp in &["simple", "medium", "complex", "review"] {
2128            if config.has_available_model_for_complexity(agent, comp) {
2129                return true;
2130            }
2131        }
2132        false
2133    };
2134    refresh_degraded_agents(
2135        store,
2136        &available_agents,
2137        &model_checker,
2138        router_config::health_check_window_hours(),
2139        router_config::degraded_rate_limit_threshold(),
2140    )
2141    .await;
2142
2143    emit_degraded_agents_if_needed(&available_agents, &config, Some(store)).await;
2144}
2145
2146/// Scan for @mentions and handle them.
2147///
2148/// Checks recent issue comments for @orch (and legacy @orchestrator) mentions.
2149/// Unified comment scanner — handles both @mentions and slash commands in one pass.
2150///
2151/// For each comment, classifies it via [`classify_comment`] and dispatches:
2152/// - Slash commands → validate + execute (with sentinel task if it was an @mention)
2153/// - @mentions without commands → create internal task for agent to respond
2154/// - Everything else → skip
2155async fn scan_comments(
2156    backend: &Arc<dyn ExternalBackend>,
2157    store: Option<&Arc<TaskStore>>,
2158    repo: &str,
2159    task_manager: &Arc<TaskManager>,
2160    prefetched_comments: Option<&[Mention]>,
2161) -> anyhow::Result<()> {
2162    let current_user = match backend.get_authenticated_user().await {
2163        Ok(Some(u)) => format!("@{u}"),
2164        Ok(None) => {
2165            tracing::debug!("backend does not support user identity, skipping comment scan");
2166            return Ok(());
2167        }
2168        Err(e) => {
2169            tracing::warn!(err = %e, "failed to get current user, skipping comment scan");
2170            return Ok(());
2171        }
2172    };
2173
2174    let comments = if let Some(c) = prefetched_comments {
2175        c.to_vec()
2176    } else {
2177        let fallback = chrono::Utc::now() - chrono::Duration::hours(24);
2178        let since_str = match kv_get_prefer_store(store, "mentions_last_checked").await {
2179            Some(ts) => ts,
2180            None => fallback.format("%Y-%m-%dT%H:%M:%SZ").to_string(),
2181        };
2182        match backend.get_mentions(&since_str).await {
2183            Ok(m) => m,
2184            Err(e) => {
2185                tracing::warn!(err = %e, "failed to get comments");
2186                return Ok(());
2187            }
2188        }
2189    };
2190
2191    // Dedup set: mention tasks already created (by source_id).
2192    let existing_mentions: HashSet<String> = if let Some(s) = store {
2193        match s.list_source_ids_by_source(repo, "mention", 30).await {
2194            Ok(ids) => ids.into_iter().collect(),
2195            Err(e) => {
2196                tracing::warn!(err = %e, "failed to load mention source IDs — skipping to prevent duplicates");
2197                return Ok(());
2198            }
2199        }
2200    } else {
2201        HashSet::new()
2202    };
2203
2204    // Lazily create GhHttp — only allocate when we actually need to make GitHub
2205    // API calls.  This avoids network initialization in test environments where
2206    // the code path never reaches an API call.
2207    let gh_cell: OnceCell<GhHttp> = OnceCell::new();
2208    let get_gh = || async { gh_cell.get_or_try_init(|| async { GhHttp::new() }).await };
2209
2210    // Pre-classify all comments up front so we can batch the is_pull_request
2211    // checks for CreateMentionTask entries before entering the main loop.
2212    let actions: Vec<CommentAction> = comments
2213        .iter()
2214        .map(|c| {
2215            classify_comment(
2216                &c.body,
2217                c.issue_url.as_deref(),
2218                &current_user,
2219                existing_mentions.contains(&c.id),
2220            )
2221        })
2222        .collect();
2223
2224    // Collect (comment_index, issue_num) for all CreateMentionTask entries that
2225    // need an is_pull_request check, then resolve them all concurrently with
2226    // join_all to avoid N × RTT serial API calls inside the loop.
2227    let pr_check_inputs: Vec<(usize, String)> = actions
2228        .iter()
2229        .enumerate()
2230        .filter_map(|(i, action)| {
2231            if let CommentAction::CreateMentionTask {
2232                issue_num: Some(num),
2233            } = action
2234            {
2235                Some((i, num.clone()))
2236            } else {
2237                None
2238            }
2239        })
2240        .collect();
2241
2242    // Only create GhHttp if we actually have PR checks to perform.
2243    let is_pr_values: Vec<bool> = if pr_check_inputs.is_empty() {
2244        Vec::new()
2245    } else {
2246        let gh = get_gh().await?;
2247        future::join_all(
2248            pr_check_inputs
2249                .iter()
2250                .map(|(_, num)| async move { gh.is_pull_request(repo, num).await }),
2251        )
2252        .await
2253    };
2254
2255    // Build a lookup map: comment_index → is_pr.
2256    let is_pr_map: HashMap<usize, bool> = pr_check_inputs
2257        .iter()
2258        .map(|(i, _)| *i)
2259        .zip(is_pr_values)
2260        .collect();
2261
2262    let mut cursor = MentionCursor::default();
2263
2264    for (comment_idx, (comment, action)) in comments.iter().zip(actions).enumerate() {
2265        match action {
2266            CommentAction::Skip => {
2267                cursor.advance(&comment.created_at);
2268            }
2269
2270            CommentAction::ExecuteCommand { command, issue_num } => {
2271                let store_opt: Option<Arc<dyn CommandStoreOps>> =
2272                    store.map(|s| Arc::clone(s) as Arc<dyn CommandStoreOps>);
2273                let gh = get_gh().await?;
2274                let outcome = validate_and_run_command(
2275                    backend,
2276                    gh,
2277                    repo,
2278                    &issue_num,
2279                    &command,
2280                    &comment.author,
2281                    &store_opt,
2282                    task_manager,
2283                )
2284                .await;
2285                match outcome {
2286                    // Validation hit a transient GitHub error: nothing ran and nothing
2287                    // was posted. Stop the cursor so the next tick retries this comment.
2288                    CommandOutcome::FetchFailed | CommandOutcome::CollaboratorCheckFailed => {
2289                        tracing::warn!(
2290                            comment_id = %comment.id,
2291                            command = %command,
2292                            "deferring slash command after a transient GitHub error"
2293                        );
2294                        cursor.block_on_gap();
2295                        break;
2296                    }
2297                    // The outcome comment (success or error) was already posted to
2298                    // GitHub, so re-processing would create duplicates.
2299                    CommandOutcome::Executed { .. }
2300                    | CommandOutcome::NotOpen { .. }
2301                    | CommandOutcome::NotCollaborator { .. } => cursor.advance(&comment.created_at),
2302                }
2303            }
2304
2305            CommentAction::ExecuteCommandForMention { command, .. } => {
2306                if let Err(e) = backend.acknowledge_mention(&comment.id).await {
2307                    tracing::debug!(err = %e, mention_id = %comment.id, "failed to acknowledge mention");
2308                    // Do NOT advance the cursor or call handle_slash_command — the GitHub
2309                    // reaction was never posted. Retry on next tick once the network
2310                    // glitch has resolved.
2311                } else {
2312                    // Parent lookup failure propagates via `?` — scan_comments returns early
2313                    // without advancing the cursor so the mention is retried on next sync.
2314                    let gh = get_gh().await?;
2315                    let advanced_safely = handle_slash_command(
2316                        backend,
2317                        store,
2318                        repo,
2319                        task_manager,
2320                        gh,
2321                        comment,
2322                        &command,
2323                        &mut cursor,
2324                    )
2325                    .await?;
2326                    if !advanced_safely {
2327                        cursor.block_on_gap();
2328                        break;
2329                    }
2330                }
2331            }
2332
2333            CommentAction::CreateMentionTask { issue_num } => {
2334                if let Err(e) = backend.acknowledge_mention(&comment.id).await {
2335                    tracing::debug!(err = %e, mention_id = %comment.id, "failed to acknowledge mention");
2336                    // Do NOT record the mention task or advance the cursor — the GitHub
2337                    // reaction was never posted. Retry on next tick once the network
2338                    // glitch has resolved.
2339                } else {
2340                    // Use the pre-fetched value — no serial API call here.
2341                    // issue_num may be None (e.g. bot mentioned with no issue URL), in
2342                    // which case is_pr_map has no entry; default to false (the None arm
2343                    // of the downstream match ignores is_pr entirely).
2344                    let is_pr = is_pr_map.get(&comment_idx).copied().unwrap_or(false);
2345
2346                    let (title, task_body) = match (&issue_num, is_pr) {
2347                        (Some(num), false) => (
2348                            format!("Respond to mention by @{} on task #{num}", comment.author),
2349                            format!(
2350                                "Mention by @{} on GitHub issue #{num}:\n\n{}\n\n---\nThis mention was posted on issue #{num}. Review the full issue to understand the context before responding.",
2351                                comment.author, comment.body
2352                            ),
2353                        ),
2354                        (Some(num), true) => (
2355                            format!("Respond to mention by @{} on PR #{num}", comment.author),
2356                            format!(
2357                                "Mention by @{} on GitHub PR #{num}:\n\n{}\n\n---\nThis mention was posted on PR #{num}. Review the full PR (diff, description, and comments) to understand the context before responding.",
2358                                comment.author, comment.body
2359                            ),
2360                        ),
2361                        (None, _) => (
2362                            format!("Respond to mention by @{}", comment.author),
2363                            format!("Mention by @{}:\n\n{}", comment.author, comment.body),
2364                        ),
2365                    };
2366
2367                    if let Some(s) = store {
2368                        // On DB lookup failure, do NOT record the mention task or advance
2369                        // the cursor — the mention will be retried on next sync.
2370                        let parent_id = match issue_num {
2371                            Some(ref num) => {
2372                                match resolve_mention_parent_id(s, repo, num, is_pr).await {
2373                                    Ok(id) => id,
2374                                    Err(e) => {
2375                                        tracing::warn!(
2376                                            mention_id = %comment.id,
2377                                            err = %e,
2378                                            "deferring mention task due to parent lookup failure"
2379                                        );
2380                                        return Err(anyhow::anyhow!(
2381                                            "deferring mention {} due to parent lookup failure",
2382                                            comment.id
2383                                        ));
2384                                    }
2385                                }
2386                            }
2387                            None => None,
2388                        };
2389                        if !record_mention_task(
2390                            s,
2391                            repo,
2392                            comment,
2393                            &title,
2394                            &task_body,
2395                            parent_id,
2396                            &mut cursor,
2397                        )
2398                        .await
2399                        {
2400                            cursor.block_on_gap();
2401                            break;
2402                        }
2403                    }
2404                }
2405            }
2406        }
2407    }
2408
2409    if let Some(ts) = cursor.into_last_safe_ts() {
2410        kv_set_prefer_store(store, "mentions_last_checked", &ts).await;
2411    }
2412
2413    Ok(())
2414}
2415
2416/// Sync skill repositories from config.
2417///
2418/// Reads the `skills:` list from config and clones/pulls each repository
2419/// to `~/.orch/skills/{repo}/`. This keeps skill documentation up-to-date
2420/// for agents.
2421async fn skills_sync() -> anyhow::Result<()> {
2422    let skills = match config::get_skills() {
2423        Ok(s) => s,
2424        Err(e) => {
2425            tracing::debug!(err = %e, "no skills configured");
2426            return Ok(());
2427        }
2428    };
2429
2430    if skills.is_empty() {
2431        tracing::debug!("no skills configured, skipping sync");
2432        return Ok(());
2433    }
2434
2435    let skills_base = home::skills_dir()?;
2436    let git_timeout = Duration::from_secs(60);
2437
2438    // Run all skill git operations concurrently — each is fully independent.
2439    // Wall-clock time drops from N × timeout to max(timeout).
2440    let futs: Vec<_> = skills
2441        .into_iter()
2442        .map(|skill| sync_one_skill(skill, skills_base.clone(), git_timeout))
2443        .collect();
2444    future::join_all(futs).await;
2445
2446    Ok(())
2447}
2448
2449/// Sync a single skill repository: `git pull` if it exists, `git clone` otherwise.
2450///
2451/// All errors are logged via `tracing::warn!` so that one failing skill does not
2452/// prevent the others from being updated.
2453async fn sync_one_skill(skill: config::SkillConfig, skills_base: PathBuf, git_timeout: Duration) {
2454    // Validate repo format to prevent path traversal
2455    if skill.repo.contains("..") || skill.repo.matches('/').count() != 1 {
2456        tracing::warn!(repo = %skill.repo, "invalid skill repo format, expected 'owner/repo'");
2457        return;
2458    }
2459
2460    let repo_dir = skills_base.join(&skill.repo);
2461    let repo_url = format!("https://github.com/{}.git", skill.repo);
2462
2463    // Use async metadata check instead of `Path::exists()` to avoid
2464    // performing blocking syscall on the reactor thread.
2465    let repo_exists = fs::metadata(&repo_dir).await.is_ok_and(|m| m.is_dir());
2466
2467    if repo_exists {
2468        // Pull latest changes with timeout
2469        tracing::debug!(repo = %skill.repo, "pulling skill repo");
2470        let pull_result = time::timeout(
2471            git_timeout,
2472            git_ops::git()
2473                .await
2474                .args(["pull", "--ff-only", "--prune"])
2475                .current_dir(&repo_dir)
2476                .output_with_context(),
2477        )
2478        .await;
2479
2480        match pull_result {
2481            Ok(Ok(output)) if !output.status.success() => {
2482                let stderr = String::from_utf8_lossy(&output.stderr);
2483                tracing::warn!(repo = %skill.repo, err = %stderr, "git pull failed");
2484            }
2485            Ok(Ok(_)) => {
2486                tracing::debug!(repo = %skill.repo, "skill repo updated");
2487            }
2488            Ok(Err(e)) => {
2489                tracing::warn!(repo = %skill.repo, err = %e, "git pull error");
2490            }
2491            Err(_) => {
2492                tracing::warn!(repo = %skill.repo, "git pull timed out after 60s");
2493            }
2494        }
2495    } else {
2496        // Clone the repository (shallow for efficiency)
2497        tracing::debug!(repo = %skill.repo, "cloning skill repo");
2498        let Some(parent) = repo_dir.parent() else {
2499            tracing::warn!(repo = %skill.repo, "skill repo path has no parent directory");
2500            return;
2501        };
2502        if let Err(e) = fs::create_dir_all(parent).await {
2503            tracing::warn!(repo = %skill.repo, err = %e, "failed to create parent dir for skill");
2504            return;
2505        }
2506        let Some(repo_dir_str) = repo_dir.to_str() else {
2507            tracing::warn!(repo = %skill.repo, "skill repo path is not valid UTF-8");
2508            return;
2509        };
2510
2511        let clone_result = time::timeout(
2512            git_timeout,
2513            git_ops::git()
2514                .await
2515                .args(["clone", "--depth", "1", &repo_url, repo_dir_str])
2516                .output_with_context(),
2517        )
2518        .await;
2519
2520        match clone_result {
2521            Ok(Ok(output)) if !output.status.success() => {
2522                let stderr = String::from_utf8_lossy(&output.stderr);
2523                tracing::warn!(repo = %skill.repo, err = %stderr, "git clone failed");
2524                // Clean up partial clone to allow retry on next tick
2525                let _ = fs::remove_dir_all(&repo_dir).await;
2526            }
2527            Ok(Ok(_)) => {
2528                tracing::info!(repo = %skill.repo, "skill repo cloned");
2529            }
2530            Ok(Err(e)) => {
2531                tracing::warn!(repo = %skill.repo, err = %e, "git clone error");
2532                let _ = fs::remove_dir_all(&repo_dir).await;
2533            }
2534            Err(_) => {
2535                tracing::warn!(repo = %skill.repo, "git clone timed out after 60s");
2536                let _ = fs::remove_dir_all(&repo_dir).await;
2537            }
2538        }
2539    }
2540}
2541
2542/// Per-agent degradation detail collected during the alerting check.
2543struct DegradedAgentDetail {
2544    /// Agent name (e.g. "claude", "codex").
2545    agent: String,
2546    /// Reason the agent is degraded: the stored cooldown reason when the agent
2547    /// itself is in cooldown, or `"no_available_model"` when all model pools
2548    /// are individually cooled.
2549    reason: String,
2550    /// Models that are individually in cooldown for this agent, deduped across
2551    /// all complexity tiers.
2552    cooled_models: Vec<String>,
2553}
2554
2555/// Per-agent pool health used for stale model pool detection.
2556struct AgentPoolHealth {
2557    agent: String,
2558    cooled_models: Vec<String>,
2559    persistent_failed_models: Vec<String>,
2560    total_configured_models: usize,
2561}
2562
2563/// Inspect router & cooldown state and emit metrics every tick.
2564///
2565/// - `metrics:orch.agents_degraded.count` is written on every call with the
2566///   current number of degraded agents (0 when healthy).
2567/// - When `count >= 3` a WARN log is emitted with full dimensions (agent
2568///   names, cooled models, cooldown reasons) and a dedicated alert metric
2569///   `metrics:orch.agents_degraded.alert` is set to `"1"`. The alert is
2570///   cleared to `"0"` when the count drops below the threshold.
2571pub(crate) async fn emit_degraded_agents_if_needed(
2572    available_agents: &[String],
2573    config: &RouterConfig,
2574    store: Option<&Arc<TaskStore>>,
2575) {
2576    // Edge-triggered: only log when the affected-agents set changes, so a
2577    // persistent condition doesn't flood the log every sync tick.
2578    const STALE_AGENTS_KV: &str = "metrics:orch.model_pool_stale_persistent.agents";
2579
2580    // Complexity tiers to check for configured models.
2581    const COMPLEXITIES: &[&str] = &["simple", "medium", "complex", "review"];
2582    const HEAVILY_COOLED_RATIO_THRESHOLD: f64 = 0.5;
2583
2584    let mut details: Vec<DegradedAgentDetail> = Vec::new();
2585    let mut pool_health: Vec<AgentPoolHealth> = Vec::new();
2586
2587    for agent in available_agents {
2588        let agent_in_cd = is_agent_in_cooldown(agent);
2589
2590        // Collect individually cooled models across all complexity tiers (deduped).
2591        let mut cooled_models: Vec<String> = Vec::new();
2592        let mut persistent_failed_models: Vec<String> = Vec::new();
2593        let mut seen_models = HashSet::new();
2594        for comp in COMPLEXITIES {
2595            if let Some(pool) = config.model_pool_for_complexity(agent, comp) {
2596                for model in pool {
2597                    if seen_models.insert(model.clone()) && is_model_in_cooldown(agent, &model) {
2598                        let key = format!("{agent}:{model}");
2599                        if matches!(
2600                            cooldown_reason(&key).as_deref(),
2601                            Some("persistent_model_error" | "model_not_found")
2602                        ) {
2603                            persistent_failed_models.push(model.clone());
2604                        }
2605                        cooled_models.push(model);
2606                    }
2607                }
2608            }
2609        }
2610        let total_configured_models = seen_models.len();
2611        pool_health.push(AgentPoolHealth {
2612            agent: agent.clone(),
2613            cooled_models: cooled_models.clone(),
2614            persistent_failed_models: persistent_failed_models.clone(),
2615            total_configured_models,
2616        });
2617
2618        // An agent is degraded if it is in agent-level cooldown OR has no
2619        // available (non-cooled) model across any complexity tier.
2620        let has_model = COMPLEXITIES
2621            .iter()
2622            .any(|comp| config.has_available_model_for_complexity(agent, comp));
2623
2624        if agent_in_cd || !has_model {
2625            let reason = if agent_in_cd {
2626                cooldown_reason(agent).unwrap_or_else(|| "agent_cooldown".to_string())
2627            } else {
2628                "no_available_model".to_string()
2629            };
2630            details.push(DegradedAgentDetail {
2631                agent: agent.clone(),
2632                reason,
2633                cooled_models,
2634            });
2635        }
2636    }
2637
2638    let count = details.len();
2639
2640    // Always persist the count metric so operators can scrape it every tick,
2641    // including when the count is 0.
2642    if let Err(e) = try_kv_set_prefer_store(
2643        store,
2644        "metrics:orch.agents_degraded.count",
2645        &count.to_string(),
2646    )
2647    .await
2648    {
2649        tracing::warn!(err = %e, "failed to persist degraded agents count metric");
2650    }
2651
2652    if count >= 3 {
2653        // Build structured dimension strings for the log.
2654        let degraded_agents: Vec<&str> = details.iter().map(|d| d.agent.as_str()).collect();
2655        let cooled_models_dim: Vec<String> = details
2656            .iter()
2657            .filter(|d| !d.cooled_models.is_empty())
2658            .map(|d| format!("{}:[{}]", d.agent, d.cooled_models.join(",")))
2659            .collect();
2660        let reasons_dim: Vec<String> = details
2661            .iter()
2662            .map(|d| format!("{}={}", d.agent, d.reason))
2663            .collect();
2664
2665        tracing::warn!(
2666            degraded_count = count,
2667            degraded_agents = ?degraded_agents,
2668            cooled_models = %cooled_models_dim.join("; "),
2669            cooldown_reasons = %reasons_dim.join("; "),
2670            "multi-agent degradation detected"
2671        );
2672
2673        // Dedicated alert metric: "1" while the threshold is exceeded.
2674        if let Err(e) =
2675            try_kv_set_prefer_store(store, "metrics:orch.agents_degraded.alert", "1").await
2676        {
2677            tracing::warn!(err = %e, "failed to persist degraded agents alert metric");
2678        }
2679    } else if let Err(e) =
2680        try_kv_set_prefer_store(store, "metrics:orch.agents_degraded.alert", "0").await
2681    {
2682        // Clear the alert metric when healthy.
2683        tracing::warn!(err = %e, "failed to clear degraded agents alert metric");
2684    }
2685
2686    // Additional operator signal for dead/stale model pools:
2687    // alert when any agent has at least 50% of configured models cooled and at
2688    // least one persistent model failure marker in cooldown reason.
2689    let heavily_cooled_agents: Vec<String> = pool_health
2690        .iter()
2691        .filter_map(|d| {
2692            if d.total_configured_models == 0 || d.persistent_failed_models.is_empty() {
2693                return None;
2694            }
2695            #[expect(
2696                clippy::cast_precision_loss,
2697                reason = "model counts are far below 2^52"
2698            )]
2699            let ratio = d.cooled_models.len() as f64 / d.total_configured_models as f64;
2700            (ratio >= HEAVILY_COOLED_RATIO_THRESHOLD).then(|| {
2701                format!(
2702                    "{}:{}/{}:{}",
2703                    d.agent,
2704                    d.cooled_models.len(),
2705                    d.total_configured_models,
2706                    d.persistent_failed_models.join(",")
2707                )
2708            })
2709        })
2710        .collect();
2711
2712    let current_dim = heavily_cooled_agents.join(",");
2713    let prev_dim = kv_get_prefer_store(store, STALE_AGENTS_KV)
2714        .await
2715        .unwrap_or_default();
2716
2717    if heavily_cooled_agents.is_empty() {
2718        if !prev_dim.is_empty() {
2719            tracing::info!("agent model pool stale condition cleared");
2720        }
2721        if let Err(e) = try_kv_set_prefer_store(store, STALE_AGENTS_KV, "").await {
2722            tracing::warn!(err = %e, "failed to clear stale model pool agents state");
2723        }
2724        if let Err(e) =
2725            try_kv_set_prefer_store(store, "metrics:orch.model_pool_stale_persistent.alert", "0")
2726                .await
2727        {
2728            tracing::warn!(err = %e, "failed to clear stale model pool alert metric");
2729        }
2730    } else {
2731        if current_dim != prev_dim {
2732            tracing::warn!(
2733                affected_agents = ?heavily_cooled_agents,
2734                "agent model pool appears stale: persistent model failures in heavily cooled pool"
2735            );
2736        }
2737        if let Err(e) = try_kv_set_prefer_store(store, STALE_AGENTS_KV, &current_dim).await {
2738            tracing::warn!(err = %e, "failed to persist stale model pool agents state");
2739        }
2740        if let Err(e) =
2741            try_kv_set_prefer_store(store, "metrics:orch.model_pool_stale_persistent.alert", "1")
2742                .await
2743        {
2744            tracing::warn!(err = %e, "failed to persist stale model pool alert metric");
2745        }
2746    }
2747}
2748
2749/// Ingest all active external tasks into the unified SQLite store.
2750///
2751/// Upserts each task so the store stays in sync with the backend.
2752/// Fetches all open issues (including unlabeled ones) so newly created
2753/// issues appear in the store immediately, not only after they get a
2754/// `status:*` label.  This is best-effort — individual task failures
2755/// are logged and skipped.
2756pub(crate) async fn ingest_external_tasks(
2757    backend: &Arc<dyn ExternalBackend>,
2758    repo: &str,
2759    store: &Arc<TaskStore>,
2760) -> anyhow::Result<()> {
2761    #[derive(Clone)]
2762    struct DuplicateTarget {
2763        id: String,
2764        status_label: String,
2765    }
2766
2767    let mut existing_by_title: HashMap<String, DuplicateTarget> = HashMap::new();
2768    let mut accepted_by_title: HashMap<String, DuplicateTarget> = HashMap::new();
2769
2770    let cutoff = (chrono::Utc::now() - chrono::Duration::hours(24))
2771        .format("%Y-%m-%dT%H:%M:%SZ")
2772        .to_string();
2773    match store.list_for_doctor(repo, &cutoff).await {
2774        Ok(recent) => {
2775            let mut insert_target = |task: &Task| {
2776                if task.origin == "internal" {
2777                    return;
2778                }
2779                let Some(ext_id) = task.external_id.as_ref() else {
2780                    return;
2781                };
2782                let status_label = task
2783                    .labels
2784                    .iter()
2785                    .find(|l| l.starts_with("status:"))
2786                    .cloned()
2787                    .unwrap_or_default();
2788                existing_by_title
2789                    .entry(task.title.clone())
2790                    .or_insert(DuplicateTarget {
2791                        id: ext_id.clone(),
2792                        status_label,
2793                    });
2794            };
2795
2796            // Prefer active tasks over recently done ones when selecting a canonical issue.
2797            for task in recent.iter().filter(|t| t.status != TaskStatus::Done) {
2798                insert_target(task);
2799            }
2800            for task in recent.iter().filter(|t| t.status == TaskStatus::Done) {
2801                insert_target(task);
2802            }
2803        }
2804        Err(e) => {
2805            tracing::warn!(repo, err = %e, "ingest: failed to load recent tasks for dedup");
2806        }
2807    }
2808
2809    // Fetch every open issue, with no `since` filter. The GitHub backend overrides
2810    // list_active_open_issues() to use one list_all_open_issues() request.
2811    //
2812    // A `since` cursor keys on updated_at, so an open issue with no task was skipped
2813    // forever once the cursor passed its updated_at, and the cursor had to be reset,
2814    // sentinelled and periodically ignored to recover (#3545, #3787, #3792). An
2815    // unfiltered listing has no such state and is cheaper too: its URL repeats, so
2816    // `get_all_pages` sends it conditionally and GitHub answers 304 while nothing
2817    // changed. A 304 does not count against the REST quota, a `since` fetch does.
2818    let all_tasks: Vec<(ExternalTask, Option<Status>)> =
2819        match backend.list_active_open_issues().await {
2820            Ok(tasks) => tasks,
2821            Err(e) => {
2822                tracing::warn!(
2823                    repo,
2824                    err = %e,
2825                    "ingest: failed to list open issues — no new issue is ingested this cycle"
2826                );
2827                return Ok(());
2828            }
2829        };
2830
2831    // Upsert into the store, collecting store IDs for batch status check
2832    // Also acknowledge newly detected issues (eyes reaction).
2833    let mut id_status_pairs: Vec<(i64, Status)> = Vec::new();
2834    // Collect (store_id, node_id) for newly ingested tasks so we can batch-fetch
2835    // estimate values from GitHub Projects after the loop.
2836    let mut new_task_node_ids: Vec<(i64, String)> = Vec::new();
2837    // Pre-load existing external IDs for this repo to avoid N+1 queries
2838    // (each get_by_external_id call is an individual SQL query). If the
2839    // lookup fails, fall back to an empty set so we conservatively treat
2840    // tasks as new (they will be upserted below).
2841    let existing_ext_ids: HashSet<String> = match store.existing_external_ids(repo).await {
2842        Ok(s) => s,
2843        Err(e) => {
2844            tracing::warn!(repo = repo, err = %e, "ingest: failed to load existing external ids — proceeding");
2845            HashSet::new()
2846        }
2847    };
2848    for (task, status) in &all_tasks {
2849        // Check if this task already exists in the store before upserting.
2850        // Use the pre-loaded HashSet to avoid issuing N individual queries.
2851        let is_new = !existing_ext_ids.contains(&task.id.0);
2852
2853        if is_new {
2854            let duplicate_target = existing_by_title
2855                .get(&task.title)
2856                .or_else(|| accepted_by_title.get(&task.title));
2857            if let Some(target) = duplicate_target
2858                && target.id != task.id.0
2859            {
2860                let label = target.status_label.as_str();
2861                let should_dedupe =
2862                    match backend.has_open_issue_with_title(&task.title, label).await {
2863                        Ok(true) => true,
2864                        Ok(false) => false,
2865                        Err(e) => {
2866                            tracing::warn!(
2867                                task_id = task.id.0,
2868                                err = %e,
2869                                "ingest: dedup check failed — defaulting to keep issue"
2870                            );
2871                            false
2872                        }
2873                    };
2874
2875                if should_dedupe {
2876                    let comment = format!(
2877                        "Duplicate of #{id} — closing to avoid duplicate work.{footer}",
2878                        id = target.id,
2879                        footer = engine::orch_footer()
2880                    );
2881                    if let Err(e) = backend.post_comment(&task.id, &comment).await {
2882                        tracing::warn!(
2883                            task_id = task.id.0,
2884                            err = %e,
2885                            "ingest: failed to comment on duplicate issue"
2886                        );
2887                    }
2888                    match backend.update_status(&task.id, Status::Done).await {
2889                        Ok(()) => {
2890                            tracing::info!(
2891                                task_id = task.id.0,
2892                                duplicate_of = target.id,
2893                                "ingest: closed duplicate issue"
2894                            );
2895                            continue;
2896                        }
2897                        Err(e) => {
2898                            tracing::warn!(
2899                                task_id = task.id.0,
2900                                err = %e,
2901                                "ingest: failed to close duplicate issue — falling through to ingest normally"
2902                            );
2903                        }
2904                    }
2905                }
2906            }
2907        }
2908
2909        match store.ensure_external_task(repo, task).await {
2910            Ok(store_id) => {
2911                if let Ok(number) = task.id.0.parse::<i64>()
2912                    && let Err(err) = dependencies::sync_issue_dependencies(
2913                        backend.as_ref(),
2914                        store,
2915                        store_id,
2916                        repo,
2917                        Some(number),
2918                        &task.body,
2919                    )
2920                    .await
2921                {
2922                    tracing::warn!(
2923                        task_id = task.id.0,
2924                        err = %err,
2925                        "ingest: dependency sync failed"
2926                    );
2927                }
2928                if let Some(s) = status {
2929                    id_status_pairs.push((store_id, *s));
2930                }
2931                // Acknowledge newly detected issues with an eyes reaction.
2932                if is_new {
2933                    let status_label = task
2934                        .labels
2935                        .iter()
2936                        .find(|l| l.starts_with("status:"))
2937                        .cloned()
2938                        .unwrap_or_default();
2939                    accepted_by_title
2940                        .entry(task.title.clone())
2941                        .or_insert(DuplicateTarget {
2942                            id: task.id.0.clone(),
2943                            status_label,
2944                        });
2945                    if let Err(e) = backend.acknowledge_issue(&task.id).await {
2946                        tracing::debug!(
2947                            task_id = task.id.0,
2948                            err = %e,
2949                            "ingest: acknowledgment failed"
2950                        );
2951                    }
2952                    // Sync to project board only for newly ingested tasks.
2953                    // Collect the returned node_id so we can batch-fetch estimates.
2954                    let task_status = status.unwrap_or(Status::New);
2955                    match backend.sync_to_project(&task.id, task_status).await {
2956                        Ok(Some(node_id)) => {
2957                            new_task_node_ids.push((store_id, node_id));
2958                        }
2959                        Ok(None) => {
2960                            // Project sync not configured or failed — no node_id to collect.
2961                        }
2962                        Err(e) => {
2963                            tracing::debug!(
2964                                task_id = task.id.0,
2965                                err = %e,
2966                                "ingest: project board sync failed"
2967                            );
2968                        }
2969                    }
2970                }
2971            }
2972            Err(e) => {
2973                tracing::debug!(task_id = task.id.0, ?e, "ingest: upsert failed");
2974            }
2975        }
2976    }
2977
2978    // Batch-fetch all upserted tasks in a single query, then sync status for NEW tasks.
2979    // Only sync status from backend → store for NEW tasks (first ingest).
2980    // Once a task exists in the store, its status is authoritative —
2981    // re-ingestion must not overwrite store-first status changes
2982    // (e.g., store has Routed but GitHub still shows New labels).
2983    if !id_status_pairs.is_empty() {
2984        let all_ids: Vec<i64> = id_status_pairs.iter().map(|(id, _)| *id).collect();
2985        let existing_map = match store.get_batch(&all_ids).await {
2986            Ok(map) => map,
2987            Err(e) => {
2988                tracing::warn!(err = %e, task_count = all_ids.len(), "ingest: get_batch failed — skipping initial status sync for newly ingested tasks this tick");
2989                return Ok(());
2990            }
2991        };
2992        for (store_id, status) in id_status_pairs {
2993            if let Some(existing) = existing_map.get(&store_id)
2994                && existing.status == TaskStatus::New
2995            {
2996                let db_status = status_to_task_status(status);
2997                if db_status != TaskStatus::New
2998                    && let Err(e) = store.update_status(store_id, db_status).await
2999                {
3000                    tracing::debug!(?e, "ingest: status sync failed");
3001                }
3002            }
3003        }
3004    }
3005
3006    // Batch-fetch estimate values from GitHub Projects for newly ingested tasks.
3007    // For each task where `tasks.estimate == 0`, populate it from the project board.
3008    if !new_task_node_ids.is_empty() {
3009        let node_ids: Vec<String> = new_task_node_ids.iter().map(|(_, n)| n.clone()).collect();
3010        match backend.get_project_item_estimates(&node_ids).await {
3011            Ok(estimates) => {
3012                for (store_id, node_id) in new_task_node_ids {
3013                    if let Some(&estimate) = estimates.get(&node_id)
3014                        && estimate > 0
3015                        && let Err(e) = store.set_estimate_if_zero(store_id, estimate).await
3016                    {
3017                        tracing::debug!(
3018                            store_id,
3019                            estimate,
3020                            err = %e,
3021                            "ingest: failed to set estimate from project"
3022                        );
3023                    }
3024                }
3025            }
3026            Err(e) => {
3027                tracing::debug!(
3028                    err = %e,
3029                    "ingest: project estimate fetch failed — skipping estimate sync this tick"
3030                );
3031            }
3032        }
3033    }
3034
3035    Ok(())
3036}
3037
3038#[cfg(test)]
3039mod tests {
3040    use super::*;
3041    use super::{CommentAction, MentionCursor, classify_comment};
3042    use crate::backends::test_helpers::NoopBackend;
3043    use crate::backends::{ExternalId, ExternalTask, Mention, Status};
3044    use crate::engine::commands::OwnerCommand;
3045    use crate::engine::cooldown::record_persistent_model_failure;
3046    use crate::engine::cooldown::set_agent_cooldown;
3047    use crate::engine::dispatch_guard::DispatchGuard;
3048    use crate::engine::router::Router;
3049    use crate::engine::router::RouterConfig;
3050    use crate::github::types::GitHubIssue;
3051    use crate::github::types::GitHubUser;
3052    use crate::store::NewTask;
3053    use crate::store::{RunTokenUsage, TaskStore};
3054    use async_trait::async_trait;
3055    use serial_test::serial;
3056    use std::slice;
3057    use std::sync::Arc;
3058    use std::sync::atomic::AtomicUsize;
3059    use std::sync::atomic::Ordering;
3060    use tokio::sync::Mutex;
3061    use tokio::sync::oneshot;
3062
3063    // ── ingest_external_tasks tests ─────────────────────────────────────────
3064
3065    /// Mock backend that returns configurable tasks per status.
3066    struct IngestMockBackend {
3067        /// Stored as (status_label, tasks) pairs since Status doesn't impl Hash.
3068        tasks: Vec<(String, Vec<ExternalTask>)>,
3069        dedup_result: bool,
3070        comments: Mutex<Vec<(String, String)>>,
3071        status_updates: Mutex<Vec<(String, Status)>>,
3072        project_syncs: Mutex<Vec<(String, Status)>>,
3073        project_reconciliations: AtomicUsize,
3074        /// Counts `list_active_open_issues` calls.
3075        list_calls: AtomicUsize,
3076    }
3077
3078    impl IngestMockBackend {
3079        fn with_tasks(tasks: Vec<(Status, ExternalTask)>) -> Arc<Self> {
3080            Self::with_tasks_and_dedup(tasks, false)
3081        }
3082
3083        fn with_tasks_and_dedup(
3084            tasks: Vec<(Status, ExternalTask)>,
3085            dedup_result: bool,
3086        ) -> Arc<Self> {
3087            let mut grouped: Vec<(String, Vec<ExternalTask>)> = Vec::new();
3088            for (status, task) in tasks {
3089                let label = status.as_label().to_string();
3090                if let Some(entry) = grouped.iter_mut().find(|(l, _)| l == &label) {
3091                    entry.1.push(task);
3092                } else {
3093                    grouped.push((label, vec![task]));
3094                }
3095            }
3096            Arc::new(Self {
3097                tasks: grouped,
3098                dedup_result,
3099                comments: Mutex::new(Vec::new()),
3100                status_updates: Mutex::new(Vec::new()),
3101                project_syncs: Mutex::new(Vec::new()),
3102                project_reconciliations: AtomicUsize::new(0),
3103                list_calls: AtomicUsize::new(0),
3104            })
3105        }
3106    }
3107
3108    #[async_trait]
3109    impl ExternalBackend for IngestMockBackend {
3110        fn name(&self) -> &'static str {
3111            "ingest-mock"
3112        }
3113        async fn create_task(&self, _: &str, _: &str, _: &[String]) -> anyhow::Result<ExternalId> {
3114            Ok(ExternalId("new".into()))
3115        }
3116        async fn get_task(&self, _: &ExternalId) -> anyhow::Result<ExternalTask> {
3117            anyhow::bail!("not implemented")
3118        }
3119        async fn list_by_status(&self, status: Status) -> anyhow::Result<Vec<ExternalTask>> {
3120            let label = status.as_label().to_string();
3121            Ok(self
3122                .tasks
3123                .iter()
3124                .find(|(l, _)| l == &label)
3125                .map(|(_, t)| t.clone())
3126                .unwrap_or_default())
3127        }
3128        async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
3129            Ok(vec![])
3130        }
3131        async fn list_active_open_issues(
3132            &self,
3133        ) -> anyhow::Result<Vec<(ExternalTask, Option<Status>)>> {
3134            self.list_calls.fetch_add(1, Ordering::Relaxed);
3135            let mut result = Vec::new();
3136            for (label, tasks) in &self.tasks {
3137                let status = match label.as_str() {
3138                    "status:routed" => Some(Status::Routed),
3139                    "status:in_progress" => Some(Status::InProgress),
3140                    "status:needs_review" => Some(Status::NeedsReview),
3141                    "status:in_review" => Some(Status::InReview),
3142                    "status:approved" => Some(Status::Approved),
3143                    "status:blocked" => Some(Status::Blocked),
3144                    _ => None,
3145                };
3146                for task in tasks {
3147                    result.push((task.clone(), status));
3148                }
3149            }
3150            Ok(result)
3151        }
3152        async fn post_comment(&self, id: &ExternalId, body: &str) -> anyhow::Result<()> {
3153            self.comments
3154                .lock()
3155                .await
3156                .push((id.0.clone(), body.to_string()));
3157            Ok(())
3158        }
3159        async fn set_labels(&self, _: &ExternalId, _: &[String]) -> anyhow::Result<()> {
3160            Ok(())
3161        }
3162        async fn remove_label(&self, _: &ExternalId, _: &str) -> anyhow::Result<()> {
3163            Ok(())
3164        }
3165        async fn get_sub_issues(&self, _: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
3166            Ok(vec![])
3167        }
3168        async fn health_check(&self) -> anyhow::Result<()> {
3169            Ok(())
3170        }
3171        async fn get_mentions(&self, _: &str) -> anyhow::Result<Vec<Mention>> {
3172            Ok(vec![])
3173        }
3174        async fn update_status(&self, id: &ExternalId, status: Status) -> anyhow::Result<()> {
3175            self.status_updates
3176                .lock()
3177                .await
3178                .push((id.0.clone(), status));
3179            Ok(())
3180        }
3181
3182        async fn has_open_issue_with_title(&self, _: &str, _: &str) -> anyhow::Result<bool> {
3183            Ok(self.dedup_result)
3184        }
3185
3186        async fn sync_to_project(
3187            &self,
3188            id: &ExternalId,
3189            status: Status,
3190        ) -> anyhow::Result<Option<String>> {
3191            self.project_syncs.lock().await.push((id.0.clone(), status));
3192            Ok(None) // Mock doesn't have real node IDs.
3193        }
3194
3195        async fn reconcile_project_statuses(&self) -> anyhow::Result<()> {
3196            self.project_reconciliations.fetch_add(1, Ordering::Relaxed);
3197            Ok(())
3198        }
3199    }
3200
3201    fn make_ext_task(id: &str, title: &str) -> ExternalTask {
3202        ExternalTask {
3203            id: ExternalId(id.to_string()),
3204            title: title.to_string(),
3205            body: String::new(),
3206            state: "open".to_string(),
3207            labels: vec![],
3208            author: "user".to_string(),
3209            created_at: "2026-01-01T00:00:00Z".to_string(),
3210            updated_at: "2026-01-01T00:00:00Z".to_string(),
3211            url: format!("https://github.com/owner/repo/issues/{id}"),
3212        }
3213    }
3214
3215    #[serial(cooldown_state)]
3216    #[tokio::test]
3217    async fn ingest_upserts_tasks_into_store() {
3218        cooldown::reset_global_state().await;
3219        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![
3220            (Status::New, make_ext_task("1", "First issue")),
3221            (Status::InProgress, make_ext_task("2", "Second issue")),
3222        ]);
3223        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3224
3225        ingest_external_tasks(&backend, "owner/repo", &store)
3226            .await
3227            .unwrap();
3228
3229        // Both tasks should be in the store
3230        let all = store.list_all("owner/repo").await.unwrap();
3231        assert_eq!(all.len(), 2);
3232
3233        let task1 = store
3234            .get_by_external_id("owner/repo", "1")
3235            .await
3236            .unwrap()
3237            .unwrap();
3238        assert_eq!(task1.title, "First issue");
3239        assert_eq!(task1.status, TaskStatus::New);
3240
3241        let task2 = store
3242            .get_by_external_id("owner/repo", "2")
3243            .await
3244            .unwrap()
3245            .unwrap();
3246        assert_eq!(task2.title, "Second issue");
3247        assert_eq!(task2.status, TaskStatus::InProgress);
3248    }
3249
3250    #[serial(cooldown_state)]
3251    #[tokio::test]
3252    async fn ingest_takes_every_open_issue_whatever_its_age_or_the_last_ingest() {
3253        // An issue opened after a restart reaches the store on the first sync, and so does
3254        // one whose updated_at predates any window a `since` cursor would have used. The
3255        // ingest keeps no cursor, so a stale KV value from an older build changes nothing.
3256        cooldown::reset_global_state().await;
3257        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3258        store
3259            .kv_set("issues_last_ingested:owner/repo", "2026-10-26T00:00:00Z")
3260            .await
3261            .unwrap();
3262
3263        let mut old = make_ext_task("73", "Filed weeks ago, never ingested");
3264        old.updated_at = "2026-01-01T00:00:00Z".to_string();
3265        let backend = IngestMockBackend::with_tasks(vec![(Status::New, old)]);
3266        let backend_dyn: Arc<dyn ExternalBackend> = Arc::<IngestMockBackend>::clone(&backend);
3267        ingest_external_tasks(&backend_dyn, "owner/repo", &store)
3268            .await
3269            .unwrap();
3270        assert!(
3271            store
3272                .get_by_external_id("owner/repo", "73")
3273                .await
3274                .unwrap()
3275                .is_some()
3276        );
3277
3278        // A second sync, standing in for the one right after a restart, sees a new issue.
3279        let backend = IngestMockBackend::with_tasks(vec![(
3280            Status::New,
3281            make_ext_task("78", "Opened after the restart"),
3282        )]);
3283        let backend_dyn: Arc<dyn ExternalBackend> = Arc::<IngestMockBackend>::clone(&backend);
3284        ingest_external_tasks(&backend_dyn, "owner/repo", &store)
3285            .await
3286            .unwrap();
3287        assert_eq!(
3288            backend.list_calls.load(Ordering::Relaxed),
3289            1,
3290            "one sync is one listing"
3291        );
3292        assert!(
3293            store
3294                .get_by_external_id("owner/repo", "78")
3295                .await
3296                .unwrap()
3297                .is_some()
3298        );
3299        assert_eq!(
3300            store
3301                .kv_get("issues_last_ingested:owner/repo")
3302                .await
3303                .unwrap()
3304                .as_deref(),
3305            Some("2026-10-26T00:00:00Z"),
3306            "the ingest must keep no cursor, so a stale one is never read or advanced"
3307        );
3308    }
3309
3310    #[serial(cooldown_state)]
3311    #[tokio::test]
3312    async fn ingest_updates_existing_tasks() {
3313        cooldown::reset_global_state().await;
3314        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![(
3315            Status::Routed,
3316            make_ext_task("42", "Updated title"),
3317        )]);
3318        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3319
3320        // Pre-create the task
3321        store
3322            .create(&NewTask {
3323                external_id: Some("42".to_string()),
3324                repo: "owner/repo".to_string(),
3325                origin: "github".to_string(),
3326                title: "Original title".to_string(),
3327                ..Default::default()
3328            })
3329            .await
3330            .unwrap();
3331
3332        ingest_external_tasks(&backend, "owner/repo", &store)
3333            .await
3334            .unwrap();
3335
3336        // Should have updated the title
3337        let task = store
3338            .get_by_external_id("owner/repo", "42")
3339            .await
3340            .unwrap()
3341            .unwrap();
3342        assert_eq!(task.title, "Updated title");
3343        assert_eq!(task.status, TaskStatus::Routed);
3344    }
3345
3346    #[serial(cooldown_state)]
3347    #[tokio::test]
3348    async fn ingest_closes_duplicate_titles() {
3349        cooldown::reset_global_state().await;
3350
3351        let backend = IngestMockBackend::with_tasks_and_dedup(
3352            vec![(Status::New, make_ext_task("2", "Duplicate issue"))],
3353            true,
3354        );
3355        let backend_dyn: Arc<dyn ExternalBackend> = Arc::<IngestMockBackend>::clone(&backend);
3356        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3357
3358        store
3359            .create(&NewTask {
3360                external_id: Some("1".to_string()),
3361                repo: "owner/repo".to_string(),
3362                origin: "github".to_string(),
3363                title: "Duplicate issue".to_string(),
3364                labels: vec!["status:new".to_string()],
3365                ..Default::default()
3366            })
3367            .await
3368            .unwrap();
3369
3370        ingest_external_tasks(&backend_dyn, "owner/repo", &store)
3371            .await
3372            .unwrap();
3373
3374        let all = store.list_all("owner/repo").await.unwrap();
3375        assert_eq!(all.len(), 1);
3376
3377        let comments = backend.comments.lock().await;
3378        assert_eq!(comments.len(), 1);
3379        assert!(comments[0].1.contains("Duplicate of #1"));
3380
3381        let status_updates = backend.status_updates.lock().await;
3382        assert_eq!(status_updates.len(), 1);
3383        assert_eq!(status_updates[0].0, "2");
3384        assert_eq!(status_updates[0].1, Status::Done);
3385    }
3386
3387    #[serial(cooldown_state)]
3388    #[tokio::test]
3389    async fn ingest_handles_empty_backend() {
3390        cooldown::reset_global_state().await;
3391        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
3392        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3393
3394        // Should not error on empty backend
3395        let result = ingest_external_tasks(&backend, "owner/repo", &store).await;
3396        assert!(result.is_ok());
3397
3398        let all = store.list_all("owner/repo").await.unwrap();
3399        assert!(all.is_empty());
3400    }
3401
3402    #[serial(cooldown_state)]
3403    #[tokio::test]
3404    async fn ingest_syncs_status_correctly_across_statuses() {
3405        cooldown::reset_global_state().await;
3406        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![
3407            (Status::New, make_ext_task("1", "New")),
3408            (Status::Routed, make_ext_task("2", "Routed")),
3409            (Status::InProgress, make_ext_task("3", "InProgress")),
3410            (Status::NeedsReview, make_ext_task("4", "NeedsReview")),
3411            (Status::InReview, make_ext_task("5", "InReview")),
3412            (Status::Blocked, make_ext_task("6", "Blocked")),
3413        ]);
3414        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3415
3416        ingest_external_tasks(&backend, "owner/repo", &store)
3417            .await
3418            .unwrap();
3419
3420        let counts = store.status_counts("owner/repo").await.unwrap();
3421        assert_eq!(counts.get("new"), Some(&1));
3422        assert_eq!(counts.get("routed"), Some(&1));
3423        assert_eq!(counts.get("in_progress"), Some(&1));
3424        assert_eq!(counts.get("needs_review"), Some(&1));
3425        assert_eq!(counts.get("in_review"), Some(&1));
3426        assert_eq!(counts.get("blocked"), Some(&1));
3427    }
3428
3429    #[serial(cooldown_state)]
3430    #[tokio::test]
3431    async fn ingest_does_not_overwrite_store_authoritative_status() {
3432        cooldown::reset_global_state().await;
3433        // Backend reports task as New (GitHub labels haven't caught up yet)
3434        let backend: Arc<dyn ExternalBackend> =
3435            IngestMockBackend::with_tasks(vec![(Status::New, make_ext_task("99", "My task"))]);
3436        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3437
3438        // Pre-create the task and advance it to Routed in the store
3439        let id = store
3440            .create(&NewTask {
3441                external_id: Some("99".to_string()),
3442                repo: "owner/repo".to_string(),
3443                origin: "github".to_string(),
3444                title: "My task".to_string(),
3445                ..Default::default()
3446            })
3447            .await
3448            .unwrap();
3449        store.update_status(id, TaskStatus::Routed).await.unwrap();
3450
3451        // Ingest should NOT overwrite Routed back to New
3452        ingest_external_tasks(&backend, "owner/repo", &store)
3453            .await
3454            .unwrap();
3455
3456        let task = store.get(id).await.unwrap();
3457        assert_eq!(
3458            task.status,
3459            TaskStatus::Routed,
3460            "ingest must not overwrite store-authoritative status"
3461        );
3462    }
3463
3464    #[serial(cooldown_state)]
3465    #[tokio::test]
3466    async fn ingest_sync_to_project_called_only_for_new_tasks() {
3467        cooldown::reset_global_state().await;
3468
3469        // Two tasks: "10" is pre-existing, "11" is new.
3470        let backend = IngestMockBackend::with_tasks(vec![
3471            (Status::New, make_ext_task("10", "Existing task")),
3472            (Status::New, make_ext_task("11", "New task")),
3473        ]);
3474        let backend_dyn: Arc<dyn ExternalBackend> = Arc::<IngestMockBackend>::clone(&backend);
3475        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3476
3477        // Pre-insert task "10" so it already exists in the store.
3478        store
3479            .create(&NewTask {
3480                external_id: Some("10".to_string()),
3481                repo: "owner/repo".to_string(),
3482                origin: "github".to_string(),
3483                title: "Existing task".to_string(),
3484                ..Default::default()
3485            })
3486            .await
3487            .unwrap();
3488
3489        ingest_external_tasks(&backend_dyn, "owner/repo", &store)
3490            .await
3491            .unwrap();
3492
3493        let syncs = backend.project_syncs.lock().await;
3494        // Only the new task "11" should trigger sync_to_project — not the pre-existing "10".
3495        assert_eq!(
3496            syncs.len(),
3497            1,
3498            "sync_to_project must be called exactly once (for new task only), got: {syncs:?}"
3499        );
3500        assert_eq!(syncs[0].0, "11", "sync_to_project must target the new task");
3501    }
3502
3503    #[serial(cooldown_state)]
3504    #[tokio::test]
3505    async fn ingest_handles_mixed_labeled_and_unlabeled_open_issues() {
3506        // Simulate a backend where some issues have no status label (routable/unlabeled)
3507        // and others carry active status labels.  Both kinds must be ingested and the
3508        // store status must reflect the label where present.
3509        struct MixedBackend;
3510
3511        #[async_trait]
3512        impl ExternalBackend for MixedBackend {
3513            fn name(&self) -> &'static str {
3514                "mixed"
3515            }
3516            async fn create_task(
3517                &self,
3518                _: &str,
3519                _: &str,
3520                _: &[String],
3521            ) -> anyhow::Result<ExternalId> {
3522                Ok(ExternalId("x".into()))
3523            }
3524            async fn get_task(&self, _: &ExternalId) -> anyhow::Result<ExternalTask> {
3525                anyhow::bail!("not implemented")
3526            }
3527            async fn list_by_status(&self, _: Status) -> anyhow::Result<Vec<ExternalTask>> {
3528                Ok(vec![])
3529            }
3530            /// Return a mix: one unlabeled (no status) + one labeled in_progress.
3531            async fn list_active_open_issues(
3532                &self,
3533            ) -> anyhow::Result<Vec<(ExternalTask, Option<Status>)>> {
3534                Ok(vec![
3535                    (make_ext_task("100", "Unlabeled routable"), None),
3536                    (
3537                        make_ext_task("101", "In progress issue"),
3538                        Some(Status::InProgress),
3539                    ),
3540                ])
3541            }
3542            async fn post_comment(&self, _: &ExternalId, _: &str) -> anyhow::Result<()> {
3543                Ok(())
3544            }
3545            async fn set_labels(&self, _: &ExternalId, _: &[String]) -> anyhow::Result<()> {
3546                Ok(())
3547            }
3548            async fn remove_label(&self, _: &ExternalId, _: &str) -> anyhow::Result<()> {
3549                Ok(())
3550            }
3551            async fn get_sub_issues(&self, _: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
3552                Ok(vec![])
3553            }
3554            async fn health_check(&self) -> anyhow::Result<()> {
3555                Ok(())
3556            }
3557            async fn get_mentions(&self, _: &str) -> anyhow::Result<Vec<Mention>> {
3558                Ok(vec![])
3559            }
3560            async fn update_status(&self, _: &ExternalId, _: Status) -> anyhow::Result<()> {
3561                Ok(())
3562            }
3563            async fn has_open_issue_with_title(&self, _: &str, _: &str) -> anyhow::Result<bool> {
3564                Ok(false)
3565            }
3566            async fn sync_to_project(
3567                &self,
3568                _: &ExternalId,
3569                _: Status,
3570            ) -> anyhow::Result<Option<String>> {
3571                Ok(None)
3572            }
3573        }
3574
3575        cooldown::reset_global_state().await;
3576
3577        let backend: Arc<dyn ExternalBackend> = Arc::new(MixedBackend);
3578        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3579
3580        ingest_external_tasks(&backend, "owner/repo", &store)
3581            .await
3582            .unwrap();
3583
3584        let all = store.list_all("owner/repo").await.unwrap();
3585        assert_eq!(all.len(), 2, "both issues must be ingested");
3586
3587        let unlabeled = store
3588            .get_by_external_id("owner/repo", "100")
3589            .await
3590            .unwrap()
3591            .unwrap();
3592        assert_eq!(
3593            unlabeled.status,
3594            TaskStatus::New,
3595            "unlabeled issue should start as New"
3596        );
3597
3598        let in_progress = store
3599            .get_by_external_id("owner/repo", "101")
3600            .await
3601            .unwrap()
3602            .unwrap();
3603        assert_eq!(
3604            in_progress.status,
3605            TaskStatus::InProgress,
3606            "labeled in_progress issue must be synced to InProgress"
3607        );
3608    }
3609
3610    // ── kv_get_prefer_store / kv_set_prefer_store ────────────────────
3611
3612    #[serial(cooldown_state)]
3613    #[tokio::test]
3614    async fn kv_get_prefer_store_reads_from_store() {
3615        cooldown::reset_global_state().await;
3616        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3617
3618        store.kv_set("k1", "store_val").await.unwrap();
3619
3620        let opt = Some(&store);
3621        let val = kv_get_prefer_store(opt, "k1").await;
3622        assert_eq!(val.as_deref(), Some("store_val"));
3623    }
3624
3625    #[serial(cooldown_state)]
3626    #[tokio::test]
3627    async fn kv_get_prefer_store_returns_none_without_store() {
3628        cooldown::reset_global_state().await;
3629        let opt: Option<&Arc<TaskStore>> = None;
3630        let val = kv_get_prefer_store(opt, "k2").await;
3631        assert_eq!(val, None);
3632    }
3633
3634    #[serial(cooldown_state)]
3635    #[tokio::test]
3636    async fn kv_set_prefer_store_writes_to_store() {
3637        cooldown::reset_global_state().await;
3638        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3639
3640        let opt = Some(&store);
3641        kv_set_prefer_store(opt, "k3", "val3").await;
3642
3643        assert_eq!(store.kv_get("k3").await.unwrap().as_deref(), Some("val3"));
3644    }
3645
3646    #[serial(cooldown_state)]
3647    #[tokio::test]
3648    async fn kv_set_prefer_store_noop_without_store() {
3649        cooldown::reset_global_state().await;
3650        let opt: Option<&Arc<TaskStore>> = None;
3651        // Should not panic
3652        kv_set_prefer_store(opt, "k4", "val4").await;
3653    }
3654
3655    // ── try_kv_get_prefer_store / try_kv_set_prefer_store ────────────────
3656
3657    #[serial(cooldown_state)]
3658    #[tokio::test]
3659    async fn try_kv_get_prefer_store_returns_value_from_store() {
3660        cooldown::reset_global_state().await;
3661        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3662        store.kv_set("try_k1", "try_val1").await.unwrap();
3663
3664        let opt = Some(&store);
3665        let result = try_kv_get_prefer_store(opt, "try_k1").await;
3666        assert_eq!(result.unwrap().as_deref(), Some("try_val1"));
3667    }
3668
3669    #[serial(cooldown_state)]
3670    #[tokio::test]
3671    async fn try_kv_get_prefer_store_returns_none_for_missing_key() {
3672        cooldown::reset_global_state().await;
3673        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3674        let opt = Some(&store);
3675
3676        let result = try_kv_get_prefer_store(opt, "nonexistent_key").await;
3677        assert_eq!(result.unwrap(), None); // Key doesn't exist, but query succeeded
3678    }
3679
3680    #[serial(cooldown_state)]
3681    #[tokio::test]
3682    async fn try_kv_get_prefer_store_returns_none_without_store() {
3683        cooldown::reset_global_state().await;
3684        let opt: Option<&Arc<TaskStore>> = None;
3685        let result = try_kv_get_prefer_store(opt, "try_k2").await;
3686        assert_eq!(result.unwrap(), None);
3687    }
3688
3689    #[serial(cooldown_state)]
3690    #[tokio::test]
3691    async fn try_kv_set_prefer_store_writes_to_store() {
3692        cooldown::reset_global_state().await;
3693        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3694        let opt = Some(&store);
3695
3696        let result = try_kv_set_prefer_store(opt, "try_k3", "try_val3").await;
3697        assert!(result.is_ok());
3698        assert_eq!(
3699            store.kv_get("try_k3").await.unwrap().as_deref(),
3700            Some("try_val3")
3701        );
3702    }
3703
3704    #[serial(cooldown_state)]
3705    #[tokio::test]
3706    async fn try_kv_set_prefer_store_ok_without_store() {
3707        cooldown::reset_global_state().await;
3708        let opt: Option<&Arc<TaskStore>> = None;
3709        // Should not error - best effort when store unavailable
3710        let result = try_kv_set_prefer_store(opt, "try_k4", "try_val4").await;
3711        assert!(result.is_ok());
3712    }
3713
3714    // ── dispatching lock tests ──────────────────────────────────────────
3715
3716    #[serial(cooldown_state)]
3717    #[test]
3718    fn dispatching_set_blocks_duplicate_processing() {
3719        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3720
3721        let repo = "owner/repo";
3722        let task_id = "42";
3723        let dispatch_key = format!("{repo}/{task_id}");
3724
3725        // Initially not in the map
3726        assert!(!dispatching.contains_key(&dispatch_key));
3727
3728        // Insert — simulates dispatch starting
3729        dispatching.insert(dispatch_key.clone(), task_id.to_string());
3730
3731        // Now should be blocked
3732        assert!(
3733            dispatching.contains_key(&dispatch_key),
3734            "task should be locked while dispatching"
3735        );
3736
3737        // Remove — simulates review completion
3738        dispatching.remove(&dispatch_key);
3739
3740        // Now should be free again
3741        assert!(
3742            !dispatching.contains_key(&dispatch_key),
3743            "task should be unlocked after review completes"
3744        );
3745    }
3746
3747    #[serial(cooldown_state)]
3748    #[test]
3749    fn dispatching_set_does_not_block_other_tasks() {
3750        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3751
3752        let repo = "owner/repo";
3753
3754        // Lock task 42
3755        let key_42 = format!("{repo}/42");
3756        dispatching.insert(key_42.clone(), "42".to_string());
3757
3758        // Task 43 should NOT be blocked
3759        let key_43 = format!("{repo}/43");
3760        assert!(
3761            dispatching.contains_key(&key_42),
3762            "task 42 should be locked"
3763        );
3764        assert!(
3765            !dispatching.contains_key(&key_43),
3766            "task 43 should not be locked"
3767        );
3768    }
3769
3770    /// Regression test for a6d8b9a.
3771    ///
3772    /// Before the fix, `sync_tick` step 5 transitioned a task to `InReview` and then
3773    /// spawned a `tokio::spawn` review agent WITHOUT first inserting the `dispatch_key`
3774    /// into the dispatching set. This created a window where `review_open_prs` (step 4)
3775    /// in a concurrent sync tick could see the task as `InReview`, find the key absent,
3776    /// and re-dispatch it — silently discarding human `CHANGES_REQUESTED` feedback when
3777    /// the review agent later completed and marked the task `Done`.
3778    ///
3779    /// The fix: insert `dispatch_key` BEFORE `tokio::spawn`, remove it AFTER the future
3780    /// completes. This test verifies that invariant using tokio channels to synchronize
3781    /// with a simulated concurrent caller (review_open_prs).
3782    #[serial(cooldown_state)]
3783    #[tokio::test]
3784    async fn dispatch_key_held_during_review_agent_execution() {
3785        cooldown::reset_global_state().await;
3786
3787        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3788
3789        let dispatch_key = "owner/repo/42".to_string();
3790
3791        // Step 1: insert before spawning — this is the invariant added by a6d8b9a.
3792        dispatching.insert(dispatch_key.clone(), "42".to_string());
3793
3794        let (agent_started_tx, agent_started_rx) = oneshot::channel::<()>();
3795        let (check_done_tx, check_done_rx) = oneshot::channel::<()>();
3796
3797        let dispatching_agent = Arc::clone(&dispatching);
3798        let key_agent = dispatch_key.clone();
3799
3800        // Step 2: spawn review agent — key is ALREADY in the set (fix invariant holds).
3801        let review_task = tokio::spawn(async move {
3802            agent_started_tx.send(()).unwrap();
3803            // Simulate review agent work — hold until the concurrent check is done.
3804            check_done_rx.await.unwrap();
3805            // Remove from dispatching set on completion (mirror of tick.rs/sync.rs).
3806            dispatching_agent.remove(&key_agent);
3807        });
3808
3809        // Step 3: concurrent review_open_prs check — simulates step 4 of another
3810        // sync_tick invocation running while the review agent is still active.
3811        agent_started_rx.await.unwrap();
3812        assert!(
3813            dispatching.contains_key(&dispatch_key),
3814            "dispatch_key must be visible in the dispatching set while the review \
3815             agent is running — without the a6d8b9a fix, review_open_prs would see \
3816             the key absent and re-dispatch the task, silently dropping \
3817             CHANGES_REQUESTED feedback"
3818        );
3819
3820        // Step 4: review agent completes.
3821        check_done_tx.send(()).unwrap();
3822        review_task.await.unwrap();
3823
3824        // Step 5: key released — review_open_prs can now act on the task if needed.
3825        assert!(
3826            !dispatching.contains_key(&dispatch_key),
3827            "dispatch_key must be removed from the dispatching set after the \
3828             review agent completes so subsequent sync ticks can process the task"
3829        );
3830    }
3831
3832    /// Regression test for issue #833.
3833    ///
3834    /// Before the fix, `dispatch_key` was inserted into the `dispatching` HashSet
3835    /// *outside* the spawned task but removed *inside* it.  When the spawned async
3836    /// block panicked (e.g. an `unwrap()` inside `review_and_merge`), Tokio caught
3837    /// the panic and terminated the task without running the cleanup code, leaving
3838    /// the key in the set permanently.  Subsequent stuck-task recovery would reset
3839    /// the task to `NeedsReview`, but the subscriber/sync would see the key still
3840    /// present and skip it forever — a permanent review loop until service restart.
3841    ///
3842    /// The fix introduces `DispatchGuard`, a RAII wrapper whose `Drop` impl removes
3843    /// the key unconditionally, even when the task unwinds via panic.
3844    #[serial(cooldown_state)]
3845    #[tokio::test]
3846    async fn dispatch_guard_releases_key_on_panic() {
3847        cooldown::reset_global_state().await;
3848
3849        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3850        let key = "owner/repo/42".to_string();
3851
3852        // Insert key before spawn — mirrors the production invariant established
3853        // by the a6d8b9a fix (key must be visible before the spawn so concurrent
3854        // review_open_prs callers see it and skip the task).
3855        dispatching.insert(key.clone(), "42".to_string());
3856
3857        // Create the guard (takes ownership of the removal obligation).
3858        let guard = DispatchGuard::new(Arc::clone(&dispatching), key.clone());
3859
3860        // Spawn a task that panics before the normal completion path.
3861        let handle = tokio::spawn(async move {
3862            let _guard = guard; // moved in; Drop removes key whether we panic or not
3863            panic!("simulated panic inside review_and_merge");
3864        });
3865
3866        // Absorb the JoinError — the panic is expected.
3867        let _ = handle.await;
3868
3869        // Key MUST be gone even though the task panicked.
3870        assert!(
3871            !dispatching.contains_key(&key),
3872            "dispatch key must be removed from the dispatching set even when the \
3873             spawned review task panics — without DispatchGuard the key leaks and \
3874             the task gets stuck in a permanent review loop"
3875        );
3876    }
3877
3878    #[serial(cooldown_state)]
3879    #[tokio::test]
3880    async fn sync_tick_reconciles_board_without_ingesting_or_transitioning_tasks() {
3881        cooldown::reset_global_state().await;
3882        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3883        let mock = IngestMockBackend::with_tasks(vec![]);
3884        let backend: Arc<dyn ExternalBackend> = Arc::<IngestMockBackend>::clone(&mock);
3885        let task_manager = Arc::new(TaskManager::with_store(
3886            Arc::clone(&backend),
3887            Arc::clone(&store),
3888            "owner/repo".to_string(),
3889        ));
3890        let dispatching = Arc::new(DashMap::new());
3891        Box::pin(sync_tick(
3892            &backend,
3893            &Arc::new(TmuxManager::new()),
3894            "owner/repo",
3895            &EngineConfig::default(),
3896            &Arc::new(RwLock::new(Router::new_for_test(
3897                RouterConfig::default(),
3898                vec!["claude".to_string()],
3899            ))),
3900            &task_manager,
3901            &store,
3902            &dispatching,
3903            &Arc::new(DashSet::new()),
3904        ))
3905        .await
3906        .unwrap();
3907        assert_eq!(mock.project_reconciliations.load(Ordering::Relaxed), 1);
3908        assert!(mock.project_syncs.lock().await.is_empty());
3909        assert!(mock.status_updates.lock().await.is_empty());
3910        assert!(dispatching.is_empty());
3911        assert!(
3912            store
3913                .existing_external_ids("owner/repo")
3914                .await
3915                .unwrap()
3916                .is_empty()
3917        );
3918    }
3919
3920    #[serial(cooldown_state)]
3921    #[tokio::test]
3922    async fn needs_review_refire_increments_and_fires_when_old_enough() {
3923        cooldown::reset_global_state().await;
3924        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3925        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
3926        let task_manager = Arc::new(TaskManager::with_store(
3927            backend,
3928            Arc::clone(&store),
3929            "owner/repo".to_string(),
3930        ));
3931        let router = reviewer_router();
3932        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3933
3934        let id = store
3935            .upsert_external(&crate::store::UpsertExternal {
3936                repo: "owner/repo",
3937                ext_id: "201",
3938                title: "Needs review old",
3939                body: "",
3940                author: "",
3941                url: "",
3942                labels: &[],
3943                origin: "github",
3944            })
3945            .await
3946            .unwrap();
3947        store
3948            .update_status(id, TaskStatus::NeedsReview)
3949            .await
3950            .unwrap();
3951        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
3952            .bind(id)
3953            .execute(store.pool())
3954            .await
3955            .unwrap();
3956
3957        let t = store.get(id).await.unwrap();
3958        assert_eq!(t.needs_review_refires, 0);
3959
3960        refire_and_escalate_stale_needs_review_global(
3961            &task_manager,
3962            &store,
3963            &router,
3964            &dispatching,
3965            &sweep_repos(),
3966        )
3967        .await
3968        .unwrap();
3969
3970        let t2 = store.get(id).await.unwrap();
3971        assert_eq!(t2.needs_review_refires, 1);
3972        assert_eq!(t2.status, TaskStatus::NeedsReview);
3973    }
3974
3975    #[serial(cooldown_state)]
3976    #[tokio::test]
3977    async fn needs_review_backoff_delays_fire_but_increments_counter() {
3978        cooldown::reset_global_state().await;
3979        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3980        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
3981        let task_manager = Arc::new(TaskManager::with_store(
3982            backend,
3983            Arc::clone(&store),
3984            "owner/repo".to_string(),
3985        ));
3986        let router = reviewer_router();
3987        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
3988
3989        let id = store
3990            .upsert_external(&crate::store::UpsertExternal {
3991                repo: "owner/repo",
3992                ext_id: "202",
3993                title: "Needs review backoff",
3994                body: "",
3995                author: "",
3996                url: "",
3997                labels: &[],
3998                origin: "github",
3999            })
4000            .await
4001            .unwrap();
4002        store
4003            .update_status(id, TaskStatus::NeedsReview)
4004            .await
4005            .unwrap();
4006
4007        store
4008            .set_fields(id, &[("needs_review_refires", serde_json::json!(1))])
4009            .await
4010            .unwrap();
4011
4012        // Age 1 minute: stale enough to consider, but backoff at refires=1 needs 2 minutes
4013        let now_minus_1 = (chrono::Utc::now() - chrono::Duration::minutes(1))
4014            .format("%Y-%m-%dT%H:%M:%SZ")
4015            .to_string();
4016        sqlx::query("UPDATE tasks SET updated_at = ? WHERE id = ?")
4017            .bind(now_minus_1.clone())
4018            .bind(id)
4019            .execute(store.pool())
4020            .await
4021            .unwrap();
4022
4023        let before = store.get(id).await.unwrap();
4024        assert_eq!(before.needs_review_refires, 1);
4025
4026        refire_and_escalate_stale_needs_review_global(
4027            &task_manager,
4028            &store,
4029            &router,
4030            &dispatching,
4031            &sweep_repos(),
4032        )
4033        .await
4034        .unwrap();
4035
4036        let after = store.get(id).await.unwrap();
4037        // Counter must not increment on a backoff delay, only on an actual re-fire
4038        assert_eq!(after.needs_review_refires, 1);
4039        assert_eq!(after.updated_at, now_minus_1);
4040        assert_eq!(after.status, TaskStatus::NeedsReview);
4041    }
4042
4043    /// Router with one claude review model, so the refire sweeps see a reviewer.
4044    fn reviewer_router() -> Router {
4045        let mut config = RouterConfig::default();
4046        config.model_map.insert(
4047            "review".to_string(),
4048            HashMap::from([("claude".to_string(), vec!["sonnet".to_string()])]),
4049        );
4050        Router::new_for_test(config, vec!["claude".to_string()])
4051    }
4052
4053    #[serial(cooldown_state)]
4054    #[tokio::test]
4055    async fn needs_review_escalates_to_blocked_after_max_refires() {
4056        cooldown::reset_global_state().await;
4057        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4058        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4059        let task_manager = Arc::new(TaskManager::with_store(
4060            backend,
4061            Arc::clone(&store),
4062            "owner/repo".to_string(),
4063        ));
4064        let router = reviewer_router();
4065        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4066
4067        let id = store
4068            .upsert_external(&crate::store::UpsertExternal {
4069                repo: "owner/repo",
4070                ext_id: "203",
4071                title: "Needs review escalate",
4072                body: "",
4073                author: "",
4074                url: "",
4075                labels: &[],
4076                origin: "github",
4077            })
4078            .await
4079            .unwrap();
4080        store
4081            .update_status(id, TaskStatus::NeedsReview)
4082            .await
4083            .unwrap();
4084
4085        // Counter at MAX (5): the next pass is attempt 6 and escalates
4086        store
4087            .set_fields(id, &[("needs_review_refires", serde_json::json!(5))])
4088            .await
4089            .unwrap();
4090        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4091            .bind(id)
4092            .execute(store.pool())
4093            .await
4094            .unwrap();
4095
4096        refire_and_escalate_stale_needs_review_global(
4097            &task_manager,
4098            &store,
4099            &router,
4100            &dispatching,
4101            &sweep_repos(),
4102        )
4103        .await
4104        .unwrap();
4105
4106        let after = store.get(id).await.unwrap();
4107        assert_eq!(after.status, TaskStatus::Blocked);
4108        assert!(after.block_reason.is_some());
4109    }
4110
4111    /// Exactly MAX_NEEDS_REVIEW_REFIRE_ATTEMPTS refires are allowed before
4112    /// escalation. Task stays NeedsReview at 5 refires and only blocks on the 6th
4113    #[serial(cooldown_state)]
4114    #[tokio::test]
4115    async fn needs_review_allows_max_refires_before_escalation() {
4116        cooldown::reset_global_state().await;
4117        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4118        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4119        let task_manager = Arc::new(TaskManager::with_store(
4120            backend,
4121            Arc::clone(&store),
4122            "owner/repo".to_string(),
4123        ));
4124        let router = reviewer_router();
4125        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4126
4127        let id = store
4128            .upsert_external(&crate::store::UpsertExternal {
4129                repo: "owner/repo",
4130                ext_id: "204",
4131                title: "Max refires test",
4132                body: "",
4133                author: "",
4134                url: "",
4135                labels: &[],
4136                origin: "github",
4137            })
4138            .await
4139            .unwrap();
4140        store
4141            .update_status(id, TaskStatus::NeedsReview)
4142            .await
4143            .unwrap();
4144
4145        // Counter at MAX-1 (4): incrementing hits MAX (5) and must not escalate
4146        store
4147            .set_fields(id, &[("needs_review_refires", serde_json::json!(4))])
4148            .await
4149            .unwrap();
4150        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4151            .bind(id)
4152            .execute(store.pool())
4153            .await
4154            .unwrap();
4155
4156        refire_and_escalate_stale_needs_review_global(
4157            &task_manager,
4158            &store,
4159            &router,
4160            &dispatching,
4161            &sweep_repos(),
4162        )
4163        .await
4164        .unwrap();
4165
4166        let after = store.get(id).await.unwrap();
4167        assert_eq!(after.status, TaskStatus::NeedsReview);
4168        assert_eq!(after.needs_review_refires, 5);
4169        assert!(after.block_reason.is_none());
4170    }
4171
4172    /// Every repo the sweep tests use, so each counts as active
4173    fn sweep_repos() -> HashSet<String> {
4174        ["owner/repo", "inactive/repo", "some/repo", "active/repo"]
4175            .into_iter()
4176            .map(String::from)
4177            .collect()
4178    }
4179
4180    /// A repo outside the active set has no review subscriber (#4024)
4181    #[serial(cooldown_state)]
4182    #[tokio::test]
4183    async fn global_sweeps_leave_tasks_of_a_removed_repo_alone() {
4184        cooldown::reset_global_state().await;
4185        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4186        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4187        let task_manager = Arc::new(TaskManager::with_store(
4188            backend,
4189            Arc::clone(&store),
4190            "active/repo".to_string(),
4191        ));
4192        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4193        let router = router_with_healthy_review_agent();
4194        let active: HashSet<String> = ["active/repo".to_string()].into_iter().collect();
4195
4196        let review_id = store
4197            .create_internal("removed/repo", "Needs review", "", "cron", "1", None)
4198            .await
4199            .unwrap();
4200        store
4201            .update_status(review_id, TaskStatus::NeedsReview)
4202            .await
4203            .unwrap();
4204        let blocked_id = store
4205            .create_internal("removed/repo", "Rebroadcast blocked", "", "cron", "2", None)
4206            .await
4207            .unwrap();
4208        store
4209            .update_status(blocked_id, TaskStatus::Blocked)
4210            .await
4211            .unwrap();
4212        store
4213            .set_block_reason(blocked_id, Some(REBROADCAST_BLOCK_REASON))
4214            .await
4215            .unwrap();
4216        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z'")
4217            .execute(store.pool())
4218            .await
4219            .unwrap();
4220
4221        refire_and_escalate_stale_needs_review_global(
4222            &task_manager,
4223            &store,
4224            &router,
4225            &dispatching,
4226            &active,
4227        )
4228        .await
4229        .unwrap();
4230        auto_recover_rebroadcast_blocked_tasks_global(&router, &task_manager, &store, &active)
4231            .await
4232            .unwrap();
4233
4234        let review = store.get(review_id).await.unwrap();
4235        assert_eq!(review.status, TaskStatus::NeedsReview);
4236        assert_eq!(review.needs_review_refires, 0);
4237        let blocked = store.get(blocked_id).await.unwrap();
4238        assert_eq!(blocked.status, TaskStatus::Blocked);
4239        assert_eq!(
4240            blocked.block_reason.as_deref(),
4241            Some(REBROADCAST_BLOCK_REASON)
4242        );
4243    }
4244
4245    // ── refire_and_escalate_stale_needs_review_global ──────────────────────
4246
4247    fn router_with_healthy_review_agent() -> Router {
4248        let mut config = RouterConfig::default();
4249        config
4250            .model_map
4251            .entry("review".to_string())
4252            .or_default()
4253            .insert("claude".to_string(), vec!["sonnet".to_string()]);
4254        let mut router = Router::new(config);
4255        router.available_agents = vec!["claude".to_string()];
4256        router
4257    }
4258
4259    /// Regression test for #3469: a task stuck in NeedsReview for a repo that
4260    /// is not the TaskManager's own repo must still be refired. The sweep
4261    /// queries via `list_all_by_status_global` and writes through the store
4262    /// directly, bypassing the repo-scoped TaskManager
4263    #[serial(cooldown_state)]
4264    #[tokio::test]
4265    async fn global_refire_sweep_refires_stale_task_from_inactive_repo() {
4266        cooldown::reset_global_state().await;
4267        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4268        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4269        // TaskManager scoped to the active repo — NOT the repo the task belongs to.
4270        let task_manager = Arc::new(TaskManager::with_store(
4271            backend,
4272            Arc::clone(&store),
4273            "active/repo".to_string(),
4274        ));
4275        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4276        let router = router_with_healthy_review_agent();
4277
4278        let id = store
4279            .upsert_external(&crate::store::UpsertExternal {
4280                repo: "inactive/repo",
4281                ext_id: "490",
4282                title: "Stale needs_review from inactive repo",
4283                body: "",
4284                author: "",
4285                url: "",
4286                labels: &[],
4287                origin: "github",
4288            })
4289            .await
4290            .unwrap();
4291        store
4292            .update_status(id, TaskStatus::NeedsReview)
4293            .await
4294            .unwrap();
4295        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4296            .bind(id)
4297            .execute(store.pool())
4298            .await
4299            .unwrap();
4300
4301        refire_and_escalate_stale_needs_review_global(
4302            &task_manager,
4303            &store,
4304            &router,
4305            &dispatching,
4306            &sweep_repos(),
4307        )
4308        .await
4309        .unwrap();
4310
4311        let task = store.get(id).await.unwrap();
4312        assert_eq!(
4313            task.needs_review_refires, 1,
4314            "cross-repo task must be refired"
4315        );
4316        assert_eq!(task.status, TaskStatus::NeedsReview);
4317    }
4318
4319    /// After `MAX_NEEDS_REVIEW_REFIRE_ATTEMPTS` refires, a cross-repo task must
4320    /// escalate to Blocked
4321    #[serial(cooldown_state)]
4322    #[tokio::test]
4323    async fn global_refire_sweep_escalates_cross_repo_task_to_blocked() {
4324        cooldown::reset_global_state().await;
4325        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4326        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4327        let task_manager = Arc::new(TaskManager::with_store(
4328            backend,
4329            Arc::clone(&store),
4330            "active/repo".to_string(),
4331        ));
4332        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4333        let router = router_with_healthy_review_agent();
4334
4335        let id = store
4336            .upsert_external(&crate::store::UpsertExternal {
4337                repo: "inactive/repo",
4338                ext_id: "493",
4339                title: "Exhausted refires from inactive repo",
4340                body: "",
4341                author: "",
4342                url: "",
4343                labels: &[],
4344                origin: "github",
4345            })
4346            .await
4347            .unwrap();
4348        store
4349            .update_status(id, TaskStatus::NeedsReview)
4350            .await
4351            .unwrap();
4352        store
4353            .set_fields(id, &[("needs_review_refires", serde_json::json!(5))])
4354            .await
4355            .unwrap();
4356        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4357            .bind(id)
4358            .execute(store.pool())
4359            .await
4360            .unwrap();
4361
4362        refire_and_escalate_stale_needs_review_global(
4363            &task_manager,
4364            &store,
4365            &router,
4366            &dispatching,
4367            &sweep_repos(),
4368        )
4369        .await
4370        .unwrap();
4371
4372        let task = store.get(id).await.unwrap();
4373        assert_eq!(task.status, TaskStatus::Blocked);
4374        assert_eq!(
4375            task.block_reason.as_deref(),
4376            Some("review agent rebroadcast escalated after repeated retries")
4377        );
4378    }
4379
4380    /// When every review agent is cooled, the sweep must skip entirely.
4381    /// Re-firing while all agents are cooled would waste the refire counter and
4382    /// could escalate a task to Blocked because agents happened to be
4383    /// temporarily unavailable
4384    #[serial(cooldown_state)]
4385    #[tokio::test]
4386    async fn global_refire_sweep_skips_when_all_review_agents_cooled() {
4387        cooldown::reset_global_state().await;
4388        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4389        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4390        let task_manager = Arc::new(TaskManager::with_store(
4391            backend,
4392            Arc::clone(&store),
4393            "active/repo".to_string(),
4394        ));
4395        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4396        // RouterConfig::default() has no "review" model map entries, so the one
4397        // available agent has no healthy model for "review" — all cooled.
4398        let router = Router::new_for_test(RouterConfig::default(), vec!["claude".to_string()]);
4399
4400        let id = store
4401            .upsert_external(&crate::store::UpsertExternal {
4402                repo: "inactive/repo",
4403                ext_id: "494",
4404                title: "Stale needs_review while agents cooled",
4405                body: "",
4406                author: "",
4407                url: "",
4408                labels: &[],
4409                origin: "github",
4410            })
4411            .await
4412            .unwrap();
4413        store
4414            .update_status(id, TaskStatus::NeedsReview)
4415            .await
4416            .unwrap();
4417        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4418            .bind(id)
4419            .execute(store.pool())
4420            .await
4421            .unwrap();
4422
4423        refire_and_escalate_stale_needs_review_global(
4424            &task_manager,
4425            &store,
4426            &router,
4427            &dispatching,
4428            &sweep_repos(),
4429        )
4430        .await
4431        .unwrap();
4432
4433        let task = store.get(id).await.unwrap();
4434        assert_eq!(
4435            task.needs_review_refires, 0,
4436            "counter must not be touched while cooled"
4437        );
4438        assert_eq!(task.status, TaskStatus::NeedsReview);
4439    }
4440
4441    /// With no available review agent at all, the sweep must not count re-fires
4442    /// or escalate to Blocked.
4443    #[serial(cooldown_state)]
4444    #[tokio::test]
4445    async fn global_refire_sweep_skips_when_no_reviewer_available() {
4446        cooldown::reset_global_state().await;
4447        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4448        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4449        let task_manager = Arc::new(TaskManager::with_store(
4450            backend,
4451            Arc::clone(&store),
4452            "active/repo".to_string(),
4453        ));
4454        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4455        let router = Router::new_for_test(RouterConfig::default(), vec![]);
4456
4457        let id = store
4458            .upsert_external(&crate::store::UpsertExternal {
4459                repo: "some/repo",
4460                ext_id: "495",
4461                title: "needs_review with no reviewer",
4462                body: "",
4463                author: "",
4464                url: "",
4465                labels: &[],
4466                origin: "github",
4467            })
4468            .await
4469            .unwrap();
4470        store
4471            .update_status(id, TaskStatus::NeedsReview)
4472            .await
4473            .unwrap();
4474        store
4475            .set_fields(id, &[("needs_review_refires", serde_json::json!(5))])
4476            .await
4477            .unwrap();
4478        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4479            .bind(id)
4480            .execute(store.pool())
4481            .await
4482            .unwrap();
4483
4484        refire_and_escalate_stale_needs_review_global(
4485            &task_manager,
4486            &store,
4487            &router,
4488            &dispatching,
4489            &sweep_repos(),
4490        )
4491        .await
4492        .unwrap();
4493
4494        let task = store.get(id).await.unwrap();
4495        assert_eq!(task.status, TaskStatus::NeedsReview);
4496        assert_eq!(task.needs_review_refires, 5);
4497    }
4498
4499    /// Regression test for #3499: a task blocked with the rebroadcast-escalation
4500    /// reason after surviving one review round (`review_cycles > 0`) must be
4501    /// recovered back to NeedsReview once review agents are routable again.
4502    /// Previously the `review_cycles != 0` filter excluded these tasks, stranding
4503    /// them permanently even when agents recovered.
4504    #[serial(cooldown_state)]
4505    #[tokio::test]
4506    async fn auto_recover_rebroadcast_blocked_task_after_review_cycle() {
4507        cooldown::reset_global_state().await;
4508        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4509        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4510        let task_manager = Arc::new(TaskManager::with_store(
4511            backend,
4512            Arc::clone(&store),
4513            "owner/repo".to_string(),
4514        ));
4515        let router = router_with_healthy_review_agent();
4516
4517        let id = store
4518            .upsert_external(&crate::store::UpsertExternal {
4519                repo: "owner/repo",
4520                ext_id: "511",
4521                title: "Rebroadcast blocked after review",
4522                body: "",
4523                author: "",
4524                url: "",
4525                labels: &[],
4526                origin: "github",
4527            })
4528            .await
4529            .unwrap();
4530        store.update_status(id, TaskStatus::Blocked).await.unwrap();
4531        store
4532            .set_block_reason(
4533                id,
4534                Some("review agent rebroadcast escalated after repeated retries"),
4535            )
4536            .await
4537            .unwrap();
4538        store
4539            .set_fields(id, &[("review_cycles", serde_json::json!(1))])
4540            .await
4541            .unwrap();
4542        store
4543            .set_fields(id, &[("needs_review_refires", serde_json::json!(6))])
4544            .await
4545            .unwrap();
4546        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4547            .bind(id)
4548            .execute(store.pool())
4549            .await
4550            .unwrap();
4551
4552        auto_recover_rebroadcast_blocked_tasks_global(
4553            &router,
4554            &task_manager,
4555            &store,
4556            &sweep_repos(),
4557        )
4558        .await
4559        .unwrap();
4560
4561        let task = store.get(id).await.unwrap();
4562        assert_eq!(
4563            task.status,
4564            TaskStatus::NeedsReview,
4565            "blocked rebroadcast task must be recovered to needs_review"
4566        );
4567        assert!(
4568            task.block_reason.is_none(),
4569            "block_reason must be cleared on recovery"
4570        );
4571        assert_eq!(
4572            task.needs_review_refires, 0,
4573            "needs_review_refires must be reset to 0 on recovery"
4574        );
4575    }
4576
4577    /// A fresh InReview task (<1 min old) should NOT be reset — the review
4578    /// agent may still be starting up.
4579    #[serial(cooldown_state)]
4580    #[tokio::test]
4581    async fn stale_in_review_recovery_skips_fresh_tasks() {
4582        cooldown::reset_global_state().await;
4583        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4584        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4585        let task_manager = Arc::new(TaskManager::with_store(
4586            Arc::clone(&backend),
4587            Arc::clone(&store),
4588            "owner/repo".to_string(),
4589        ));
4590        let tmux = Arc::new(TmuxManager::new());
4591        let router = Arc::new(RwLock::new(Router::new_for_test(
4592            RouterConfig::default(),
4593            vec!["claude".to_string()],
4594        )));
4595        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4596        let auto_merge_in_flight: Arc<DashSet<String>> = Arc::new(DashSet::new());
4597
4598        let id = store
4599            .upsert_external(&crate::store::UpsertExternal {
4600                repo: "owner/repo",
4601                ext_id: "66",
4602                title: "Fresh review",
4603                body: "",
4604                author: "",
4605                url: "",
4606                labels: &[],
4607                origin: "github",
4608            })
4609            .await
4610            .unwrap();
4611        store.update_status(id, TaskStatus::InReview).await.unwrap();
4612        store
4613            .set_fields(id, &[("branch", serde_json::json!("feature/66"))])
4614            .await
4615            .unwrap();
4616        // Don't backdate — task is fresh (just created)
4617
4618        Box::pin(sync_tick(
4619            &backend,
4620            &tmux,
4621            "owner/repo",
4622            &EngineConfig::default(),
4623            &router,
4624            &task_manager,
4625            &store,
4626            &dispatching,
4627            &auto_merge_in_flight,
4628        ))
4629        .await
4630        .unwrap();
4631
4632        // Should stay InReview — too young to be considered orphaned
4633        let task = store.get(id).await.unwrap();
4634        assert_eq!(task.status, TaskStatus::InReview);
4635    }
4636
4637    #[serial(cooldown_state)]
4638    #[tokio::test]
4639    async fn auto_unblock_routes_recoverable_failure() {
4640        cooldown::reset_global_state().await;
4641        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4642        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4643        let task_manager = Arc::new(TaskManager::with_store(
4644            backend,
4645            Arc::clone(&store),
4646            "owner/repo".to_string(),
4647        ));
4648        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4649
4650        let id = store
4651            .upsert_external(&crate::store::UpsertExternal {
4652                repo: "owner/repo",
4653                ext_id: "77",
4654                title: "Blocked task",
4655                body: "",
4656                author: "",
4657                url: "",
4658                labels: &[],
4659                origin: "github",
4660            })
4661            .await
4662            .unwrap();
4663        store.update_status(id, TaskStatus::Blocked).await.unwrap();
4664        store
4665            .set_fields(
4666                id,
4667                &[
4668                    ("agent", serde_json::json!("codex")),
4669                    ("model", serde_json::json!("gpt-5")),
4670                ],
4671            )
4672            .await
4673            .unwrap();
4674
4675        let run_id = store
4676            .start_run(&crate::store::StartRun {
4677                task_id: id,
4678                attempt: 1,
4679                run_type: "agent",
4680                agent: "codex",
4681                model: "gpt-5",
4682                command: "codex --model gpt-5",
4683                prompt: "do thing",
4684            })
4685            .await
4686            .unwrap();
4687        store
4688            .complete_run(&crate::store::CompleteRun {
4689                run_id,
4690                exit_code: Some(1),
4691                stdout: "",
4692                stderr: "",
4693                parsed: "",
4694                outcome: "rate_limit",
4695                error: "rate limit exceeded",
4696                tokens: RunTokenUsage::default(),
4697            })
4698            .await
4699            .unwrap();
4700
4701        auto_unblock_blocked_tasks(
4702            "owner/repo",
4703            &task_manager,
4704            &store,
4705            &dispatching,
4706            &Arc::new(DashSet::new()),
4707        )
4708        .await
4709        .unwrap();
4710
4711        let task = store.get(id).await.unwrap();
4712        assert_eq!(task.status, TaskStatus::New);
4713        assert!(task.agent.is_none());
4714        assert!(task.model.is_none());
4715        assert_eq!(task.auto_unblock_count, 1);
4716        assert_ne!(task.auto_unblock_last_at, "");
4717        assert_eq!(task.auto_unblock_last_reason, "RateLimit");
4718    }
4719
4720    #[serial(cooldown_state)]
4721    #[tokio::test]
4722    async fn auto_unblock_skips_manual_block() {
4723        cooldown::reset_global_state().await;
4724        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4725        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4726        let task_manager = Arc::new(TaskManager::with_store(
4727            backend,
4728            Arc::clone(&store),
4729            "owner/repo".to_string(),
4730        ));
4731        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4732
4733        let id = store
4734            .upsert_external(&crate::store::UpsertExternal {
4735                repo: "owner/repo",
4736                ext_id: "88",
4737                title: "Manually blocked",
4738                body: "",
4739                author: "",
4740                url: "",
4741                labels: &[],
4742                origin: "github",
4743            })
4744            .await
4745            .unwrap();
4746        store.update_status(id, TaskStatus::Blocked).await.unwrap();
4747        store
4748            .set_block_reason(id, Some("waiting on input"))
4749            .await
4750            .unwrap();
4751
4752        let run_id = store
4753            .start_run(&crate::store::StartRun {
4754                task_id: id,
4755                attempt: 1,
4756                run_type: "agent",
4757                agent: "codex",
4758                model: "gpt-5",
4759                command: "codex --model gpt-5",
4760                prompt: "do thing",
4761            })
4762            .await
4763            .unwrap();
4764        store
4765            .complete_run(&crate::store::CompleteRun {
4766                run_id,
4767                exit_code: Some(1),
4768                stdout: "",
4769                stderr: "",
4770                parsed: "",
4771                outcome: "rate_limit",
4772                error: "rate limit exceeded",
4773                tokens: RunTokenUsage::default(),
4774            })
4775            .await
4776            .unwrap();
4777
4778        auto_unblock_blocked_tasks(
4779            "owner/repo",
4780            &task_manager,
4781            &store,
4782            &dispatching,
4783            &Arc::new(DashSet::new()),
4784        )
4785        .await
4786        .unwrap();
4787
4788        let task = store.get(id).await.unwrap();
4789        assert_eq!(task.status, TaskStatus::Blocked);
4790        assert_eq!(task.auto_unblock_count, 0);
4791    }
4792
4793    // ── FailureCategory::as_str ─────────────────────────────────────────
4794
4795    #[serial(cooldown_state)]
4796    #[test]
4797    fn failure_category_as_str() {
4798        // as_str() must return a stable, human-readable string used as the
4799        // reason key in auto_unblock tracking (stored in auto_unblock_last_reason).
4800        assert_eq!(FailureCategory::RateLimit.as_str(), "RateLimit");
4801        assert_eq!(FailureCategory::Timeout.as_str(), "Timeout");
4802        assert_eq!(
4803            FailureCategory::ModelUnavailable.as_str(),
4804            "ModelUnavailable"
4805        );
4806        assert_eq!(FailureCategory::MaxAttempts.as_str(), "MaxAttempts");
4807        assert_eq!(FailureCategory::Unknown.as_str(), "Unknown");
4808        assert_eq!(FailureCategory::SilentExit0.as_str(), "SilentExit0");
4809        assert_eq!(FailureCategory::ConnectionError.as_str(), "ConnectionError");
4810        assert_eq!(FailureCategory::CliFlagError.as_str(), "CliFlagError");
4811        assert_eq!(
4812            FailureCategory::AllAgentsExhausted.as_str(),
4813            "AllAgentsExhausted"
4814        );
4815        assert_eq!(FailureCategory::ParseError.as_str(), "ParseError");
4816        assert_eq!(FailureCategory::FalseFailure.as_str(), "FalseFailure");
4817        assert_eq!(FailureCategory::PushFailed.as_str(), "PushFailed");
4818        assert_eq!(FailureCategory::PrCreateFailed.as_str(), "PrCreateFailed");
4819    }
4820
4821    // ── classify_failure: ModelUnavailable ──────────────────────────────
4822
4823    #[serial(cooldown_state)]
4824    #[test]
4825    fn classify_failure_model_unavailable_phrase_is_recoverable() {
4826        let category = classify_failure(
4827            "model unavailable (anthropic/claude-sonnet-4-6): Model not found: anthropic/claude-sonnet-4-6",
4828            "error",
4829        );
4830        assert_eq!(
4831            category,
4832            FailureCategory::ModelUnavailable,
4833            "\"model unavailable\" error must classify as ModelUnavailable"
4834        );
4835        assert!(
4836            category.is_recoverable(),
4837            "ModelUnavailable must be recoverable so auto-unblock can re-route"
4838        );
4839    }
4840
4841    // ── classify_failure: CreditExhausted (billing cycle) ──────────────
4842    #[serial(cooldown_state)]
4843    #[test]
4844    fn classify_failure_kimi_billing_cycle_is_credit_exhausted_not_rate_limit() {
4845        // Regression test for issue #3529: Kimi's billing-cycle exhaustion message
4846        // contains "usage limit", which also matches the generic rate-limit content
4847        // check. Credit exhaustion must be checked first so the failure is
4848        // categorized as CreditExhausted, not RateLimit, even when the stored
4849        // outcome is still "rate_limit" (e.g. from a task_run recorded before this
4850        // fix landed).
4851        let error = "kimi rate limit: Failed to authenticate. API Error: 403 You've reached \
4852                      your usage limit for this billing cycle. Your quota will be refreshed soon.";
4853        let category = classify_failure(error, "rate_limit");
4854        assert_eq!(
4855            category,
4856            FailureCategory::CreditExhausted,
4857            "billing-cycle exhaustion must classify as CreditExhausted, not RateLimit"
4858        );
4859        assert!(category.is_recoverable());
4860    }
4861
4862    #[serial(cooldown_state)]
4863    #[test]
4864    fn classify_failure_billing_cycle_exhausted_outcome_is_credit_exhausted() {
4865        // New task_runs store the specific "billing_cycle_exhausted" outcome
4866        // (post-fix). classify_failure must recognize it directly too.
4867        let category = classify_failure("some billing message", "billing_cycle_exhausted");
4868        assert_eq!(category, FailureCategory::CreditExhausted);
4869    }
4870
4871    #[serial(cooldown_state)]
4872    #[test]
4873    fn classify_failure_generic_rate_limit_is_still_rate_limit() {
4874        // Genuine rate limits (no billing-cycle/credit-exhaustion markers) must
4875        // remain RateLimit, not be swallowed by the reordered credit-exhaustion check.
4876        let category = classify_failure("429 Too Many Requests", "rate_limit");
4877        assert_eq!(category, FailureCategory::RateLimit);
4878    }
4879
4880    // ── classify_failure: PrCreateFailed ──────────────────────────────
4881    #[serial(cooldown_state)]
4882    #[test]
4883    fn classify_failure_pr_create_failed_specific_patterns() {
4884        // These should be classified as PrCreateFailed
4885        assert_eq!(
4886            classify_failure("create pr failed", ""),
4887            FailureCategory::PrCreateFailed
4888        );
4889        assert_eq!(
4890            classify_failure("failed to create pull request", ""),
4891            FailureCategory::PrCreateFailed
4892        );
4893        assert_eq!(
4894            classify_failure("pull request creation failed", ""),
4895            FailureCategory::PrCreateFailed
4896        );
4897
4898        // These should NOT be classified as PrCreateFailed (they were before the fix)
4899        assert_ne!(
4900            classify_failure("network failed while fetching pull request status", ""),
4901            FailureCategory::PrCreateFailed
4902        );
4903        assert_ne!(
4904            classify_failure("connection failed, retrying pull request check", ""),
4905            FailureCategory::PrCreateFailed
4906        );
4907        assert_ne!(
4908            classify_failure("pull request created but failed to merge", ""),
4909            FailureCategory::PrCreateFailed
4910        );
4911    }
4912
4913    #[serial(cooldown_state)]
4914    #[test]
4915    fn classify_failure_model_not_found_is_recoverable() {
4916        let category = classify_failure("Model not found: gpt-5-ultra", "error");
4917        assert_eq!(
4918            category,
4919            FailureCategory::ModelUnavailable,
4920            "\"model not found\" error must classify as ModelUnavailable"
4921        );
4922        assert!(category.is_recoverable());
4923    }
4924
4925    #[serial(cooldown_state)]
4926    #[test]
4927    fn classify_failure_model_does_not_exist_is_recoverable() {
4928        let category = classify_failure("The model `claude-opus-99` does not exist", "error");
4929        assert_eq!(category, FailureCategory::ModelUnavailable);
4930        assert!(category.is_recoverable());
4931    }
4932
4933    #[serial(cooldown_state)]
4934    #[tokio::test]
4935    async fn auto_unblock_routes_task_blocked_by_model_unavailable() {
4936        cooldown::reset_global_state().await;
4937        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4938        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
4939        let task_manager = Arc::new(TaskManager::with_store(
4940            backend,
4941            Arc::clone(&store),
4942            "owner/repo".to_string(),
4943        ));
4944        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4945
4946        let id = store
4947            .upsert_external(&crate::store::UpsertExternal {
4948                repo: "owner/repo",
4949                ext_id: "1202",
4950                title: "Task blocked by model unavailable",
4951                body: "",
4952                author: "",
4953                url: "",
4954                labels: &[],
4955                origin: "github",
4956            })
4957            .await
4958            .unwrap();
4959        store.update_status(id, TaskStatus::Blocked).await.unwrap();
4960
4961        let run_id = store
4962            .start_run(&crate::store::StartRun {
4963                task_id: id,
4964                attempt: 1,
4965                run_type: "agent",
4966                agent: "opencode",
4967                model: "anthropic/claude-sonnet-4-6",
4968                command: "opencode",
4969                prompt: "fix the bug",
4970            })
4971            .await
4972            .unwrap();
4973        store
4974            .complete_run(&crate::store::CompleteRun {
4975                run_id,
4976                exit_code: Some(1),
4977                stdout: "",
4978                stderr: "",
4979                parsed: "",
4980                outcome: "error",
4981                error: "model unavailable (anthropic/claude-sonnet-4-6): Model not found: anthropic/claude-sonnet-4-6",
4982                tokens: RunTokenUsage::default(),
4983            })
4984            .await
4985            .unwrap();
4986
4987        auto_unblock_blocked_tasks(
4988            "owner/repo",
4989            &task_manager,
4990            &store,
4991            &dispatching,
4992            &Arc::new(DashSet::new()),
4993        )
4994        .await
4995        .unwrap();
4996
4997        let task = store.get(id).await.unwrap();
4998        assert_eq!(
4999            task.status,
5000            TaskStatus::New,
5001            "task blocked by model unavailable must be auto-unblocked and re-routed"
5002        );
5003        assert_eq!(task.auto_unblock_count, 1);
5004        assert_eq!(task.auto_unblock_last_reason, "ModelUnavailable");
5005    }
5006
5007    // ── auto_unblock: reason-reset (regression for #1227) ─────────────────
5008
5009    #[serial(cooldown_state)]
5010    #[tokio::test]
5011    async fn auto_unblock_resets_count_when_reason_changes() {
5012        cooldown::reset_global_state().await;
5013        // Regression test for #1227: when a task is blocked for a different failure
5014        // reason than the last auto-unblock, the counter should reset to 0 (immediate
5015        // retry) instead of accumulating across unrelated failures.
5016        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5017        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
5018        let task_manager = Arc::new(TaskManager::with_store(
5019            backend,
5020            Arc::clone(&store),
5021            "owner/repo".to_string(),
5022        ));
5023        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
5024
5025        let id = store
5026            .upsert_external(&crate::store::UpsertExternal {
5027                repo: "owner/repo",
5028                ext_id: "1227",
5029                title: "Task with different failure reason",
5030                body: "",
5031                author: "",
5032                url: "",
5033                labels: &[],
5034                origin: "github",
5035            })
5036            .await
5037            .unwrap();
5038        store.update_status(id, TaskStatus::Blocked).await.unwrap();
5039        // Simulate: task was previously auto-unblocked for ModelUnavailable (count=1)
5040        store
5041            .set_fields(
5042                id,
5043                &[
5044                    ("auto_unblock_count", serde_json::json!(1)),
5045                    (
5046                        "auto_unblock_last_reason",
5047                        serde_json::json!("ModelUnavailable"),
5048                    ),
5049                ],
5050            )
5051            .await
5052            .unwrap();
5053
5054        // Add a run with a DIFFERENT failure: RateLimit
5055        let run_id = store
5056            .start_run(&crate::store::StartRun {
5057                task_id: id,
5058                attempt: 2,
5059                run_type: "agent",
5060                agent: "claude",
5061                model: "opus",
5062                command: "claude",
5063                prompt: "fix the bug",
5064            })
5065            .await
5066            .unwrap();
5067        store
5068            .complete_run(&crate::store::CompleteRun {
5069                run_id,
5070                exit_code: Some(1),
5071                stdout: "",
5072                stderr: "",
5073                parsed: "",
5074                outcome: "error",
5075                error: "rate limit reached for model opus",
5076                tokens: RunTokenUsage::default(),
5077            })
5078            .await
5079            .unwrap();
5080
5081        auto_unblock_blocked_tasks(
5082            "owner/repo",
5083            &task_manager,
5084            &store,
5085            &dispatching,
5086            &Arc::new(DashSet::new()),
5087        )
5088        .await
5089        .unwrap();
5090
5091        let task = store.get(id).await.unwrap();
5092        assert_eq!(
5093            task.status,
5094            TaskStatus::New,
5095            "task must be auto-unblocked even with count=1 from a different reason"
5096        );
5097        // Count is reset to 0, then incremented to 1 for the new reason
5098        assert_eq!(task.auto_unblock_count, 1);
5099        assert_eq!(
5100            task.auto_unblock_last_reason, "RateLimit",
5101            "reason must be updated to the new failure category"
5102        );
5103    }
5104
5105    /**
5106     * A task blocked for "exceeded max attempts" has already run the configured number
5107     * of times. Auto-unblock must leave it alone — flipping it back to new/needs_review
5108     * on the next tick restarted the very loop that produced the block (#3783)
5109     **/
5110    #[serial(cooldown_state)]
5111    #[tokio::test]
5112    async fn auto_unblock_leaves_max_attempts_blocked_task_alone() {
5113        cooldown::reset_global_state().await;
5114        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5115        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
5116        let task_manager = Arc::new(TaskManager::with_store(
5117            backend,
5118            Arc::clone(&store),
5119            "owner/repo".to_string(),
5120        ));
5121        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
5122
5123        let id = store
5124            .upsert_external(&crate::store::UpsertExternal {
5125                repo: "owner/repo",
5126                ext_id: "3783",
5127                title: "Approved work sent back to the agent",
5128                body: "",
5129                author: "",
5130                url: "",
5131                labels: &[],
5132                origin: "github",
5133            })
5134            .await
5135            .unwrap();
5136        store.update_status(id, TaskStatus::Blocked).await.unwrap();
5137
5138        let run_id = store
5139            .start_run(&crate::store::StartRun {
5140                task_id: id,
5141                attempt: 11,
5142                run_type: "agent",
5143                agent: "claude",
5144                model: "opus",
5145                command: "claude",
5146                prompt: "fix the bug",
5147            })
5148            .await
5149            .unwrap();
5150        store
5151            .complete_run(&crate::store::CompleteRun {
5152                run_id,
5153                exit_code: Some(1),
5154                stdout: "",
5155                stderr: "",
5156                parsed: "",
5157                outcome: "error",
5158                error: "exceeded max attempts (11/10). Use `/retry` to reset.",
5159                tokens: RunTokenUsage::default(),
5160            })
5161            .await
5162            .unwrap();
5163
5164        auto_unblock_blocked_tasks(
5165            "owner/repo",
5166            &task_manager,
5167            &store,
5168            &dispatching,
5169            &Arc::new(DashSet::new()),
5170        )
5171        .await
5172        .unwrap();
5173
5174        let task = store.get(id).await.unwrap();
5175        assert_eq!(
5176            task.status,
5177            TaskStatus::Blocked,
5178            "max-attempts block must survive the next tick until a person acts"
5179        );
5180        assert_eq!(
5181            task.auto_unblock_count, 0,
5182            "no auto-unblock attempt should be recorded"
5183        );
5184    }
5185
5186    #[serial(cooldown_state)]
5187    #[tokio::test]
5188    async fn auto_unblock_stores_reason_on_first_unblock() {
5189        cooldown::reset_global_state().await;
5190        // When a task is auto-unblocked for the first time, the failure reason
5191        // should be stored so subsequent blocks can be compared.
5192        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5193        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
5194        let task_manager = Arc::new(TaskManager::with_store(
5195            backend,
5196            Arc::clone(&store),
5197            "owner/repo".to_string(),
5198        ));
5199        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
5200
5201        let id = store
5202            .upsert_external(&crate::store::UpsertExternal {
5203                repo: "owner/repo",
5204                ext_id: "1227b",
5205                title: "Task for first-time reason storage",
5206                body: "",
5207                author: "",
5208                url: "",
5209                labels: &[],
5210                origin: "github",
5211            })
5212            .await
5213            .unwrap();
5214        store.update_status(id, TaskStatus::Blocked).await.unwrap();
5215
5216        let run_id = store
5217            .start_run(&crate::store::StartRun {
5218                task_id: id,
5219                attempt: 1,
5220                run_type: "agent",
5221                agent: "claude",
5222                model: "opus",
5223                command: "claude",
5224                prompt: "fix the bug",
5225            })
5226            .await
5227            .unwrap();
5228        store
5229            .complete_run(&crate::store::CompleteRun {
5230                run_id,
5231                exit_code: Some(1),
5232                stdout: "",
5233                stderr: "",
5234                parsed: "",
5235                outcome: "timeout",
5236                error: "task timed out",
5237                tokens: RunTokenUsage::default(),
5238            })
5239            .await
5240            .unwrap();
5241
5242        auto_unblock_blocked_tasks(
5243            "owner/repo",
5244            &task_manager,
5245            &store,
5246            &dispatching,
5247            &Arc::new(DashSet::new()),
5248        )
5249        .await
5250        .unwrap();
5251
5252        let task = store.get(id).await.unwrap();
5253        assert_eq!(task.status, TaskStatus::New);
5254        assert_eq!(task.auto_unblock_count, 1);
5255        assert_eq!(
5256            task.auto_unblock_last_reason, "Timeout",
5257            "reason must be stored on first auto-unblock"
5258        );
5259    }
5260
5261    // ── classify_failure: sparse checkout (regression for substring false positive) ──
5262
5263    #[serial(cooldown_state)]
5264    #[test]
5265    fn classify_failure_sparse_checkout_not_parse_error() {
5266        // Regression test for #1204: "sparse" substring contains "parse",
5267        // so overly broad substring match would misclassify git sparse-checkout errors
5268        let category = classify_failure("error: cannot reset paths in a sparse checkout", "error");
5269        assert_ne!(
5270            category,
5271            FailureCategory::ParseError,
5272            "sparse checkout errors must NOT be classified as ParseError"
5273        );
5274        assert_eq!(
5275            category,
5276            FailureCategory::Unknown,
5277            "sparse checkout errors should be Unknown (not recoverable auto-unblock)"
5278        );
5279    }
5280
5281    #[serial(cooldown_state)]
5282    #[test]
5283    fn classify_failure_sparse_index_not_parse_error() {
5284        // Another sparse-checkout variant that contains "parse"
5285        let category = classify_failure("failed due to sparse index configuration", "error");
5286        assert_ne!(category, FailureCategory::ParseError);
5287        assert_eq!(category, FailureCategory::Unknown);
5288    }
5289
5290    // ── classify_failure: truncated (issue #3473) ───────────────────────
5291
5292    /// A review output truncated by the model's token budget (opencode
5293    /// step_finish reason=length) must classify as recoverable, so a task
5294    /// blocked after repeated truncations can be auto-unblocked instead of
5295    /// requiring manual intervention.
5296    #[serial(cooldown_state)]
5297    #[test]
5298    fn classify_failure_truncated_is_recoverable() {
5299        let category = classify_failure(
5300            "truncated: review output was cut off before completion \
5301             (output/reasoning token budget exceeded)",
5302            "truncated",
5303        );
5304        assert_eq!(
5305            category,
5306            FailureCategory::ParseError,
5307            "truncated outcome must classify as a recoverable parse-style failure"
5308        );
5309        assert!(
5310            category.is_recoverable(),
5311            "truncated failures must be recoverable so auto-unblock can retry"
5312        );
5313    }
5314
5315    #[serial(cooldown_state)]
5316    #[test]
5317    fn dispatching_key_includes_repo_for_cross_project_isolation() {
5318        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
5319
5320        // Lock task 42 in repo A
5321        let key_a = "owner/repo-a/42".to_string();
5322        dispatching.insert(key_a.clone(), "42".to_string());
5323
5324        // Same task ID in repo B should NOT be blocked
5325        let key_b = "owner/repo-b/42".to_string();
5326        assert!(dispatching.contains_key(&key_a));
5327        assert!(
5328            !dispatching.contains_key(&key_b),
5329            "same task ID in different repo should not be blocked"
5330        );
5331    }
5332
5333    #[serial(cooldown_state)]
5334    #[tokio::test]
5335    async fn emit_degraded_agents_writes_metric_when_three_or_more_agents_degraded() {
5336        cooldown::reset_global_state().await;
5337
5338        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5339
5340        // Create a router and set deterministic available agents
5341        let mut config = RouterConfig::default();
5342        config
5343            .model_map
5344            .entry("medium".to_string())
5345            .or_default()
5346            .insert(
5347                "test-agent-3a".to_string(),
5348                vec!["test-model-a".to_string()],
5349            );
5350        config
5351            .model_map
5352            .entry("medium".to_string())
5353            .or_default()
5354            .insert(
5355                "test-agent-3b".to_string(),
5356                vec!["test-model-b".to_string()],
5357            );
5358        config
5359            .model_map
5360            .entry("medium".to_string())
5361            .or_default()
5362            .insert(
5363                "test-agent-3c".to_string(),
5364                vec!["test-model-c".to_string()],
5365            );
5366        config
5367            .model_map
5368            .entry("medium".to_string())
5369            .or_default()
5370            .insert(
5371                "test-agent-3d".to_string(),
5372                vec!["test-model-d".to_string()],
5373            );
5374        let mut router = Router::new(config);
5375        router.available_agents = vec![
5376            "test-agent-3a".to_string(),
5377            "test-agent-3b".to_string(),
5378            "test-agent-3c".to_string(),
5379            "test-agent-3d".to_string(),
5380        ];
5381
5382        // Place three agents into cooldown
5383        set_agent_cooldown("test-agent-3a", 3600).await;
5384        set_agent_cooldown("test-agent-3b", 3600).await;
5385        set_agent_cooldown("test-agent-3c", 3600).await;
5386
5387        // Call the helper and assert KV metrics written
5388        emit_degraded_agents_if_needed(&router.available_agents, &router.config, Some(&store))
5389            .await;
5390
5391        // Count metric always written
5392        let count_val = store
5393            .kv_get("metrics:orch.agents_degraded.count")
5394            .await
5395            .unwrap();
5396        assert_eq!(count_val.as_deref(), Some("3"));
5397
5398        // Alert metric set to "1" when threshold crossed
5399        let alert_val = store
5400            .kv_get("metrics:orch.agents_degraded.alert")
5401            .await
5402            .unwrap();
5403        assert_eq!(alert_val.as_deref(), Some("1"));
5404    }
5405
5406    #[serial(cooldown_state)]
5407    #[tokio::test]
5408    async fn emit_degraded_agents_clears_alert_when_below_threshold() {
5409        cooldown::reset_global_state().await;
5410
5411        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5412
5413        let mut config = RouterConfig::default();
5414        config
5415            .model_map
5416            .entry("medium".to_string())
5417            .or_default()
5418            .insert(
5419                "test-agent-2a".to_string(),
5420                vec!["test-model-a".to_string()],
5421            );
5422        config
5423            .model_map
5424            .entry("medium".to_string())
5425            .or_default()
5426            .insert(
5427                "test-agent-2b".to_string(),
5428                vec!["test-model-b".to_string()],
5429            );
5430        let mut router = Router::new(config);
5431        router.available_agents = vec!["test-agent-2a".to_string(), "test-agent-2b".to_string()];
5432
5433        // Place only two agents into cooldown — below the threshold of 3
5434        set_agent_cooldown("test-agent-2a", 3600).await;
5435        set_agent_cooldown("test-agent-2b", 3600).await;
5436
5437        emit_degraded_agents_if_needed(&router.available_agents, &router.config, Some(&store))
5438            .await;
5439
5440        // Count metric still written (always-emit)
5441        let count_val = store
5442            .kv_get("metrics:orch.agents_degraded.count")
5443            .await
5444            .unwrap();
5445        assert_eq!(count_val.as_deref(), Some("2"));
5446
5447        // Alert metric cleared (set to "0") — no WARN should fire
5448        let alert_val = store
5449            .kv_get("metrics:orch.agents_degraded.alert")
5450            .await
5451            .unwrap();
5452        assert_eq!(alert_val.as_deref(), Some("0"));
5453
5454        let stale_pool_alert = store
5455            .kv_get("metrics:orch.model_pool_stale_persistent.alert")
5456            .await
5457            .unwrap();
5458        assert_eq!(stale_pool_alert.as_deref(), Some("0"));
5459    }
5460
5461    #[serial(cooldown_state)]
5462    #[tokio::test]
5463    async fn emit_degraded_agents_sets_stale_pool_alert_for_persistent_model_failures() {
5464        cooldown::reset_global_state().await;
5465
5466        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5467
5468        let mut config = RouterConfig::default();
5469        config
5470            .model_map
5471            .entry("medium".to_string())
5472            .or_default()
5473            .insert(
5474                "test-agent-stale".to_string(),
5475                vec!["model-a".to_string(), "model-b".to_string()],
5476            );
5477        let mut router = Router::new(config);
5478        router.available_agents = vec!["test-agent-stale".to_string()];
5479
5480        // 50% cooled + persistent reason should trigger stale-pool alert.
5481        record_persistent_model_failure("test-agent-stale", "model-a").await;
5482
5483        emit_degraded_agents_if_needed(&router.available_agents, &router.config, Some(&store))
5484            .await;
5485
5486        let stale_pool_alert = store
5487            .kv_get("metrics:orch.model_pool_stale_persistent.alert")
5488            .await
5489            .unwrap();
5490        assert_eq!(stale_pool_alert.as_deref(), Some("1"));
5491    }
5492
5493    // ── classify_comment tests ──────────────────────────────────────────
5494
5495    #[serial(cooldown_state)]
5496    #[test]
5497    fn classify_skip_already_processed() {
5498        let action = classify_comment(
5499            "@orch /retry",
5500            Some("https://api.github.com/repos/o/r/issues/1"),
5501            "@bot",
5502            true,
5503        );
5504        assert_eq!(action, CommentAction::Skip);
5505    }
5506
5507    #[serial(cooldown_state)]
5508    #[test]
5509    fn classify_skip_irrelevant_comment() {
5510        let action = classify_comment("just a regular comment", None, "@bot", false);
5511        assert_eq!(action, CommentAction::Skip);
5512    }
5513
5514    #[serial(cooldown_state)]
5515    #[test]
5516    fn classify_command_without_mention() {
5517        let action = classify_comment(
5518            "/retry",
5519            Some("https://api.github.com/repos/o/r/issues/42"),
5520            "@bot",
5521            false,
5522        );
5523        assert_eq!(
5524            action,
5525            CommentAction::ExecuteCommand {
5526                command: OwnerCommand::Retry,
5527                issue_num: "42".to_string(),
5528            }
5529        );
5530    }
5531
5532    #[serial(cooldown_state)]
5533    #[test]
5534    fn classify_command_with_mention() {
5535        let action = classify_comment(
5536            "@orch /close",
5537            Some("https://api.github.com/repos/o/r/issues/7"),
5538            "@bot",
5539            false,
5540        );
5541        assert_eq!(
5542            action,
5543            CommentAction::ExecuteCommandForMention {
5544                command: OwnerCommand::Close,
5545                issue_num: "7".to_string(),
5546            }
5547        );
5548    }
5549
5550    #[serial(cooldown_state)]
5551    #[test]
5552    fn classify_mention_without_command() {
5553        let action = classify_comment(
5554            "@orch can you look at this?",
5555            Some("https://api.github.com/repos/o/r/issues/99"),
5556            "@bot",
5557            false,
5558        );
5559        assert_eq!(
5560            action,
5561            CommentAction::CreateMentionTask {
5562                issue_num: Some("99".to_string()),
5563            }
5564        );
5565    }
5566
5567    #[serial(cooldown_state)]
5568    #[test]
5569    fn classify_mention_without_issue_url() {
5570        let action = classify_comment("@orch help me", None, "@bot", false);
5571        assert_eq!(action, CommentAction::CreateMentionTask { issue_num: None });
5572    }
5573
5574    #[serial(cooldown_state)]
5575    #[test]
5576    fn classify_command_no_mention_no_url_skips() {
5577        let action = classify_comment("/retry", None, "@bot", false);
5578        assert_eq!(action, CommentAction::Skip);
5579    }
5580
5581    #[serial(cooldown_state)]
5582    #[test]
5583    fn classify_mention_via_current_user() {
5584        let action = classify_comment(
5585            "@bot please help",
5586            Some("https://api.github.com/repos/o/r/issues/5"),
5587            "@bot",
5588            false,
5589        );
5590        assert_eq!(
5591            action,
5592            CommentAction::CreateMentionTask {
5593                issue_num: Some("5".to_string()),
5594            }
5595        );
5596    }
5597
5598    #[serial(cooldown_state)]
5599    #[test]
5600    fn classify_command_with_mention_no_url_creates_task() {
5601        // @mention + command but no issue URL → can't run command, create mention task
5602        let action = classify_comment("@orch /retry", None, "@bot", false);
5603        assert_eq!(action, CommentAction::CreateMentionTask { issue_num: None });
5604    }
5605
5606    #[serial(cooldown_state)]
5607    #[test]
5608    fn mention_cursor_stops_advancing_after_gap() {
5609        let mut cursor = MentionCursor::default();
5610        cursor.advance("2026-01-01T00:00:00Z");
5611        cursor.block_on_gap();
5612        cursor.advance("2026-01-02T00:00:00Z");
5613
5614        assert_eq!(
5615            cursor.into_last_safe_ts().as_deref(),
5616            Some("2026-01-01T00:00:00Z")
5617        );
5618    }
5619
5620    #[serial(cooldown_state)]
5621    #[test]
5622    fn mention_cursor_tracks_max_timestamp_until_gap() {
5623        let mut cursor = MentionCursor::default();
5624        cursor.advance("2026-01-01T00:00:00Z");
5625        cursor.advance("2026-01-01T00:05:00Z");
5626        cursor.block_on_gap();
5627        cursor.advance("2026-01-01T00:10:00Z");
5628
5629        assert_eq!(
5630            cursor.into_last_safe_ts().as_deref(),
5631            Some("2026-01-01T00:05:00Z")
5632        );
5633    }
5634
5635    // ── scan_comments: acknowledge_mention failure does not advance cursor ────
5636
5637    /// Backend that tracks acknowledge_mention calls and can be configured to fail them.
5638    struct AckTrackingBackend {
5639        ack_calls: Mutex<Vec<String>>,
5640        posted: Mutex<Vec<String>>,
5641        ack_failure: bool,
5642        inner: NoopBackend,
5643    }
5644
5645    impl AckTrackingBackend {
5646        fn new(ack_failure: bool) -> Arc<Self> {
5647            Arc::new(Self {
5648                ack_calls: Mutex::new(Vec::new()),
5649                posted: Mutex::new(Vec::new()),
5650                ack_failure,
5651                inner: NoopBackend,
5652            })
5653        }
5654
5655        fn fail_on_ack(&self) -> bool {
5656            self.ack_failure
5657        }
5658    }
5659
5660    #[async_trait]
5661    impl ExternalBackend for AckTrackingBackend {
5662        fn name(&self) -> &'static str {
5663            "ack-tracking"
5664        }
5665
5666        async fn acknowledge_mention(&self, comment_id: &str) -> anyhow::Result<()> {
5667            self.ack_calls.lock().await.push(comment_id.to_string());
5668            if self.fail_on_ack() {
5669                // Return a transient error to simulate network glitch.
5670                anyhow::bail!("network error: temporary failure posting reaction");
5671            }
5672            Ok(())
5673        }
5674
5675        // Override get_authenticated_user to ensure scan proceeds.
5676        async fn get_authenticated_user(&self) -> anyhow::Result<Option<String>> {
5677            Ok(Some("bot".into()))
5678        }
5679
5680        // Override get_mentions to return empty (we use prefetched comments).
5681        async fn get_mentions(&self, _since: &str) -> anyhow::Result<Vec<Mention>> {
5682            Ok(vec![])
5683        }
5684
5685        // Delegate everything else to NoopBackend.
5686        async fn create_task(&self, t: &str, b: &str, l: &[String]) -> anyhow::Result<ExternalId> {
5687            self.inner.create_task(t, b, l).await
5688        }
5689        async fn get_task(&self, id: &ExternalId) -> anyhow::Result<ExternalTask> {
5690            self.inner.get_task(id).await
5691        }
5692        async fn list_by_status(&self, s: Status) -> anyhow::Result<Vec<ExternalTask>> {
5693            self.inner.list_by_status(s).await
5694        }
5695        async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
5696            self.inner.list_routable().await
5697        }
5698        async fn post_comment(&self, id: &ExternalId, b: &str) -> anyhow::Result<()> {
5699            self.posted.lock().await.push(b.to_string());
5700            self.inner.post_comment(id, b).await
5701        }
5702        async fn set_labels(&self, id: &ExternalId, l: &[String]) -> anyhow::Result<()> {
5703            self.inner.set_labels(id, l).await
5704        }
5705        async fn remove_label(&self, id: &ExternalId, l: &str) -> anyhow::Result<()> {
5706            self.inner.remove_label(id, l).await
5707        }
5708        async fn get_sub_issues(&self, id: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
5709            self.inner.get_sub_issues(id).await
5710        }
5711        async fn create_sub_task(
5712            &self,
5713            p: &ExternalId,
5714            t: &str,
5715            b: &str,
5716            l: &[String],
5717        ) -> anyhow::Result<ExternalId> {
5718            self.inner.create_sub_task(p, t, b, l).await
5719        }
5720        async fn ensure_status_label(&self, l: &str) -> anyhow::Result<()> {
5721            self.inner.ensure_status_label(l).await
5722        }
5723        async fn has_open_issue_with_title(&self, t: &str, l: &str) -> anyhow::Result<bool> {
5724            self.inner.has_open_issue_with_title(t, l).await
5725        }
5726        async fn health_check(&self) -> anyhow::Result<()> {
5727            self.inner.health_check().await
5728        }
5729        async fn is_pr_merged(&self, b: &str) -> anyhow::Result<bool> {
5730            self.inner.is_pr_merged(b).await
5731        }
5732        async fn update_status(&self, id: &ExternalId, s: Status) -> anyhow::Result<()> {
5733            self.inner.update_status(id, s).await
5734        }
5735    }
5736
5737    /// When acknowledge_mention fails for CreateMentionTask, the cursor must NOT
5738    /// advance and no mention task should be created. The next tick will retry.
5739    #[serial(cooldown_state)]
5740    #[tokio::test]
5741    async fn create_mention_task_skips_on_ack_failure() {
5742        cooldown::reset_global_state().await;
5743        let backend: Arc<AckTrackingBackend> = AckTrackingBackend::new(true);
5744        let store: Arc<TaskStore> = Arc::new(TaskStore::open_memory().await.unwrap());
5745
5746        // Set an initial cursor so we can verify it wasn't advanced.
5747        let initial_ts = "2026-01-01T00:00:00Z";
5748        store
5749            .kv_set("mentions_last_checked", initial_ts)
5750            .await
5751            .unwrap();
5752
5753        let mention = Mention {
5754            id: "mention-abc".into(),
5755            body: "@bot can you help?".into(),
5756            author: "alice".into(),
5757            created_at: "2026-01-02T00:00:00Z".into(),
5758            issue_url: Some("https://api.github.com/repos/owner/repo/issues/42".into()),
5759        };
5760
5761        let backend_trait: Arc<dyn ExternalBackend> = Arc::<AckTrackingBackend>::clone(&backend);
5762        let task_manager = Arc::new(TaskManager::new(Arc::clone(&backend_trait)));
5763
5764        scan_comments(
5765            &backend_trait,
5766            Some(&store),
5767            "owner/repo",
5768            &task_manager,
5769            Some(&[mention]),
5770        )
5771        .await
5772        .unwrap();
5773
5774        // acknowledge_mention was called with the mention ID.
5775        let calls = backend.ack_calls.lock().await;
5776        assert_eq!(
5777            &**calls,
5778            &["mention-abc"],
5779            "acknowledge_mention must be called"
5780        );
5781
5782        // No mention task was created.
5783        let all = store.list_all("owner/repo").await.unwrap();
5784        assert!(
5785            all.is_empty(),
5786            "no task should be created when acknowledge_mention fails"
5787        );
5788
5789        // Cursor was NOT advanced.
5790        let cursor = store.kv_get("mentions_last_checked").await.unwrap();
5791        assert_eq!(
5792            cursor.as_deref(),
5793            Some(initial_ts),
5794            "cursor must not advance when acknowledge_mention fails"
5795        );
5796    }
5797
5798    /// When acknowledge_mention fails for ExecuteCommandForMention, handle_slash_command
5799    /// must NOT be called and no mention task should be created.
5800    #[serial(cooldown_state)]
5801    #[tokio::test]
5802    async fn execute_command_for_mention_skips_on_ack_failure() {
5803        cooldown::reset_global_state().await;
5804        let backend: Arc<AckTrackingBackend> = AckTrackingBackend::new(true);
5805        let store: Arc<TaskStore> = Arc::new(TaskStore::open_memory().await.unwrap());
5806
5807        let initial_ts = "2026-01-01T00:00:00Z";
5808        store
5809            .kv_set("mentions_last_checked", initial_ts)
5810            .await
5811            .unwrap();
5812
5813        // Mention with a slash command and issue URL → ExecuteCommandForMention.
5814        let mention = Mention {
5815            id: "mention-def".into(),
5816            body: "@bot /close".into(),
5817            author: "alice".into(),
5818            created_at: "2026-01-02T00:00:00Z".into(),
5819            issue_url: Some("https://api.github.com/repos/owner/repo/issues/42".into()),
5820        };
5821
5822        let backend_trait: Arc<dyn ExternalBackend> = Arc::<AckTrackingBackend>::clone(&backend);
5823        let task_manager = Arc::new(TaskManager::new(Arc::clone(&backend_trait)));
5824
5825        scan_comments(
5826            &backend_trait,
5827            Some(&store),
5828            "owner/repo",
5829            &task_manager,
5830            Some(&[mention]),
5831        )
5832        .await
5833        .unwrap();
5834
5835        // acknowledge_mention was called.
5836        let calls = backend.ack_calls.lock().await;
5837        assert_eq!(&**calls, &["mention-def"]);
5838
5839        // No mention task was created (handle_slash_command was not called).
5840        let all = store.list_all("owner/repo").await.unwrap();
5841        assert!(
5842            all.is_empty(),
5843            "no task should be created when acknowledge_mention fails for ExecuteCommandForMention"
5844        );
5845
5846        // Cursor was NOT advanced.
5847        let cursor = store.kv_get("mentions_last_checked").await.unwrap();
5848        assert_eq!(
5849            cursor.as_deref(),
5850            Some(initial_ts),
5851            "cursor must not advance when acknowledge_mention fails for ExecuteCommandForMention"
5852        );
5853    }
5854
5855    // ── slash commands retry after a transient GitHub error ──────────────────
5856
5857    /// GitHub stub whose `get_issue` fails once, then succeeds.
5858    #[derive(Default)]
5859    struct FlakyGh {
5860        get_issue_calls: AtomicUsize,
5861    }
5862
5863    #[async_trait]
5864    impl CommandGh for FlakyGh {
5865        async fn get_issue(&self, _repo: &str, number: &str) -> anyhow::Result<GitHubIssue> {
5866            if self.get_issue_calls.fetch_add(1, Ordering::SeqCst) == 0 {
5867                anyhow::bail!("network error: temporary failure fetching issue");
5868            }
5869            Ok(GitHubIssue {
5870                number: number.parse().unwrap_or(0),
5871                id: 1,
5872                title: "t".into(),
5873                body: None,
5874                state: "open".into(),
5875                labels: Vec::new(),
5876                user: GitHubUser {
5877                    login: "alice".into(),
5878                },
5879                created_at: String::new(),
5880                updated_at: String::new(),
5881                html_url: String::new(),
5882                node_id: None,
5883                pull_request: None,
5884                author_association: Some("OWNER".into()),
5885                state_reason: None,
5886            })
5887        }
5888
5889        async fn is_collaborator(&self, _repo: &str, _username: &str) -> anyhow::Result<bool> {
5890            Ok(true)
5891        }
5892
5893        async fn close_issue(&self, _repo: &str, _number: &str) -> anyhow::Result<()> {
5894            Ok(())
5895        }
5896    }
5897
5898    /// A transient `get_issue` failure must leave the cursor before the comment so
5899    /// the next scan retries it — and the command must then run exactly once.
5900    #[serial(cooldown_state)]
5901    #[tokio::test]
5902    async fn slash_command_retries_after_transient_fetch_failure() {
5903        cooldown::reset_global_state().await;
5904        let backend: Arc<AckTrackingBackend> = AckTrackingBackend::new(false);
5905        let store: Arc<TaskStore> = Arc::new(TaskStore::open_memory().await.unwrap());
5906        let initial_ts = "2026-01-01T00:00:00Z";
5907
5908        let mention = Mention {
5909            id: "mention-ghi".into(),
5910            body: "@bot /unblock".into(),
5911            author: "alice".into(),
5912            created_at: "2026-01-02T00:00:00Z".into(),
5913            issue_url: Some("https://api.github.com/repos/owner/repo/issues/42".into()),
5914        };
5915
5916        let backend_trait: Arc<dyn ExternalBackend> = Arc::<AckTrackingBackend>::clone(&backend);
5917        let task_manager = Arc::new(TaskManager::new(Arc::clone(&backend_trait)));
5918        let gh = FlakyGh::default();
5919        let mut cursor = MentionCursor::default();
5920
5921        // First scan: get_issue fails.
5922        let advanced_safely = handle_slash_command(
5923            &backend_trait,
5924            Some(&store),
5925            "owner/repo",
5926            &task_manager,
5927            &gh,
5928            &mention,
5929            &OwnerCommand::Unblock,
5930            &mut cursor,
5931        )
5932        .await
5933        .unwrap();
5934
5935        assert!(
5936            !advanced_safely,
5937            "a transient fetch failure must stop cursor advancement"
5938        );
5939        assert!(
5940            cursor.last_safe_ts.is_none(),
5941            "cursor must stay before the comment so the next tick retries it"
5942        );
5943        assert!(
5944            store.list_all("owner/repo").await.unwrap().is_empty(),
5945            "no mention task on a transient failure"
5946        );
5947        assert!(
5948            backend.posted.lock().await.is_empty(),
5949            "nothing is posted to GitHub on a transient failure"
5950        );
5951        assert_eq!(
5952            filter_mentions_by_since(slice::from_ref(&mention), initial_ts).len(),
5953            1,
5954            "the comment is still in range for the next scan"
5955        );
5956
5957        // Second scan: get_issue succeeds and the command runs.
5958        let mut cursor = MentionCursor::default();
5959        let advanced_safely = handle_slash_command(
5960            &backend_trait,
5961            Some(&store),
5962            "owner/repo",
5963            &task_manager,
5964            &gh,
5965            &mention,
5966            &OwnerCommand::Unblock,
5967            &mut cursor,
5968        )
5969        .await
5970        .unwrap();
5971
5972        assert!(advanced_safely);
5973        assert_eq!(
5974            cursor.into_last_safe_ts().as_deref(),
5975            Some("2026-01-02T00:00:00Z"),
5976            "cursor advances once the command actually runs"
5977        );
5978        assert_eq!(
5979            backend.posted.lock().await.len(),
5980            1,
5981            "the command runs once, posting a single confirmation"
5982        );
5983        assert_eq!(
5984            store.list_all("owner/repo").await.unwrap().len(),
5985            1,
5986            "one sentinel mention task for the executed command"
5987        );
5988    }
5989
5990    /// In the plain `ExecuteCommand` arm the GitHub client points at a dead port
5991    /// under `cfg(test)`, so `get_issue` always fails — the cursor must stay put.
5992    #[serial(cooldown_state)]
5993    #[tokio::test]
5994    async fn execute_command_keeps_cursor_on_fetch_failure() {
5995        cooldown::reset_global_state().await;
5996        let backend: Arc<AckTrackingBackend> = AckTrackingBackend::new(false);
5997        let store: Arc<TaskStore> = Arc::new(TaskStore::open_memory().await.unwrap());
5998
5999        let initial_ts = "2026-01-01T00:00:00Z";
6000        store
6001            .kv_set("mentions_last_checked", initial_ts)
6002            .await
6003            .unwrap();
6004
6005        // Slash command without a mention → ExecuteCommand.
6006        let comment = Mention {
6007            id: "comment-jkl".into(),
6008            body: "/unblock".into(),
6009            author: "alice".into(),
6010            created_at: "2026-01-02T00:00:00Z".into(),
6011            issue_url: Some("https://api.github.com/repos/owner/repo/issues/42".into()),
6012        };
6013
6014        let backend_trait: Arc<dyn ExternalBackend> = Arc::<AckTrackingBackend>::clone(&backend);
6015        let task_manager = Arc::new(TaskManager::new(Arc::clone(&backend_trait)));
6016
6017        scan_comments(
6018            &backend_trait,
6019            Some(&store),
6020            "owner/repo",
6021            &task_manager,
6022            Some(&[comment]),
6023        )
6024        .await
6025        .unwrap();
6026
6027        assert!(
6028            backend.posted.lock().await.is_empty(),
6029            "nothing is posted when the issue fetch fails"
6030        );
6031        assert_eq!(
6032            store
6033                .kv_get("mentions_last_checked")
6034                .await
6035                .unwrap()
6036                .as_deref(),
6037            Some(initial_ts),
6038            "cursor must not advance past a command that never ran"
6039        );
6040    }
6041
6042    // ── is_ci_failure_block ────────────────────────────────────────────────
6043
6044    fn make_task(block_reason: Option<String>) -> Task {
6045        Task {
6046            id: 1,
6047            external_id: None,
6048            repo: "owner/repo".to_string(),
6049            origin: String::new(),
6050            title: String::new(),
6051            body: String::new(),
6052            status: TaskStatus::Blocked,
6053            source: String::new(),
6054            source_id: String::new(),
6055            author: String::new(),
6056            url: String::new(),
6057            labels: vec![],
6058            agent: None,
6059            model: None,
6060            complexity: String::new(),
6061            estimate: 0,
6062            route_reason: String::new(),
6063            agent_profile: String::new(),
6064            selected_skills: String::new(),
6065            route_attempts: 0,
6066            attempts: 0,
6067            branch: String::new(),
6068            worktree: String::new(),
6069            worktree_cleaned: false,
6070            owner_node: None,
6071            summary: String::new(),
6072            last_error: String::new(),
6073            parent_id: None,
6074            block_reason,
6075            pr_number: Some(42),
6076            pr_review_context: String::new(),
6077            last_review_ts: String::new(),
6078            review_ts_map: String::new(),
6079            last_comment_review_ts: String::new(),
6080            merge_conflict_retries: 0,
6081            ci_merge_failures: 0,
6082            pr_create_failures: 0,
6083            push_failures: 0,
6084            network_retries: 0,
6085            timeout_retries: 0,
6086            review_agent_failures: 0,
6087            review_resets: 0,
6088            review_cycles: 0,
6089            review_invocations: 0,
6090            needs_review_refires: 0,
6091            input_tokens: 0,
6092            output_tokens: 0,
6093            total_cost_usd: 0.0,
6094            model_reroute_chain: String::new(),
6095            limit_reroute_chain: String::new(),
6096            memory: vec![],
6097            delegations: vec![],
6098            auto_unblock_count: 0,
6099            auto_unblock_last_at: String::new(),
6100            auto_unblock_last_reason: String::new(),
6101            ci_recovery_count: 0,
6102            no_code_reroutes: 0,
6103            no_code_last_agent: String::new(),
6104            created_at: String::new(),
6105            updated_at: String::new(),
6106        }
6107    }
6108
6109    #[serial(cooldown_state)]
6110    #[test]
6111    fn is_ci_failure_block_matches_ci_failure_limit() {
6112        let task = make_task(Some(
6113            "CI failure limit (3) reached during auto-merge".to_string(),
6114        ));
6115        assert!(is_ci_failure_block(&task));
6116    }
6117
6118    #[serial(cooldown_state)]
6119    #[test]
6120    fn is_ci_failure_block_matches_ci_checks_timeout() {
6121        let task = make_task(Some(
6122            "CI checks timed out after 3 auto-merge failures".to_string(),
6123        ));
6124        assert!(is_ci_failure_block(&task));
6125    }
6126
6127    #[serial(cooldown_state)]
6128    #[test]
6129    fn is_ci_failure_block_false_for_regular_block() {
6130        let task = make_task(Some("waiting on input".to_string()));
6131        assert!(!is_ci_failure_block(&task));
6132    }
6133
6134    #[serial(cooldown_state)]
6135    #[test]
6136    fn is_ci_failure_block_false_for_none_reason() {
6137        let task = make_task(None);
6138        assert!(!is_ci_failure_block(&task));
6139    }
6140
6141    // ── ci_failure_unblock_cooldown_elapsed ────────────────────────────────
6142
6143    #[serial(cooldown_state)]
6144    #[test]
6145    fn ci_failure_cooldown_zero_count_immediate() {
6146        // count=0 means never attempted — always allow immediate attempt
6147        let task = make_task(None);
6148        assert!(ci_failure_unblock_cooldown_elapsed(&task));
6149    }
6150
6151    #[serial(cooldown_state)]
6152    #[test]
6153    fn ci_failure_cooldown_empty_last_at_immediate() {
6154        // count=1 but empty last_at means it was never recorded — allow immediate
6155        let mut task = make_task(None);
6156        task.auto_unblock_count = 1;
6157        assert!(ci_failure_unblock_cooldown_elapsed(&task));
6158    }
6159
6160    #[serial(cooldown_state)]
6161    #[test]
6162    fn ci_failure_cooldown_invalid_timestamp_immediate() {
6163        // invalid timestamp should not block
6164        let mut task = make_task(None);
6165        task.auto_unblock_count = 1;
6166        task.auto_unblock_last_at = "not-a-timestamp".to_string();
6167        assert!(ci_failure_unblock_cooldown_elapsed(&task));
6168    }
6169
6170    #[serial(cooldown_state)]
6171    #[test]
6172    fn ci_failure_cooldown_high_count_still_eventually_elapses() {
6173        // count >= 3 must NOT be a permanent block — verify a sufficiently old
6174        // timestamp eventually elapses the cooldown.
6175        let long_ago = (chrono::Utc::now() - chrono::Duration::days(30)).to_rfc3339();
6176        for count in [3i32, 5, 99] {
6177            let mut task = make_task(None);
6178            task.auto_unblock_count = count;
6179            task.auto_unblock_last_at = long_ago.clone();
6180            assert!(
6181                ci_failure_unblock_cooldown_elapsed(&task),
6182                "count={count} should eventually elapse"
6183            );
6184        }
6185    }
6186
6187    #[serial(cooldown_state)]
6188    #[test]
6189    fn ci_failure_cooldown_recent_timestamp_still_blocks() {
6190        // A recent timestamp should still gate the check at every count.
6191        let just_now = chrono::Utc::now().to_rfc3339();
6192        for count in [1i32, 2, 3, 99] {
6193            let mut task = make_task(None);
6194            task.auto_unblock_count = count;
6195            task.auto_unblock_last_at = just_now.clone();
6196            assert!(
6197                !ci_failure_unblock_cooldown_elapsed(&task),
6198                "count={count} should still be cooling down"
6199            );
6200        }
6201    }
6202
6203    // ── auto_unblock_ci_failure_blocked_tasks_global ───────────────────────
6204
6205    #[serial(cooldown_state)]
6206    #[tokio::test]
6207    async fn global_ci_failure_sweep_returns_ok_with_no_blocked_tasks() {
6208        cooldown::reset_global_state().await;
6209        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6210        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6211        let task_manager = Arc::new(TaskManager::with_store(
6212            backend,
6213            Arc::clone(&store),
6214            "owner/repo".to_string(),
6215        ));
6216        auto_unblock_ci_failure_blocked_tasks_global(
6217            &task_manager,
6218            &store,
6219            &Arc::new(DashSet::new()),
6220        )
6221        .await
6222        .unwrap();
6223    }
6224
6225    #[serial(cooldown_state)]
6226    #[tokio::test]
6227    async fn global_ci_failure_sweep_skips_tasks_in_cooldown() {
6228        // Tasks with a fresh auto_unblock_last_at are skipped before reaching GhHttp,
6229        // so this test works without a real GitHub token.
6230        cooldown::reset_global_state().await;
6231        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6232        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6233        let task_manager = Arc::new(TaskManager::with_store(
6234            backend,
6235            Arc::clone(&store),
6236            "active/repo".to_string(),
6237        ));
6238
6239        // Insert a CI-failure task from an inactive repo with a very recent last_at.
6240        let id = store
6241            .upsert_external(&crate::store::UpsertExternal {
6242                repo: "inactive/repo",
6243                ext_id: "10",
6244                title: "Old blocked task",
6245                body: "",
6246                author: "",
6247                url: "",
6248                labels: &[],
6249                origin: "github",
6250            })
6251            .await
6252            .unwrap();
6253        store.update_status(id, TaskStatus::Blocked).await.unwrap();
6254        let just_now = chrono::Utc::now().to_rfc3339();
6255        store
6256            .set_fields(
6257                id,
6258                &[
6259                    (
6260                        "block_reason",
6261                        serde_json::json!("CI failure limit reached during auto-merge"),
6262                    ),
6263                    ("pr_number", serde_json::json!(99)),
6264                    ("auto_unblock_count", serde_json::json!(1)),
6265                    ("auto_unblock_last_at", serde_json::json!(just_now)),
6266                ],
6267            )
6268            .await
6269            .unwrap();
6270
6271        auto_unblock_ci_failure_blocked_tasks_global(
6272            &task_manager,
6273            &store,
6274            &Arc::new(DashSet::new()),
6275        )
6276        .await
6277        .unwrap();
6278
6279        // Cooldown not elapsed — task must remain untouched.
6280        let task = store.get(id).await.unwrap();
6281        assert_eq!(
6282            task.auto_unblock_count, 1,
6283            "task in cooldown must not be processed"
6284        );
6285    }
6286
6287    #[serial(cooldown_state)]
6288    #[tokio::test]
6289    async fn global_ci_failure_sweep_ignores_non_ci_failure_blocks() {
6290        cooldown::reset_global_state().await;
6291        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6292        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6293        let task_manager = Arc::new(TaskManager::with_store(
6294            backend,
6295            Arc::clone(&store),
6296            "active/repo".to_string(),
6297        ));
6298
6299        // Insert a non-CI-failure blocked task — should be ignored entirely.
6300        let id = store
6301            .upsert_external(&crate::store::UpsertExternal {
6302                repo: "inactive/repo",
6303                ext_id: "20",
6304                title: "Manual block",
6305                body: "",
6306                author: "",
6307                url: "",
6308                labels: &[],
6309                origin: "github",
6310            })
6311            .await
6312            .unwrap();
6313        store.update_status(id, TaskStatus::Blocked).await.unwrap();
6314        store
6315            .set_fields(
6316                id,
6317                &[("block_reason", serde_json::json!("waiting for human input"))],
6318            )
6319            .await
6320            .unwrap();
6321
6322        auto_unblock_ci_failure_blocked_tasks_global(
6323            &task_manager,
6324            &store,
6325            &Arc::new(DashSet::new()),
6326        )
6327        .await
6328        .unwrap();
6329
6330        // Non-CI-failure tasks must remain blocked.
6331        let task = store.get(id).await.unwrap();
6332        assert_eq!(task.status, TaskStatus::Blocked);
6333    }
6334
6335    // ── auto_recover_rebroadcast_blocked_tasks_global ───────────────────────
6336
6337    #[serial(cooldown_state)]
6338    #[tokio::test]
6339    async fn global_rebroadcast_sweep_returns_ok_with_no_blocked_tasks() {
6340        cooldown::reset_global_state().await;
6341        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6342        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6343        let task_manager = Arc::new(TaskManager::with_store(
6344            backend,
6345            Arc::clone(&store),
6346            "owner/repo".to_string(),
6347        ));
6348        let router = router_with_healthy_review_agent();
6349        auto_recover_rebroadcast_blocked_tasks_global(
6350            &router,
6351            &task_manager,
6352            &store,
6353            &sweep_repos(),
6354        )
6355        .await
6356        .unwrap();
6357    }
6358
6359    #[serial(cooldown_state)]
6360    #[tokio::test]
6361    async fn global_rebroadcast_sweep_ignores_non_rebroadcast_blocks() {
6362        cooldown::reset_global_state().await;
6363        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6364        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6365        let task_manager = Arc::new(TaskManager::with_store(
6366            backend,
6367            Arc::clone(&store),
6368            "active/repo".to_string(),
6369        ));
6370        let router = router_with_healthy_review_agent();
6371
6372        let id = store
6373            .upsert_external(&crate::store::UpsertExternal {
6374                repo: "inactive/repo",
6375                ext_id: "30",
6376                title: "Manual block",
6377                body: "",
6378                author: "",
6379                url: "",
6380                labels: &[],
6381                origin: "github",
6382            })
6383            .await
6384            .unwrap();
6385        store.update_status(id, TaskStatus::Blocked).await.unwrap();
6386        store
6387            .set_fields(
6388                id,
6389                &[("block_reason", serde_json::json!("waiting for human input"))],
6390            )
6391            .await
6392            .unwrap();
6393        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
6394            .bind(id)
6395            .execute(store.pool())
6396            .await
6397            .unwrap();
6398
6399        auto_recover_rebroadcast_blocked_tasks_global(
6400            &router,
6401            &task_manager,
6402            &store,
6403            &sweep_repos(),
6404        )
6405        .await
6406        .unwrap();
6407
6408        let task = store.get(id).await.unwrap();
6409        assert_eq!(task.status, TaskStatus::Blocked);
6410        assert_eq!(
6411            task.block_reason.as_deref(),
6412            Some("waiting for human input")
6413        );
6414    }
6415
6416    #[serial(cooldown_state)]
6417    #[tokio::test]
6418    async fn global_rebroadcast_sweep_skips_freshly_blocked_tasks() {
6419        cooldown::reset_global_state().await;
6420        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6421        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6422        let task_manager = Arc::new(TaskManager::with_store(
6423            backend,
6424            Arc::clone(&store),
6425            "active/repo".to_string(),
6426        ));
6427        let router = router_with_healthy_review_agent();
6428
6429        let id = store
6430            .upsert_external(&crate::store::UpsertExternal {
6431                repo: "inactive/repo",
6432                ext_id: "40",
6433                title: "Just escalated",
6434                body: "",
6435                author: "",
6436                url: "",
6437                labels: &[],
6438                origin: "github",
6439            })
6440            .await
6441            .unwrap();
6442        store.update_status(id, TaskStatus::Blocked).await.unwrap();
6443        store
6444            .set_fields(
6445                id,
6446                &[("block_reason", serde_json::json!(REBROADCAST_BLOCK_REASON))],
6447            )
6448            .await
6449            .unwrap();
6450        // No backdating — escalated "just now" in this same tick, must not be
6451        // immediately un-blocked by the same pass.
6452
6453        auto_recover_rebroadcast_blocked_tasks_global(
6454            &router,
6455            &task_manager,
6456            &store,
6457            &sweep_repos(),
6458        )
6459        .await
6460        .unwrap();
6461
6462        let task = store.get(id).await.unwrap();
6463        assert_eq!(task.status, TaskStatus::Blocked);
6464    }
6465
6466    /// Regression test for #3532: a rebroadcast-blocked task whose repo is not
6467    /// the TaskManager's own repo must still be recovered. The sweep queries
6468    /// across all repos and writes through the store directly, bypassing the
6469    /// repo-scoped TaskManager
6470    #[serial(cooldown_state)]
6471    #[tokio::test]
6472    async fn global_rebroadcast_sweep_recovers_task_from_inactive_repo() {
6473        cooldown::reset_global_state().await;
6474        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6475        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6476        let task_manager = Arc::new(TaskManager::with_store(
6477            backend,
6478            Arc::clone(&store),
6479            "active/repo".to_string(),
6480        ));
6481        let router = router_with_healthy_review_agent();
6482
6483        let id = store
6484            .upsert_external(&crate::store::UpsertExternal {
6485                repo: "inactive/repo",
6486                ext_id: "458",
6487                title: "Rebroadcast blocked from inactive repo",
6488                body: "",
6489                author: "",
6490                url: "",
6491                labels: &[],
6492                origin: "github",
6493            })
6494            .await
6495            .unwrap();
6496        store.update_status(id, TaskStatus::Blocked).await.unwrap();
6497        store
6498            .set_fields(
6499                id,
6500                &[
6501                    ("block_reason", serde_json::json!(REBROADCAST_BLOCK_REASON)),
6502                    ("needs_review_refires", serde_json::json!(6)),
6503                ],
6504            )
6505            .await
6506            .unwrap();
6507        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
6508            .bind(id)
6509            .execute(store.pool())
6510            .await
6511            .unwrap();
6512
6513        auto_recover_rebroadcast_blocked_tasks_global(
6514            &router,
6515            &task_manager,
6516            &store,
6517            &sweep_repos(),
6518        )
6519        .await
6520        .unwrap();
6521
6522        let task = store.get(id).await.unwrap();
6523        assert_eq!(
6524            task.status,
6525            TaskStatus::NeedsReview,
6526            "cross-repo rebroadcast-blocked task must be recovered to needs_review"
6527        );
6528        assert!(
6529            task.block_reason.is_none(),
6530            "block_reason must be cleared on recovery"
6531        );
6532        assert_eq!(
6533            task.needs_review_refires, 0,
6534            "needs_review_refires must be reset to 0 on recovery"
6535        );
6536    }
6537
6538    /// With no routable review agent, the sweep must not reset the task, that
6539    /// would immediately re-exhaust the refire counter
6540    #[serial(cooldown_state)]
6541    #[tokio::test]
6542    async fn global_rebroadcast_sweep_skips_reset_without_routable_review_agent() {
6543        cooldown::reset_global_state().await;
6544        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6545        let backend: Arc<dyn ExternalBackend> = IngestMockBackend::with_tasks(vec![]);
6546        let task_manager = Arc::new(TaskManager::with_store(
6547            backend,
6548            Arc::clone(&store),
6549            "active/repo".to_string(),
6550        ));
6551        // RouterConfig::default() has no "review" model map entries, so the one
6552        // available agent has no healthy model for "review" — nothing is routable.
6553        let router = Router::new_for_test(RouterConfig::default(), vec!["claude".to_string()]);
6554
6555        let id = store
6556            .upsert_external(&crate::store::UpsertExternal {
6557                repo: "inactive/repo",
6558                ext_id: "459",
6559                title: "Rebroadcast blocked, no routable agents",
6560                body: "",
6561                author: "",
6562                url: "",
6563                labels: &[],
6564                origin: "github",
6565            })
6566            .await
6567            .unwrap();
6568        store.update_status(id, TaskStatus::Blocked).await.unwrap();
6569        store
6570            .set_fields(
6571                id,
6572                &[("block_reason", serde_json::json!(REBROADCAST_BLOCK_REASON))],
6573            )
6574            .await
6575            .unwrap();
6576        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
6577            .bind(id)
6578            .execute(store.pool())
6579            .await
6580            .unwrap();
6581
6582        auto_recover_rebroadcast_blocked_tasks_global(
6583            &router,
6584            &task_manager,
6585            &store,
6586            &sweep_repos(),
6587        )
6588        .await
6589        .unwrap();
6590
6591        let task = store.get(id).await.unwrap();
6592        assert_eq!(task.status, TaskStatus::Blocked);
6593        assert_eq!(task.block_reason.as_deref(), Some(REBROADCAST_BLOCK_REASON));
6594    }
6595}