Skip to main content

orch/engine/subscribers/
review.rs

1//! Reacts to NeedsReview events — spawns review agent immediately.
2
3use crate::backends::{ExternalBackend, ExternalId, ExternalTask, Status};
4use crate::config;
5use crate::engine;
6use crate::engine::auto_merge;
7use crate::engine::cooldown::AGENT_COOLDOWN_SECS;
8use crate::engine::dispatch_guard::DispatchGuard;
9use crate::engine::events::TaskEvent;
10use crate::engine::review;
11use crate::engine::review::{MAX_REVIEW_AGENT_FAILURES, ReviewDecision, review_and_merge};
12use crate::engine::review_gate;
13use crate::engine::review_gate::CiGate;
14use crate::engine::review_shipcheck;
15use crate::engine::review_shipcheck::Prerun;
16use crate::engine::router::Router;
17use crate::engine::runner::agent::Shipcheck;
18use crate::engine::runner::agents::patterns;
19use crate::engine::runner::git_ops;
20use crate::engine::tasks;
21use crate::engine::tasks::TaskManager;
22use crate::github::http::GhHttp;
23use crate::repo_context::REPO_CONTEXT;
24use crate::store;
25use crate::store::TaskStatus;
26use crate::store::tasks as store_tasks;
27use crate::store::{TaskStore, opt_store_get_task};
28use crate::tmux::TmuxManager;
29use dashmap::DashMap;
30use dashmap::mapref::entry::Entry;
31use std::path::PathBuf;
32use std::sync::Arc;
33use std::time::Duration;
34use tokio::sync::broadcast;
35use tokio::sync::{RwLock, Semaphore};
36use tokio::time;
37
38enum ReviewOutcome {
39    Reset,
40    RateLimited,
41    /// The PR was approved but the merge did not land for a reason outside the
42    /// review (mergeability not computed yet, rate limit, transient GitHub error).
43    /// The task moves to Approved so `review_open_prs` retries the merge, instead of
44    /// resetting to NeedsReview (which spawns a redundant review agent)
45    RetryMerge,
46    /// The review approved the PR. The task moves to Approved, where orch waits for CI and
47    /// merges with no agent running
48    Approved,
49    /// Re-route to the task agent (e.g. merge conflicts that need the agent to fix).
50    Reroute,
51    Block(String),
52    Ok,
53}
54
55/// Spawn a task that listens for NeedsReview events and triggers the review agent.
56///
57/// This is the sole trigger for NeedsReview → InReview transitions. The sync tick
58/// no longer has a competing NeedsReview loop (removed in fix for issue #857 — both
59/// paths firing simultaneously caused double failure-counter increments).
60#[expect(
61    clippy::too_many_arguments,
62    reason = "the arguments have no natural grouping"
63)]
64pub fn spawn(
65    mut rx: broadcast::Receiver<TaskEvent>,
66    backend: Arc<dyn ExternalBackend>,
67    tmux: Arc<TmuxManager>,
68    semaphore: Arc<Semaphore>,
69    task_manager: Arc<TaskManager>,
70    router: Arc<RwLock<Router>>,
71    dispatching: Arc<DashMap<String, String>>,
72    store: Arc<TaskStore>,
73    repo: String,
74) {
75    tokio::spawn(async move {
76        loop {
77            match rx.recv().await {
78                Ok(event) if event.new_status == "needs_review" && event.repo == repo => {
79                    // Check if review agent is enabled (read fresh each time).
80                    let enable_review =
81                        config::get("workflow.enable_review_agent").map_or(true, |v| v != "false");
82                    if !enable_review {
83                        continue;
84                    }
85
86                    let task_id = &event.task_id;
87                    let dispatch_key = format!("{repo}/{task_id}");
88
89                    // Atomically claim the task before any async work so concurrent
90                    // review events cannot double-spawn the same review flow.
91                    // Using DashMap lets us distinguish same-task duplicates (expected) from
92                    // key collisions involving a different task (should never happen).
93                    {
94                        match dispatching.entry(dispatch_key.clone()) {
95                            Entry::Occupied(existing) => {
96                                let existing_id = existing.get().clone();
97                                drop(existing);
98                                if existing_id == *task_id {
99                                    tracing::debug!(
100                                        task_id,
101                                        "task locked by dispatch flow, skipping event-driven review"
102                                    );
103                                } else {
104                                    tracing::warn!(
105                                        task_id,
106                                        existing_task_id = existing_id,
107                                        dispatch_key,
108                                        "dispatch key collision: unexpected task already holds this key"
109                                    );
110                                }
111                                continue;
112                            }
113                            Entry::Vacant(slot) => {
114                                slot.insert(task_id.clone());
115                            }
116                        }
117                    }
118
119                    // Create guard IMMEDIATELY after inserting the key — owns removal for all subsequent exit paths
120                    let dispatch_guard =
121                        DispatchGuard::new(Arc::clone(&dispatching), dispatch_key.clone());
122
123                    tracing::info!(task_id, "event-driven review triggered");
124
125                    // Look up the task from the store to get an ExternalTask.
126                    let task = {
127                        if tasks::is_internal_id(task_id)
128                            || match store.has_external_tasks(&repo).await {
129                                Ok(has_external) => has_external,
130                                Err(e) => {
131                                    tracing::warn!(
132                                        repo = %repo,
133                                        task_id,
134                                        error = %e,
135                                        "failed to check external task sentinel; keeping store-first review lookup"
136                                    );
137                                    true
138                                }
139                            }
140                        {
141                            // O(1) indexed lookup — resolve task_id to its SQLite row id,
142                            // then fetch directly instead of scanning the whole needs_review set.
143                            let result = if let Some(numeric_id) = tasks::parse_internal_id(task_id)
144                            {
145                                store.get(numeric_id).await.map(Some)
146                            } else {
147                                store.get_by_external_id(&repo, task_id).await
148                            };
149                            match result {
150                                Ok(Some(t)) if t.status == TaskStatus::NeedsReview => {
151                                    Some(tasks::store_task_to_external(&t))
152                                }
153                                Ok(Some(t)) => {
154                                    tracing::debug!(
155                                        task_id,
156                                        status = t.status.as_str(),
157                                        "task found but not in needs_review status, skipping"
158                                    );
159                                    None
160                                }
161                                Ok(None) => None,
162                                Err(e) => {
163                                    tracing::warn!(task_id, err = %e, "store lookup failed for review");
164                                    None
165                                }
166                            }
167                        } else {
168                            // Fall back to backend
169                            match backend.list_by_status(Status::NeedsReview).await {
170                                Ok(tasks) => tasks.into_iter().find(|t| t.id.0 == *task_id),
171                                Err(e) => {
172                                    tracing::warn!(task_id, err = %e, "backend lookup failed for review");
173                                    None
174                                }
175                            }
176                        }
177                    };
178
179                    let Some(task) = task else {
180                        tracing::debug!(
181                            task_id,
182                            "task not found in needs_review list (may have already transitioned)"
183                        );
184                        continue;
185                    };
186
187                    if let Some(stored) =
188                        opt_store_get_task(&Some(Arc::clone(&store)), &repo, &task.id.0).await
189                    {
190                        let no_code_review = stored.worktree.trim().is_empty()
191                            && stored.branch.trim().is_empty()
192                            && stored.pr_number.is_none();
193                        if no_code_review {
194                            tracing::info!(
195                                task_id,
196                                "no worktree/branch/pr for needs_review task — skipping review dispatch and marking done"
197                            );
198                            store::store_reset_failure_counters(
199                                &Some(Arc::clone(&store)),
200                                &repo,
201                                &task.id.0,
202                            )
203                            .await;
204                            if let Err(e) = store::store_set_result(
205                                &Some(Arc::clone(&store)),
206                                &repo,
207                                &task.id.0,
208                                &[("needs_review_refires", serde_json::json!(0))],
209                            )
210                            .await
211                            {
212                                tracing::warn!(
213                                    task_id,
214                                    err = %e,
215                                    "failed to reset needs_review_refires while skipping no-code review"
216                                );
217                            }
218                            if let Err(e) = task_manager
219                                .update_task_status(&task.id, Status::Done)
220                                .await
221                            {
222                                tracing::error!(
223                                    task_id,
224                                    err = %e,
225                                    "failed to mark no-code needs_review task as done"
226                                );
227                            }
228                            continue;
229                        }
230                    }
231
232                    // If no agents are currently routable (all cooled, degraded, or without
233                    // an available review model), skip this event and let sync re-fire when
234                    // an agent is available again. Sleeping inline here would block every
235                    // subsequent NeedsReview event for this repo for the duration of the
236                    // cooldown (potentially hours/days), causing sync's rebroadcast loop to
237                    // escalate piled-up tasks to Blocked.
238                    {
239                        let router_guard = router.read().await;
240                        let no_routable = !router_guard.available_agents.is_empty()
241                            && router_guard.healthy_agent_count("review") == 0;
242                        drop(router_guard);
243                        if no_routable {
244                            tracing::info!(
245                                task_id,
246                                "no routable review agents — leaving task in needs_review for sync to re-fire"
247                            );
248                            continue;
249                        }
250                    }
251
252                    if !ci_allows_review(&store, &task_manager, &repo, &task).await {
253                        continue;
254                    }
255
256                    // Try to acquire a semaphore permit.
257                    let Ok(permit) = Arc::clone(&semaphore).try_acquire_owned() else {
258                        tracing::debug!(
259                            task_id,
260                            "all review slots busy; sync catch-up will re-fire when a slot opens"
261                        );
262                        continue;
263                    };
264
265                    // Transition to InReview — this IS the atomic guard against duplicates.
266                    if let Err(e) = task_manager
267                        .update_task_status(&task.id, Status::InReview)
268                        .await
269                    {
270                        tracing::warn!(task_id, err = %e, "failed to transition to InReview");
271                        drop(permit);
272                        continue;
273                    }
274
275                    let backend_c = Arc::clone(&backend);
276                    let task_manager_c = Arc::clone(&task_manager);
277                    let tmux_c = Arc::clone(&tmux);
278                    let router_c = Arc::clone(&router);
279                    let store_c = Arc::clone(&store);
280                    let repo_s = repo.clone();
281                    tokio::spawn(REPO_CONTEXT.scope(repo_s.clone(), async move {
282                        let _dispatch_guard = dispatch_guard; // released on drop (normal or panic)
283                        let tid = task.id.0.clone();
284
285                        // Re-check agent routability inside the spawned task. A concurrent
286                        // review that ran between the outer dispatch check and now may have
287                        // set a cooldown — bail early rather than burning a review run
288                        // against a rate-limited agent.
289                        let pre_check_all_cooled = {
290                            let r = router_c.read().await;
291                            !r.available_agents.is_empty() && r.healthy_agent_count("review") == 0
292                        };
293                        if pre_check_all_cooled {
294                            tracing::info!(
295                                task_id = tid,
296                                "all agents cooled at review spawn time — resetting to needs_review for sync to re-fire"
297                            );
298                            if let Err(e) = task_manager_c
299                                .update_task_status(
300                                    &ExternalId(tid.clone()),
301                                    Status::NeedsReview,
302                                )
303                                .await
304                            {
305                                tracing::error!(task_id = %tid, err = %e, "pre-check: update_task_status(NeedsReview) failed — task may be stuck in InReview");
306                            }
307                            drop(permit);
308                            return;
309                        }
310
311                        let outcome = match Box::pin(review_and_merge(
312                            &task,
313                            &backend_c,
314                            &tmux_c,
315                            &repo_s,
316                            &router_c,
317                            &task_manager_c,
318                            &store_c,
319                        ))
320                        .await
321                        {
322                            Ok(ReviewDecision::Blocked(reason)) => {
323                                tracing::error!(
324                                    task_id = tid,
325                                    reason,
326                                    "review gate blocked after repeated failures — marking task blocked"
327                                );
328                                ReviewOutcome::Block(format!(
329                                    "review gate blocked after repeated failures: {reason}"
330                                ))
331                            }
332                            Ok(ReviewDecision::Failed(reason)) => {
333                                classify_review_failure(
334                                    &store_c, &repo_s, &tid, &reason, "review agent",
335                                )
336                                .await
337                            }
338                            Err(e) => {
339                                let reason = format!("{e:#}");
340                                classify_review_failure(
341                                    &store_c, &repo_s, &tid, &reason, "review_and_merge",
342                                )
343                                .await
344                            }
345                            Ok(ReviewDecision::Approve { .. }) => {
346                                // On approval we clear transient failure counters
347                                // but preserve the `review_cycles` counter which
348                                // tracks how many times the PR has requested
349                                // changes and is used as a circuit-breaker.
350                                store::store_reset_failure_counters(
351                                    &Some(Arc::clone(&store_c)),
352                                    &repo_s,
353                                    &tid,
354                                )
355                                .await;
356                                // Also reset needs_review_refires so the sync backoff doesn't persist across a successful review
357                                if let Err(e) = store::store_set_result(
358                                    &Some(Arc::clone(&store_c)),
359                                    &repo_s,
360                                    &tid,
361                                    &[("needs_review_refires", serde_json::json!(0))],
362                                )
363                                .await
364                                {
365                                    tracing::warn!(task_id = %tid, err = %e, "failed to write needs_review_refires to store");
366                                }
367
368                                // Approval only means the code looks good, the merge is the
369                                // completion signal. review_open_prs merges Approved tasks and
370                                // moves them to Done
371                                ReviewOutcome::Approved
372                            }
373                            Ok(ReviewDecision::Rerouted) => {
374                                // Task was re-routed because there was no PR and no commits.
375                                // Do NOT reset failure counters — no_code_reroutes must accumulate
376                                // so the max_reroute_attempts safety limit can be reached.
377                                ReviewOutcome::Ok
378                            }
379                            Ok(ReviewDecision::Skipped) => {
380                                // Skipped indicates there's nothing to review (no PR etc.).
381                                // Clear transient failure counters but preserve review_cycles.
382                                store::store_reset_failure_counters(
383                                    &Some(Arc::clone(&store_c)),
384                                    &repo_s,
385                                    &tid,
386                                )
387                                .await;
388                                if let Err(e) = store::store_set_result(
389                                    &Some(Arc::clone(&store_c)),
390                                    &repo_s,
391                                    &tid,
392                                    &[("needs_review_refires", serde_json::json!(0))],
393                                )
394                                .await
395                                {
396                                    tracing::warn!(task_id = %tid, err = %e, "failed to write needs_review_refires to store");
397                                }
398                                ReviewOutcome::Ok
399                            }
400                            Ok(ReviewDecision::RequestChanges { .. }) => {
401                                // handle_review_changes already incremented review_cycles —
402                                // only reset transient per-attempt counters.
403                                store::store_reset_failure_counters(
404                                    &Some(Arc::clone(&store_c)),
405                                    &repo_s,
406                                    &tid,
407                                )
408                                .await;
409                                if let Err(e) = store::store_set_result(
410                                    &Some(Arc::clone(&store_c)),
411                                    &repo_s,
412                                    &tid,
413                                    &[("needs_review_refires", serde_json::json!(0))],
414                                )
415                                .await
416                                {
417                                    tracing::warn!(task_id = %tid, err = %e, "failed to write needs_review_refires to store");
418                                }
419                                ReviewOutcome::Ok
420                            }
421                        };
422                        // Freshness guard: re-fetch the task's current status before applying
423                        // any outcome transitions. A concurrent path (e.g. PR merge, manual
424                        // close, or another review cycle) may have already moved the task out
425                        // of `in_review`. If so, discard the stale outcome rather than
426                        // overwriting the newer state.
427                        //
428                        // IMPORTANT: Distinguish between DB errors and legitimate status changes.
429                        // A DB lookup failure should NOT be treated as "task moved out of in_review"
430                        // — we should retain the review outcome rather than discarding it on
431                        // transient errors.
432                        let fetched: Result<Option<(TaskStatus, Option<String>)>, anyhow::Error> = async {
433                            let id = match store_c.resolve_task_id(&repo_s, &tid).await {
434                                Ok(Some(id)) => id,
435                                Ok(None) => return Ok(None), // Task doesn't exist — treat as missing
436                                Err(e) => return Err(e),      // DB error — propagate, don't discard
437                            };
438                            let task = store_c.get(id).await?;
439                            Ok(Some((task.status, task.block_reason)))
440                        }
441                        .await;
442
443                        let (current_status, current_block_reason) = match fetched {
444                            Ok(Some((status, block_reason))) => (Some(status), block_reason),
445                            Ok(None) => {
446                                // Task doesn't exist in store — treat as not in_review
447                                (None, None)
448                            }
449                            Err(e) => {
450                                // DB error — log and keep the review outcome instead of discarding
451                                tracing::error!(
452                                    task_id = tid,
453                                    error = %e,
454                                    "review outcome retained — DB lookup failed, not treating as stale"
455                                );
456                                // Skip the stale outcome check and proceed with the outcome
457                                // by treating it as if the task is still InReview
458                                (Some(TaskStatus::InReview), None)
459                            }
460                        };
461
462                        if !matches!(current_status, Some(TaskStatus::InReview)) {
463                            let ci_blocked = matches!(
464                                current_status,
465                                Some(TaskStatus::Blocked)
466                            ) && current_block_reason
467                                .as_deref()
468                                .is_some_and(|r| r.contains("CI failure limit"));
469                            let expected = ci_blocked
470                                || matches!(
471                                    current_status,
472                                    Some(TaskStatus::Done | TaskStatus::Approved |
473TaskStatus::Routed | TaskStatus::New)
474                                );
475                            if expected {
476                                tracing::debug!(
477                                    task_id = tid,
478                                    current_status = current_status.as_ref().map_or("unknown", store_tasks::TaskStatus::as_str),
479                                    "review outcome discarded — task transitioned away from in_review (expected)"
480                                );
481                            } else {
482                                tracing::warn!(
483                                    task_id = tid,
484                                    current_status = current_status.as_ref().map_or("unknown", store_tasks::TaskStatus::as_str),
485                                    "review outcome discarded — task is no longer in_review (stale review agent result)"
486                                );
487                            }
488
489                            // Kill the orphaned review session
490                            let stale_session = tmux_c.session_name(&repo_s, &format!("{tid}-review"));
491                            if let Err(e) = tmux_c.kill_session(&stale_session).await {
492                                tracing::debug!(task_id = tid, error = %e, "failed to kill stale review session");
493                            }
494
495                            drop(permit);
496                            return;
497                        }
498
499                        match outcome {
500                            ReviewOutcome::Reset => {
501                                // Kill any stale tmux review session before resetting — the
502                                // session may still be alive if the agent hit a rate limit,
503                                // if two concurrent reviewers raced and the loser's spawn
504                                // failed with "duplicate session", or if review_and_merge
505                                // returned an error before reaching its own cleanup.
506                                // Killing a non-existent session is a harmless no-op.
507                                let stale_session =
508                                    tmux_c.session_name(&repo_s, &format!("{tid}-review"));
509                                tmux_c.kill_session(&stale_session).await.ok();
510                                // Backoff before re-queuing to prevent rapid spin
511                                // if the review agent keeps failing instantly.
512                                time::sleep(Duration::from_secs(10)).await;
513                                if let Err(e) = task_manager_c
514                                    .update_task_status(
515                                        &ExternalId(tid.clone()),
516                                        Status::NeedsReview,
517                                    )
518                                    .await
519                                {
520                                    tracing::error!(task_id = %tid, err = %e, "update_task_status(NeedsReview) failed — task may be stuck in InReview");
521                                }
522                            }
523                            ReviewOutcome::Reroute => {
524                                // Merge conflict or similar — route back to the
525                                // task agent so it can fix the issue.
526                                let stale_session =
527                                    tmux_c.session_name(&repo_s, &format!("{tid}-review"));
528                                tmux_c.kill_session(&stale_session).await.ok();
529                                tracing::info!(
530                                    task_id = tid,
531                                    "re-routing to task agent to resolve merge conflict"
532                                );
533                                if let Err(e) = task_manager_c
534                                    .update_task_status(
535                                        &ExternalId(tid.clone()),
536                                        Status::Routed,
537                                    )
538                                    .await
539                                {
540                                    tracing::error!(task_id = %tid, err = %e, "update_task_status(Routed) failed");
541                                }
542                            }
543                            ReviewOutcome::Approved | ReviewOutcome::RetryMerge => {
544                                // The review is over, so its session goes. The merge may have
545                                // landed already, or CI sent the task back, so only in_review moves
546                                let stale_session =
547                                    tmux_c.session_name(&repo_s, &format!("{tid}-review"));
548                                tmux_c.kill_session(&stale_session).await.ok();
549                                match task_manager_c
550                                    .update_task_status_if(
551                                        &ExternalId(tid.clone()),
552                                        Status::Approved,
553                                        Status::InReview,
554                                    )
555                                    .await
556                                {
557                                    Ok(true) => tracing::info!(
558                                        task_id = tid,
559                                        "approval received, task approved until the PR is merged"
560                                    ),
561                                    Ok(false) => tracing::debug!(
562                                        task_id = tid,
563                                        "approval received after the task left in_review"
564                                    ),
565                                    Err(e) => tracing::error!(
566                                        task_id = %tid,
567                                        err = %e,
568                                        "update_task_status_if(Approved) failed, task may be stuck in in_review"
569                                    ),
570                                }
571                            }
572                            ReviewOutcome::RateLimited => {
573                                // Kill stale tmux session — same as Reset.
574                                let stale_session =
575                                    tmux_c.session_name(&repo_s, &format!("{tid}-review"));
576                                tmux_c.kill_session(&stale_session).await.ok();
577
578                                // Compute wait duration: if any agent is available right now,
579                                // use a short backoff (30 s). If every agent is cooled, sleep
580                                // until the earliest cooldown expires so we don't spin.
581                                let wait_secs = {
582                                    let r = router_c.read().await;
583                                    if r.healthy_agent_count("review") > 0 {
584                                        30u64
585                                    } else {
586                                        let now = chrono::Utc::now().timestamp();
587                                        r.earliest_cooldown_until(Some("review"))
588                                            .map_or(AGENT_COOLDOWN_SECS as u64, |until| u64::try_from((until - now).max(1)).unwrap_or(1))
589                                    }
590                                };
591                                tracing::info!(
592                                    task_id = tid,
593                                    wait_secs,
594                                    "deferring review retry due to agent rate limit cooldown"
595                                );
596                                time::sleep(Duration::from_secs(wait_secs)).await;
597                                if let Err(e) = task_manager_c
598                                    .update_task_status(
599                                        &ExternalId(tid.clone()),
600                                        Status::NeedsReview,
601                                    )
602                                    .await
603                                {
604                                    tracing::error!(task_id = %tid, err = %e, "update_task_status(NeedsReview) failed after rate limit backoff — task may be stuck in InReview");
605                                }
606                            }
607                             ReviewOutcome::Block(reason) => {
608                                 // Persist block_reason BEFORE transitioning to Blocked to avoid
609                                 // a race where auto_unblock sees a blocked task without a reason
610                                 // and immediately unblocks it.
611                                 let fields = [
612                                     (
613                                         "block_reason",
614                                         serde_json::json!("review agent blocked — exceeded failure threshold"),
615                                     ),
616                                     ("last_error", serde_json::json!(reason)),
617                                 ];
618                                 if let Err(e) = task_manager_c
619                                     .update_task_status_and_result(
620                                         &ExternalId(tid.clone()),
621                                         Status::Blocked,
622                                         &fields,
623                                     )
624                                     .await
625                                 {
626                                     tracing::error!(task_id = %tid, err = %e, "update_task_status_and_result(Blocked) failed — task may be stuck in InReview");
627                                 }
628                             }
629                            ReviewOutcome::Ok => {}
630                        }
631
632                        drop(permit);
633                        // _dispatch_guard dropped here — releases the per-task lock.
634                    }));
635
636                    // Small delay between dispatches to allow cooldowns from
637                    // early failures to propagate before subsequent reviews are
638                    // dispatched. This prevents batch dispatch of N reviews to
639                    // the same rate-limited agent when events arrive concurrently.
640                    time::sleep(Duration::from_secs(2)).await;
641                }
642                Ok(_) => {} // Not a needs_review event or different repo
643                Err(broadcast::error::RecvError::Lagged(n)) => {
644                    tracing::debug!(
645                        n,
646                        "review subscriber lagged; sync catch-up will re-fire missed NeedsReview tasks"
647                    );
648                }
649                Err(_) => break, // Channel closed
650            }
651        }
652    });
653}
654
655/**
656 * Record a failed review attempt on the task's activity timeline and hand the outcome
657 * back to `classify_review_failure`, which returns it unchanged. Without this the only
658 * trace of a failed attempt was the final `last_error` written at escalation, so
659 * `orch task inspect` showed that a task gave up but never why each attempt failed.
660 *
661 * `external` marks a cause outside the review itself (agent rate limit, GitHub API
662 * limit, no review model). Those also clear `needs_review_refires`: the refire quota
663 * exists to catch a task the subscriber never picks up, and spending it on a cause the
664 * review agent cannot control escalated healthy tasks to Blocked during a rate-limit
665 * window. The retry itself is already handled by the `RateLimited` / `Reset` arms
666 **/
667/// The CI gate (#3904): true when the review may start. A task held for CI stays in needs_review
668/// and its refire count goes back to 0, so the sync sweep checks it again and never blocks it
669async fn ci_allows_review(
670    store: &Arc<TaskStore>,
671    task_manager: &Arc<TaskManager>,
672    repo: &str,
673    task: &ExternalTask,
674) -> bool {
675    let task_id = task.id.0.as_str();
676    let stored = opt_store_get_task(&Some(Arc::clone(store)), repo, task_id).await;
677    let Some(pr_number) = stored
678        .as_ref()
679        .and_then(|t| t.pr_number)
680        .and_then(|n| u64::try_from(n).ok())
681    else {
682        return true;
683    };
684    let worktree = stored
685        .map(|t| PathBuf::from(t.worktree))
686        .filter(|p| !p.as_os_str().is_empty() && p.exists());
687    let gh = GhHttp::new();
688    // A conflicting or stale branch is fixed before CI is read, since a rebase starts CI again
689    if let Ok(gh) = &gh
690        && !auto_merge::prepare_branch_for_review(task, repo, pr_number, gh, task_manager, store)
691            .await
692    {
693        reset_refires(store, repo, task_id).await;
694        return false;
695    }
696    let gate = match gh {
697        Ok(gh) => review_gate::review_ci_gate(&gh, store, repo, task_id, pr_number).await,
698        Err(e) => Err(e),
699    };
700
701    let hold = match gate {
702        Ok(CiGate::Start) => return true,
703        // The merge still waits for green CI, so a lookup error does not hold the review
704        Err(e) => {
705            tracing::warn!(task_id, pr_number, err = %e, "CI gate lookup failed, starting the review");
706            return true;
707        }
708        Ok(CiGate::Red(failed)) => {
709            route_back_on_red_ci(store, task_manager, repo, task, pr_number, &failed).await;
710            return false;
711        }
712        Ok(CiGate::Wait) => "CI still running on the PR head, review waits",
713        // The reviewer runs shipcheck when no check ran, so orch runs it first, outside the review's limit
714        Ok(CiGate::NotStarted { waited_out }) => match local_ci_worktree(repo, worktree).await {
715            Some(wt) => match review_shipcheck::prerun(&wt).await {
716                Prerun::Ready => return true,
717                Prerun::Running => "shipcheck running on the PR head, review waits",
718            },
719            None if waited_out => {
720                tracing::warn!(
721                    task_id,
722                    pr_number,
723                    "no CI ran on the PR head, starting the review anyway"
724                );
725                return true;
726            }
727            None => "no CI started on the PR head yet, review waits",
728        },
729    };
730
731    tracing::info!(task_id, pr_number, "{hold}");
732    reset_refires(store, repo, task_id).await;
733
734    false
735}
736
737/// A task held before its review keeps its refire count at 0, so the sync sweep never blocks it
738async fn reset_refires(store: &Arc<TaskStore>, repo: &str, task_id: &str) {
739    if let Err(e) = store::store_set_result(
740        &Some(Arc::clone(store)),
741        repo,
742        task_id,
743        &[("needs_review_refires", serde_json::json!(0))],
744    )
745    .await
746    {
747        tracing::warn!(task_id, err = %e, "failed to reset needs_review_refires for a task held before its review");
748    }
749}
750
751/// The task worktree, when the reviewer there would run `shipcheck run`
752async fn local_ci_worktree(repo: &str, worktree: Option<PathBuf>) -> Option<PathBuf> {
753    let wt = worktree?;
754    let sandboxed = review::review_sandboxed(repo);
755    let shipcheck = Shipcheck::detect(&wt).await.for_prompt(sandboxed);
756    (shipcheck == Shipcheck::Available).then_some(wt)
757}
758
759/// CI failed before the review, so the task agent fixes it without a review round
760async fn route_back_on_red_ci(
761    store: &Arc<TaskStore>,
762    task_manager: &Arc<TaskManager>,
763    repo: &str,
764    task: &ExternalTask,
765    pr_number: u64,
766    failed: &[String],
767) {
768    let task_id = task.id.0.as_str();
769    let failures = match store::store_increment(
770        &Some(Arc::clone(store)),
771        repo,
772        task_id,
773        "ci_merge_failures",
774    )
775    .await
776    {
777        Ok(v) => v,
778        Err(e) => {
779            tracing::warn!(task_id, err = %e, "failed to increment ci_merge_failures, review waits");
780            return;
781        }
782    };
783    let checks = failed.join(", ");
784    let max = auto_merge::MAX_CI_MERGE_FAILURES;
785    let (status, fields) = if failures >= max {
786        tracing::error!(
787            task_id,
788            pr_number,
789            failures,
790            checks,
791            "CI failed before review, CI failure limit reached"
792        );
793        (
794            Status::Blocked,
795            vec![(
796                "block_reason",
797                serde_json::json!(format!("CI failure limit ({max}) reached before review")),
798            )],
799        )
800    } else {
801        tracing::warn!(
802            task_id,
803            pr_number,
804            failures,
805            checks,
806            "CI failed before review, routing back to the task agent"
807        );
808        (
809            Status::Routed,
810            vec![
811                (
812                    "last_error",
813                    serde_json::json!(format!(
814                        "CI failed on PR #{pr_number} before review: {checks}. Fix the failing checks"
815                    )),
816                ),
817                (
818                    "route_reason",
819                    serde_json::json!("re-dispatch after CI failed before review"),
820                ),
821            ],
822        )
823    };
824    if let Err(e) = task_manager
825        .update_task_status_and_result(&task.id, status, &fields)
826        .await
827    {
828        tracing::error!(task_id, err = %e, "failed to move a task whose CI failed before review");
829    }
830}
831
832async fn recorded(
833    store: &Arc<TaskStore>,
834    repo: &str,
835    task_id: &str,
836    reason: &str,
837    cause: &str,
838    external: bool,
839    outcome: ReviewOutcome,
840) -> ReviewOutcome {
841    let store_id = match store.resolve_task_id(repo, task_id).await {
842        Ok(Some(id)) => id,
843        Ok(None) => {
844            tracing::warn!(
845                task_id,
846                "review attempt not recorded — task not found in store"
847            );
848            return outcome;
849        }
850        Err(e) => {
851            tracing::warn!(task_id, err = %e, "review attempt not recorded — task id lookup failed");
852            return outcome;
853        }
854    };
855
856    if let Err(e) = store
857        .append_activity(
858            store_id,
859            "review_attempt_failed",
860            None,
861            None,
862            None,
863            None,
864            Some(&serde_json::json!({
865                "cause": cause,
866                "external": external,
867                "reason": reason,
868            })),
869        )
870        .await
871    {
872        tracing::warn!(task_id, err = %e, "failed to record review attempt failure");
873    }
874
875    if external
876        && let Err(e) = store
877            .set_fields(store_id, &[("needs_review_refires", serde_json::json!(0))])
878            .await
879    {
880        tracing::warn!(task_id, err = %e, "failed to clear needs_review_refires after external review failure");
881    }
882
883    outcome
884}
885
886/// Resets that end with no decision and do not count as a failure, before the task blocks
887pub(crate) const MAX_REVIEW_RESETS: u64 = 6;
888
889/// A reset that does not count toward `MAX_REVIEW_AGENT_FAILURES` still raises `review_resets`,
890/// so a review that never produces a decision stops bouncing between needs_review and in_review
891async fn reset_or_block(
892    store: &Arc<TaskStore>,
893    repo: &str,
894    task_id: &str,
895    reason: &str,
896    cause: &str,
897    external: bool,
898    context: &str,
899) -> ReviewOutcome {
900    let resets = match store::store_increment(
901        &Some(Arc::clone(store)),
902        repo,
903        task_id,
904        "review_resets",
905    )
906    .await
907    {
908        Ok(v) => v,
909        Err(e) => {
910            tracing::warn!(task_id, err = %e, "failed to increment review_resets");
911            0
912        }
913    };
914    let outcome = if resets >= MAX_REVIEW_RESETS {
915        tracing::error!(
916            task_id,
917            reason,
918            resets,
919            "{context} ended {resets} times without a decision, blocking task"
920        );
921        ReviewOutcome::Block(format!(
922            "{context} ended {resets} times without a decision: {reason}"
923        ))
924    } else {
925        ReviewOutcome::Reset
926    };
927    recorded(store, repo, task_id, reason, cause, external, outcome).await
928}
929
930/// Classify a review failure and return the appropriate outcome.
931///
932/// Both the `Ok(ReviewDecision::Failed)` and `Err(e)` arms share the same
933/// logic; callers convert the error to a string before calling this helper.
934/// `context` is used in log messages to distinguish the two call sites
935/// ("review agent" vs "review_and_merge").
936async fn classify_review_failure(
937    store: &Arc<TaskStore>,
938    repo: &str,
939    task_id: &str,
940    reason: &str,
941    context: &str,
942) -> ReviewOutcome {
943    // Rate-limit and auth failures are not real review failures — don't count them
944    // toward MAX_REVIEW_AGENT_FAILURES. The cooldown system steers the next
945    // attempt to an available agent.
946    let lower_reason = reason.to_lowercase();
947
948    // Parse errors are orch's fault, not review feedback. Re-route for retry
949    // without counting toward the failure threshold, so parse errors don't
950    // incorrectly block tasks that have no actual review feedback.
951    if lower_reason.contains("parse error") {
952        tracing::warn!(
953            task_id,
954            reason,
955            "{context} hit parse error — re-routing for retry without counting as failure"
956        );
957        return reset_or_block(store, repo, task_id, reason, "parse_error", false, context).await;
958    }
959
960    // Session exited mid-thought with no result envelope (issue #3734). Same
961    // treatment as a parse error: retry without counting toward the failure cap
962    if lower_reason.contains("incomplete:") {
963        tracing::warn!(
964            task_id,
965            reason,
966            "{context} session incomplete — re-routing for retry without counting as failure"
967        );
968        return reset_or_block(
969            store,
970            repo,
971            task_id,
972            reason,
973            "incomplete_session",
974            false,
975            context,
976        )
977        .await;
978    }
979
980    // The review output was cut off by the model's output/reasoning token
981    // budget, not a genuine parse/format failure. Same treatment as a parse
982    // error: re-route without counting toward MAX_REVIEW_AGENT_FAILURES.
983    if lower_reason.contains("truncated") {
984        tracing::warn!(
985            task_id,
986            reason,
987            "{context} output truncated by token budget — re-routing for retry without counting as failure"
988        );
989        return reset_or_block(
990            store,
991            repo,
992            task_id,
993            reason,
994            "truncated_output",
995            false,
996            context,
997        )
998        .await;
999    }
1000
1001    // Transient mergeability check: GitHub hasn't finished computing yet.
1002    // This is a normal, expected transient state for freshly-pushed PRs.
1003    // Return RetryMerge (not Reset) so the task stays in InReview — avoids
1004    // spawning a redundant review agent that would just re-approve anyway.
1005    if lower_reason.contains("not yet computed") {
1006        tracing::warn!(
1007            task_id,
1008            reason,
1009            "{context} PR mergeability check still pending — will retry merge without re-reviewing"
1010        );
1011        return recorded(
1012            store,
1013            repo,
1014            task_id,
1015            reason,
1016            "mergeability_pending",
1017            true,
1018            ReviewOutcome::RetryMerge,
1019        )
1020        .await;
1021    }
1022
1023    // Merge conflicts should be handled by handle_merge_conflict() in
1024    // auto_merge.rs (which routes to the task agent). If a conflict error
1025    // leaks here, route to the task agent instead of resetting to NeedsReview
1026    // (which would re-spawn the review agent in a loop).
1027    if lower_reason.contains("merge conflict") || lower_reason.contains("not mergeable") {
1028        tracing::warn!(
1029            task_id,
1030            reason,
1031            "{context} hit merge conflict — routing to task agent to resolve"
1032        );
1033        return recorded(
1034            store,
1035            repo,
1036            task_id,
1037            reason,
1038            "merge_conflict",
1039            false,
1040            ReviewOutcome::Reroute,
1041        )
1042        .await;
1043    }
1044
1045    // Any other merge failure on approved work (rate limit, 5xx, stale lease, a check
1046    // still pending) is orch's problem, not the agent's. Keep the task in InReview so
1047    // review_open_prs retries the merge once the cause clears, and never re-dispatch the
1048    // agent — that burned attempts until the task blocked on work that was already done
1049    if lower_reason.contains("merge failed") {
1050        tracing::warn!(
1051            task_id,
1052            reason,
1053            "{context} merge failed without a conflict — retrying the merge, not re-dispatching the agent"
1054        );
1055        return recorded(
1056            store,
1057            repo,
1058            task_id,
1059            reason,
1060            "merge_retry",
1061            true,
1062            ReviewOutcome::RetryMerge,
1063        )
1064        .await;
1065    }
1066
1067    // No configured review model for any available agent, or the selected agent
1068    // has no review model. This is a configuration/cooldown gap, not a review
1069    // failure. Defer without incrementing the failure counter so the task retries
1070    // once a review model is available.
1071    if lower_reason.contains("no review agent") || lower_reason.contains("review model missing") {
1072        tracing::warn!(
1073            task_id,
1074            reason,
1075            "{context}: no review model available — deferring retry until one is available"
1076        );
1077        return recorded(
1078            store,
1079            repo,
1080            task_id,
1081            reason,
1082            "no_review_model",
1083            true,
1084            ReviewOutcome::RateLimited,
1085        )
1086        .await;
1087    }
1088
1089    // GhHttp's own throttle says "rate-limited", and GitHub answers 429 or abuse detection
1090    // without the words "rate limit"
1091    if lower_reason.contains("rate limit")
1092        || lower_reason.contains("rate-limited")
1093        || lower_reason.contains("rate limited")
1094        || lower_reason.contains("too many requests")
1095        || lower_reason.contains("abuse detection")
1096        || lower_reason.contains("failed (429)")
1097        || lower_reason.contains("auth error")
1098    {
1099        tracing::warn!(
1100            task_id,
1101            reason,
1102            "{context} hit rate limit or auth error — deferring retry until cooldown expires"
1103        );
1104        return recorded(
1105            store,
1106            repo,
1107            task_id,
1108            reason,
1109            "rate_limit",
1110            true,
1111            ReviewOutcome::RateLimited,
1112        )
1113        .await;
1114    }
1115
1116    if git_ops::is_transient_github_api_error(reason) {
1117        tracing::warn!(
1118            task_id,
1119            reason,
1120            "{context} hit transient GitHub error — deferring retry without counting as failure"
1121        );
1122        return reset_or_block(
1123            store,
1124            repo,
1125            task_id,
1126            reason,
1127            "github_api_error",
1128            true,
1129            context,
1130        )
1131        .await;
1132    }
1133
1134    // Provider-side network errors (ConnectionRefused, timeouts, etc.) are
1135    // transient infrastructure issues, not review feedback. Re-route without
1136    // counting toward MAX_REVIEW_AGENT_FAILURES.
1137    if patterns::detect_network_error(reason).is_some() {
1138        tracing::warn!(
1139            task_id,
1140            reason,
1141            "{context} hit transient provider network error — re-routing for retry without counting as failure"
1142        );
1143        return reset_or_block(
1144            store,
1145            repo,
1146            task_id,
1147            reason,
1148            "provider_network_error",
1149            true,
1150            context,
1151        )
1152        .await;
1153    }
1154
1155    let failures = match store::store_increment(
1156        &Some(Arc::clone(store)),
1157        repo,
1158        task_id,
1159        "review_agent_failures",
1160    )
1161    .await
1162    {
1163        Ok(v) => v,
1164        Err(e) => {
1165            tracing::warn!(task_id, err = %e, "failed to increment review_agent_failures — skipping failure escalation this tick");
1166            // Treat as 0 for now (no escalation)
1167            0
1168        }
1169    };
1170    let blocking = failures >= MAX_REVIEW_AGENT_FAILURES;
1171    // Post failure comment to the PR so the history is visible.
1172    post_review_failure_comment(store, repo, task_id, reason, failures, blocking).await;
1173    if blocking {
1174        tracing::error!(
1175            task_id,
1176            reason,
1177            failures,
1178            "{context} failed too many times — blocking task"
1179        );
1180        recorded(
1181            store,
1182            repo,
1183            task_id,
1184            reason,
1185            "review_failure",
1186            false,
1187            ReviewOutcome::Block(format!("{context} failed {failures} times: {reason}")),
1188        )
1189        .await
1190    } else {
1191        tracing::error!(
1192            task_id,
1193            reason,
1194            failures,
1195            "{context} failed — resetting to NeedsReview for retry"
1196        );
1197        recorded(
1198            store,
1199            repo,
1200            task_id,
1201            reason,
1202            "review_failure",
1203            false,
1204            ReviewOutcome::Reset,
1205        )
1206        .await
1207    }
1208}
1209
1210/// Post a comment on the PR explaining why the review agent failed.
1211/// Best-effort — if the PR number is missing or the API call fails, we just log.
1212async fn post_review_failure_comment(
1213    store: &Arc<TaskStore>,
1214    repo: &str,
1215    task_id: &str,
1216    reason: &str,
1217    failures: u64,
1218    blocking: bool,
1219) {
1220    let pr_number = match opt_store_get_task(&Some(Arc::clone(store)), repo, task_id).await {
1221        Some(t) => match t.pr_number {
1222            Some(n) => n,
1223            None => return, // No PR to comment on
1224        },
1225        _ => return, // No PR to comment on
1226    };
1227
1228    let status = if blocking {
1229        "Task blocked for human review."
1230    } else {
1231        "Retrying with a different agent."
1232    };
1233
1234    let comment = format!(
1235        "⚠️ Review agent failed (attempt {}/{})\n\n**Reason:** {}\n\n{}{}",
1236        failures,
1237        MAX_REVIEW_AGENT_FAILURES,
1238        reason,
1239        status,
1240        engine::orch_footer()
1241    );
1242
1243    let Ok(gh) = GhHttp::new() else { return };
1244    if let Err(e) = gh.add_comment(repo, &pr_number.to_string(), &comment).await {
1245        tracing::debug!(
1246            task_id,
1247            pr_number,
1248            error = %e,
1249            "failed to post review failure comment"
1250        );
1251    }
1252}
1253
1254#[cfg(test)]
1255mod tests {
1256    use super::*;
1257
1258    /// An incomplete mid-thought review (no result envelope) must reset for
1259    /// retry without counting toward MAX_REVIEW_AGENT_FAILURES (issue #3734).
1260    #[tokio::test]
1261    async fn classify_review_failure_incomplete_resets_without_counting() {
1262        let store = Arc::new(TaskStore::open_memory().await.unwrap());
1263        let repo = "owner/repo";
1264        let store_id = store
1265            .create_internal(
1266                repo,
1267                "Incomplete review test",
1268                "",
1269                "review",
1270                "review-3734",
1271                None,
1272            )
1273            .await
1274            .unwrap();
1275        let task_id = format!("internal:{store_id}");
1276
1277        for _ in 0..=MAX_REVIEW_AGENT_FAILURES {
1278            let outcome = classify_review_failure(
1279                &store,
1280                repo,
1281                &task_id,
1282                "incomplete: review session exited before producing a decision",
1283                "review agent",
1284            )
1285            .await;
1286            assert!(
1287                matches!(outcome, ReviewOutcome::Reset),
1288                "incomplete review failure must reset for retry, not block"
1289            );
1290        }
1291
1292        let task = opt_store_get_task(&Some(Arc::clone(&store)), repo, &task_id)
1293            .await
1294            .expect("task should exist");
1295        assert_eq!(
1296            task.review_agent_failures, 0,
1297            "incomplete failures must not count toward review_agent_failures"
1298        );
1299    }
1300
1301    /// A truncated review output (opencode step_finish reason=length) must be
1302    /// treated like a parse error: reset for retry without counting toward
1303    /// MAX_REVIEW_AGENT_FAILURES, so a run of token-budget flukes never
1304    /// permanently blocks the task (issue #3473).
1305    #[tokio::test]
1306    async fn classify_review_failure_truncated_resets_without_counting() {
1307        let store = Arc::new(TaskStore::open_memory().await.unwrap());
1308        let repo = "owner/repo";
1309        let store_id = store
1310            .create_internal(
1311                repo,
1312                "Truncated review test",
1313                "",
1314                "review",
1315                "review-1",
1316                None,
1317            )
1318            .await
1319            .unwrap();
1320        let task_id = format!("internal:{store_id}");
1321
1322        for _ in 0..=MAX_REVIEW_AGENT_FAILURES {
1323            let outcome = classify_review_failure(
1324                &store,
1325                repo,
1326                &task_id,
1327                "truncated: review output was cut off before completion \
1328                 (output/reasoning token budget exceeded)",
1329                "review agent",
1330            )
1331            .await;
1332            assert!(
1333                matches!(outcome, ReviewOutcome::Reset),
1334                "truncated review failure must reset for retry, not block"
1335            );
1336        }
1337
1338        let task = opt_store_get_task(&Some(Arc::clone(&store)), repo, &task_id)
1339            .await
1340            .expect("task should exist");
1341        assert_eq!(
1342            task.review_agent_failures, 0,
1343            "truncated failures must not count toward review_agent_failures"
1344        );
1345    }
1346
1347    /**
1348     * After an approved review, a merge that fails for anything other than a content
1349     * conflict must retry the merge with the task still in InReview. Re-dispatching the
1350     * agent re-ran finished work until the task hit the attempt limit (#3783)
1351     **/
1352    #[tokio::test]
1353    async fn classify_review_failure_non_conflict_merge_failure_retries_merge() {
1354        let store = Arc::new(TaskStore::open_memory().await.unwrap());
1355        let repo = "owner/repo";
1356        let store_id = store
1357            .create_internal(repo, "Merge retry", "", "review", "review-3783", None)
1358            .await
1359            .unwrap();
1360        let task_id = format!("internal:{store_id}");
1361
1362        let error_messages = [
1363            "merge failed: GitHub REST API rate-limited, skipping call for 187s",
1364            "merge failed: GitHub PR merge failed (502 Bad Gateway)",
1365            "merge failed: required status check still pending",
1366        ];
1367
1368        for _ in 0..=MAX_REVIEW_AGENT_FAILURES {
1369            for msg in &error_messages {
1370                let outcome =
1371                    classify_review_failure(&store, repo, &task_id, msg, "review agent").await;
1372                assert!(
1373                    matches!(outcome, ReviewOutcome::RetryMerge),
1374                    "non-conflict merge failure '{msg}' must retry the merge, not re-dispatch the agent"
1375                );
1376            }
1377        }
1378
1379        let task = opt_store_get_task(&Some(Arc::clone(&store)), repo, &task_id)
1380            .await
1381            .expect("task should exist");
1382        assert_eq!(
1383            task.review_agent_failures, 0,
1384            "a stuck merge is not a review failure"
1385        );
1386    }
1387
1388    /// A genuine content conflict is the one merge failure the agent has to fix
1389    #[tokio::test]
1390    async fn classify_review_failure_merge_conflict_still_reroutes() {
1391        let store = Arc::new(TaskStore::open_memory().await.unwrap());
1392        let repo = "owner/repo";
1393        let store_id = store
1394            .create_internal(repo, "Merge conflict", "", "review", "review-3783b", None)
1395            .await
1396            .unwrap();
1397        let task_id = format!("internal:{store_id}");
1398
1399        for msg in [
1400            "merge failed: PR is not mergeable",
1401            "merge conflict detected on PR #12",
1402        ] {
1403            let outcome =
1404                classify_review_failure(&store, repo, &task_id, msg, "review agent").await;
1405            assert!(
1406                matches!(outcome, ReviewOutcome::Reroute),
1407                "content conflict '{msg}' must go back to the task agent"
1408            );
1409        }
1410    }
1411
1412    /// Transient provider network errors (ConnectionRefused, timeouts) must be
1413    /// treated like parse errors: reset for retry without counting toward
1414    /// MAX_REVIEW_AGENT_FAILURES (issue #3482).
1415    #[tokio::test]
1416    async fn classify_review_failure_network_error_resets_without_counting() {
1417        let store = Arc::new(TaskStore::open_memory().await.unwrap());
1418        let repo = "owner/repo";
1419        let store_id = store
1420            .create_internal(
1421                repo,
1422                "Network error review test",
1423                "",
1424                "review",
1425                "review-2",
1426                None,
1427            )
1428            .await
1429            .unwrap();
1430        let task_id = format!("internal:{store_id}");
1431
1432        let error_messages = [
1433            "API Error: Unable to connect to API (ConnectionRefused)",
1434            "connection refused",
1435            "ECONNREFUSED",
1436            "Upstream idle timeout exceeded",
1437            "connect ETIMEDOUT",
1438            "network request failed",
1439        ];
1440
1441        for msg in &error_messages {
1442            // Each message on its own, the reset cap has its own test
1443            store
1444                .set_fields(store_id, &[("review_resets", serde_json::json!(0))])
1445                .await
1446                .unwrap();
1447            let outcome =
1448                classify_review_failure(&store, repo, &task_id, msg, "review agent").await;
1449            assert!(
1450                matches!(outcome, ReviewOutcome::Reset),
1451                "network error '{msg}' must reset for retry, not block"
1452            );
1453        }
1454
1455        let task = opt_store_get_task(&Some(Arc::clone(&store)), repo, &task_id)
1456            .await
1457            .expect("task should exist");
1458        assert_eq!(
1459            task.review_agent_failures, 0,
1460            "network errors must not count toward review_agent_failures"
1461        );
1462    }
1463
1464    /// When no agent has a configured review model (or the fallback agent has none),
1465    /// the review must be deferred, not counted as a review failure. Counting it
1466    /// would eventually block tasks for a configuration/cooldown gap (issue #3584).
1467    #[tokio::test]
1468    async fn classify_review_failure_missing_review_model_is_rate_limited() {
1469        let store = Arc::new(TaskStore::open_memory().await.unwrap());
1470        let repo = "owner/repo";
1471        let store_id = store
1472            .create_internal(
1473                repo,
1474                "Missing review model test",
1475                "",
1476                "review",
1477                "review-3",
1478                None,
1479            )
1480            .await
1481            .unwrap();
1482        let task_id = format!("internal:{store_id}");
1483
1484        let error_messages = [
1485            "no review agent with a configured review model is available for task internal:123",
1486            "review model missing for task internal:123 after agent selection — refusing to dispatch",
1487        ];
1488
1489        for _ in 0..=MAX_REVIEW_AGENT_FAILURES {
1490            for msg in &error_messages {
1491                let outcome =
1492                    classify_review_failure(&store, repo, &task_id, msg, "review_and_merge").await;
1493                assert!(
1494                    matches!(outcome, ReviewOutcome::RateLimited),
1495                    "missing review model error '{msg}' must defer, not block"
1496                );
1497            }
1498        }
1499
1500        let task = opt_store_get_task(&Some(Arc::clone(&store)), repo, &task_id)
1501            .await
1502            .expect("task should exist");
1503        assert_eq!(
1504            task.review_agent_failures, 0,
1505            "missing review model errors must not count toward review_agent_failures"
1506        );
1507    }
1508
1509    /// Resets that do not count as failures still stop at `MAX_REVIEW_RESETS`
1510    #[tokio::test]
1511    async fn classify_review_failure_blocks_after_max_resets() {
1512        let store = Arc::new(TaskStore::open_memory().await.unwrap());
1513        let repo = "owner/repo";
1514        let store_id = store
1515            .create_internal(repo, "Looping review", "", "review", "review-resets", None)
1516            .await
1517            .unwrap();
1518        let task_id = format!("internal:{store_id}");
1519        let msg = "parse error: failed to parse review response";
1520
1521        for attempt in 1..MAX_REVIEW_RESETS {
1522            let outcome =
1523                classify_review_failure(&store, repo, &task_id, msg, "review agent").await;
1524            assert!(
1525                matches!(outcome, ReviewOutcome::Reset),
1526                "reset {attempt} must retry the review"
1527            );
1528        }
1529        let outcome = classify_review_failure(&store, repo, &task_id, msg, "review agent").await;
1530        assert!(
1531            matches!(outcome, ReviewOutcome::Block(ref r) if r.contains("without a decision")),
1532            "reset {MAX_REVIEW_RESETS} must block the task"
1533        );
1534
1535        let task = opt_store_get_task(&Some(Arc::clone(&store)), repo, &task_id)
1536            .await
1537            .expect("task should exist");
1538        assert_eq!(
1539            u64::try_from(task.review_resets).unwrap(),
1540            MAX_REVIEW_RESETS
1541        );
1542        assert_eq!(
1543            task.review_agent_failures, 0,
1544            "a reset must not count as a review failure"
1545        );
1546    }
1547
1548    /**
1549     * A review that fails on a rate limit is retried once the cooldown clears, and must
1550     * never spend the rebroadcast quota: `needs_review_refires` is cleared so the sync
1551     * sweep cannot escalate the task to Blocked for a cause outside the review (#3767)
1552     **/
1553    #[tokio::test]
1554    async fn classify_review_failure_rate_limit_clears_refire_quota() {
1555        let store = Arc::new(TaskStore::open_memory().await.unwrap());
1556        let repo = "owner/repo";
1557        let store_id = store
1558            .create_internal(
1559                repo,
1560                "Rate limited review",
1561                "",
1562                "review",
1563                "review-3767",
1564                None,
1565            )
1566            .await
1567            .unwrap();
1568        let task_id = format!("internal:{store_id}");
1569
1570        let error_messages = [
1571            "claude failed: rate limit reached for model sonnet",
1572            "GitHub API GET https://api.github.com/repos/owner/repo/pulls/65 failed (403): API rate limit exceeded",
1573            "GitHub core API rate-limited, skipping call for 1800s",
1574            "GitHub API GET https://api.github.com/repos/owner/repo/pulls/65 failed (429): Too Many Requests",
1575        ];
1576
1577        // The sweep has already burned most of the quota before the rate-limit window
1578        store
1579            .set_fields(store_id, &[("needs_review_refires", serde_json::json!(5))])
1580            .await
1581            .unwrap();
1582
1583        for _ in 0..=MAX_REVIEW_AGENT_FAILURES {
1584            for msg in &error_messages {
1585                let outcome =
1586                    classify_review_failure(&store, repo, &task_id, msg, "review agent").await;
1587                assert!(
1588                    matches!(outcome, ReviewOutcome::RateLimited | ReviewOutcome::Reset),
1589                    "rate limit '{msg}' must defer for retry, not block"
1590                );
1591            }
1592        }
1593
1594        let task = opt_store_get_task(&Some(Arc::clone(&store)), repo, &task_id)
1595            .await
1596            .expect("task should exist");
1597        assert_eq!(
1598            task.needs_review_refires, 0,
1599            "a rate limit must not consume the rebroadcast quota"
1600        );
1601        assert_eq!(
1602            task.review_agent_failures, 0,
1603            "rate limits must not count toward review_agent_failures"
1604        );
1605        assert_ne!(
1606            task.status,
1607            TaskStatus::Blocked,
1608            "a rate-limited review must leave the task retryable"
1609        );
1610
1611        let attempts = store
1612            .get_activity(store_id, None)
1613            .await
1614            .unwrap()
1615            .into_iter()
1616            .filter(|a| a.event_type == "review_attempt_failed")
1617            .collect::<Vec<_>>();
1618        assert_eq!(
1619            attempts.len(),
1620            (usize::try_from(MAX_REVIEW_AGENT_FAILURES).unwrap() + 1) * error_messages.len(),
1621            "every failed attempt must be recorded for `orch task inspect`"
1622        );
1623        assert!(
1624            attempts.iter().all(|a| a
1625                .details
1626                .get("external")
1627                .and_then(serde_json::Value::as_bool)
1628                == Some(true)),
1629            "rate limits must be recorded as an external cause"
1630        );
1631        assert!(
1632            attempts.iter().any(
1633                |a| a.details.get("reason").and_then(|v| v.as_str()) == Some(error_messages[0])
1634            ),
1635            "the recorded attempt must carry that attempt's error"
1636        );
1637    }
1638}