Skip to main content

orch/engine/subscribers/
review.rs

1//! Reacts to NeedsReview events — spawns review agent immediately.
2
3use crate::backends::{ExternalBackend, ExternalId, ExternalTask, Status};
4use crate::engine::cooldown::AGENT_COOLDOWN_SECS;
5use crate::engine::dispatch_guard::DispatchGuard;
6use crate::engine::events::TaskEvent;
7use crate::engine::review::{MAX_REVIEW_AGENT_FAILURES, ReviewDecision, review_and_merge};
8use crate::engine::review_gate::CiGate;
9use crate::engine::review_shipcheck::Prerun;
10use crate::engine::router::Router;
11use crate::engine::tasks::TaskManager;
12use crate::github::http::GhHttp;
13use crate::repo_context::REPO_CONTEXT;
14use crate::store::{TaskStore, opt_store_get_task};
15use crate::tmux::TmuxManager;
16use dashmap::DashMap;
17use std::sync::Arc;
18use tokio::sync::broadcast;
19use tokio::sync::{RwLock, Semaphore};
20
21enum ReviewOutcome {
22    Reset,
23    RateLimited,
24    /// The PR was approved but the merge did not land for a reason outside the
25    /// review (mergeability not computed yet, rate limit, transient GitHub error).
26    /// The task moves to Approved so `review_open_prs` retries the merge, instead of
27    /// resetting to NeedsReview (which spawns a redundant review agent)
28    RetryMerge,
29    /// The review approved the PR. The task moves to Approved, where orch waits for CI and
30    /// merges with no agent running
31    Approved,
32    /// Re-route to the task agent (e.g. merge conflicts that need the agent to fix).
33    Reroute,
34    Block(String),
35    Ok,
36}
37
38/// Spawn a task that listens for NeedsReview events and triggers the review agent.
39///
40/// This is the sole trigger for NeedsReview → InReview transitions. The sync tick
41/// no longer has a competing NeedsReview loop (removed in fix for issue #857 — both
42/// paths firing simultaneously caused double failure-counter increments).
43#[expect(
44    clippy::too_many_arguments,
45    reason = "the arguments have no natural grouping"
46)]
47pub fn spawn(
48    mut rx: broadcast::Receiver<TaskEvent>,
49    backend: Arc<dyn ExternalBackend>,
50    tmux: Arc<TmuxManager>,
51    semaphore: Arc<Semaphore>,
52    task_manager: Arc<TaskManager>,
53    router: Arc<RwLock<Router>>,
54    dispatching: Arc<DashMap<String, String>>,
55    store: Arc<TaskStore>,
56    repo: String,
57) {
58    tokio::spawn(async move {
59        loop {
60            match rx.recv().await {
61                Ok(event) if event.new_status == "needs_review" && event.repo == repo => {
62                    // Check if review agent is enabled (read fresh each time).
63                    let enable_review = crate::config::get("workflow.enable_review_agent")
64                        .map_or(true, |v| v != "false");
65                    if !enable_review {
66                        continue;
67                    }
68
69                    let task_id = &event.task_id;
70                    let dispatch_key = format!("{repo}/{task_id}");
71
72                    // Atomically claim the task before any async work so concurrent
73                    // review events cannot double-spawn the same review flow.
74                    // Using DashMap lets us distinguish same-task duplicates (expected) from
75                    // key collisions involving a different task (should never happen).
76                    {
77                        use dashmap::mapref::entry::Entry;
78                        match dispatching.entry(dispatch_key.clone()) {
79                            Entry::Occupied(existing) => {
80                                let existing_id = existing.get().clone();
81                                drop(existing);
82                                if existing_id == *task_id {
83                                    tracing::debug!(
84                                        task_id,
85                                        "task locked by dispatch flow, skipping event-driven review"
86                                    );
87                                } else {
88                                    tracing::warn!(
89                                        task_id,
90                                        existing_task_id = existing_id,
91                                        dispatch_key,
92                                        "dispatch key collision: unexpected task already holds this key"
93                                    );
94                                }
95                                continue;
96                            }
97                            Entry::Vacant(slot) => {
98                                slot.insert(task_id.clone());
99                            }
100                        }
101                    }
102
103                    // Create guard IMMEDIATELY after inserting the key — owns removal for all subsequent exit paths
104                    let dispatch_guard = DispatchGuard::new(
105                        std::sync::Arc::clone(&dispatching),
106                        dispatch_key.clone(),
107                    );
108
109                    tracing::info!(task_id, "event-driven review triggered");
110
111                    // Look up the task from the store to get an ExternalTask.
112                    let task = {
113                        if crate::engine::tasks::is_internal_id(task_id)
114                            || match store.has_external_tasks(&repo).await {
115                                Ok(has_external) => has_external,
116                                Err(e) => {
117                                    tracing::warn!(
118                                        repo = %repo,
119                                        task_id,
120                                        error = %e,
121                                        "failed to check external task sentinel; keeping store-first review lookup"
122                                    );
123                                    true
124                                }
125                            }
126                        {
127                            // O(1) indexed lookup — resolve task_id to its SQLite row id,
128                            // then fetch directly instead of scanning the whole needs_review set.
129                            let result = if let Some(numeric_id) =
130                                crate::engine::tasks::parse_internal_id(task_id)
131                            {
132                                store.get(numeric_id).await.map(Some)
133                            } else {
134                                store.get_by_external_id(&repo, task_id).await
135                            };
136                            match result {
137                                Ok(Some(t))
138                                    if t.status == crate::store::TaskStatus::NeedsReview =>
139                                {
140                                    Some(crate::engine::tasks::store_task_to_external(&t))
141                                }
142                                Ok(Some(t)) => {
143                                    tracing::debug!(
144                                        task_id,
145                                        status = t.status.as_str(),
146                                        "task found but not in needs_review status, skipping"
147                                    );
148                                    None
149                                }
150                                Ok(None) => None,
151                                Err(e) => {
152                                    tracing::warn!(task_id, err = %e, "store lookup failed for review");
153                                    None
154                                }
155                            }
156                        } else {
157                            // Fall back to backend
158                            match backend.list_by_status(Status::NeedsReview).await {
159                                Ok(tasks) => tasks.into_iter().find(|t| t.id.0 == *task_id),
160                                Err(e) => {
161                                    tracing::warn!(task_id, err = %e, "backend lookup failed for review");
162                                    None
163                                }
164                            }
165                        }
166                    };
167
168                    let Some(task) = task else {
169                        tracing::debug!(
170                            task_id,
171                            "task not found in needs_review list (may have already transitioned)"
172                        );
173                        continue;
174                    };
175
176                    if let Some(stored) =
177                        opt_store_get_task(&Some(std::sync::Arc::clone(&store)), &repo, &task.id.0)
178                            .await
179                    {
180                        let no_code_review = stored.worktree.trim().is_empty()
181                            && stored.branch.trim().is_empty()
182                            && stored.pr_number.is_none();
183                        if no_code_review {
184                            tracing::info!(
185                                task_id,
186                                "no worktree/branch/pr for needs_review task — skipping review dispatch and marking done"
187                            );
188                            crate::store::store_reset_failure_counters(
189                                &Some(std::sync::Arc::clone(&store)),
190                                &repo,
191                                &task.id.0,
192                            )
193                            .await;
194                            if let Err(e) = crate::store::store_set_result(
195                                &Some(std::sync::Arc::clone(&store)),
196                                &repo,
197                                &task.id.0,
198                                &[("needs_review_refires", serde_json::json!(0))],
199                            )
200                            .await
201                            {
202                                tracing::warn!(
203                                    task_id,
204                                    err = %e,
205                                    "failed to reset needs_review_refires while skipping no-code review"
206                                );
207                            }
208                            if let Err(e) = task_manager
209                                .update_task_status(&task.id, Status::Done)
210                                .await
211                            {
212                                tracing::error!(
213                                    task_id,
214                                    err = %e,
215                                    "failed to mark no-code needs_review task as done"
216                                );
217                            }
218                            continue;
219                        }
220                    }
221
222                    // If no agents are currently routable (all cooled, degraded, or without
223                    // an available review model), skip this event and let sync re-fire when
224                    // an agent is available again. Sleeping inline here would block every
225                    // subsequent NeedsReview event for this repo for the duration of the
226                    // cooldown (potentially hours/days), causing sync's rebroadcast loop to
227                    // escalate piled-up tasks to Blocked.
228                    {
229                        let router_guard = router.read().await;
230                        let no_routable = !router_guard.available_agents.is_empty()
231                            && router_guard.healthy_agent_count("review") == 0;
232                        drop(router_guard);
233                        if no_routable {
234                            tracing::info!(
235                                task_id,
236                                "no routable review agents — leaving task in needs_review for sync to re-fire"
237                            );
238                            continue;
239                        }
240                    }
241
242                    if !ci_allows_review(&store, &task_manager, &repo, &task).await {
243                        continue;
244                    }
245
246                    // Try to acquire a semaphore permit.
247                    let Ok(permit) = std::sync::Arc::clone(&semaphore).try_acquire_owned() else {
248                        tracing::debug!(
249                            task_id,
250                            "all parallel slots busy; sync catch-up will re-fire when a slot opens"
251                        );
252                        continue;
253                    };
254
255                    // Transition to InReview — this IS the atomic guard against duplicates.
256                    if let Err(e) = task_manager
257                        .update_task_status(&task.id, Status::InReview)
258                        .await
259                    {
260                        tracing::warn!(task_id, err = %e, "failed to transition to InReview");
261                        drop(permit);
262                        continue;
263                    }
264
265                    let backend_c = std::sync::Arc::clone(&backend);
266                    let task_manager_c = std::sync::Arc::clone(&task_manager);
267                    let tmux_c = std::sync::Arc::clone(&tmux);
268                    let router_c = std::sync::Arc::clone(&router);
269                    let store_c = std::sync::Arc::clone(&store);
270                    let repo_s = repo.clone();
271                    tokio::spawn(REPO_CONTEXT.scope(repo_s.clone(), async move {
272                        let _dispatch_guard = dispatch_guard; // released on drop (normal or panic)
273                        let tid = task.id.0.clone();
274
275                        // Re-check agent routability inside the spawned task. A concurrent
276                        // review that ran between the outer dispatch check and now may have
277                        // set a cooldown — bail early rather than burning a review run
278                        // against a rate-limited agent.
279                        let pre_check_all_cooled = {
280                            let r = router_c.read().await;
281                            !r.available_agents.is_empty() && r.healthy_agent_count("review") == 0
282                        };
283                        if pre_check_all_cooled {
284                            tracing::info!(
285                                task_id = tid,
286                                "all agents cooled at review spawn time — resetting to needs_review for sync to re-fire"
287                            );
288                            if let Err(e) = task_manager_c
289                                .update_task_status(
290                                    &ExternalId(tid.clone()),
291                                    Status::NeedsReview,
292                                )
293                                .await
294                            {
295                                tracing::error!(task_id = %tid, err = %e, "pre-check: update_task_status(NeedsReview) failed — task may be stuck in InReview");
296                            }
297                            drop(permit);
298                            return;
299                        }
300
301                        let outcome = match Box::pin(review_and_merge(
302                            &task,
303                            &backend_c,
304                            &tmux_c,
305                            &repo_s,
306                            &router_c,
307                            &task_manager_c,
308                            &store_c,
309                        ))
310                        .await
311                        {
312                            Ok(ReviewDecision::Blocked(reason)) => {
313                                tracing::error!(
314                                    task_id = tid,
315                                    reason,
316                                    "review gate blocked after repeated failures — marking task blocked"
317                                );
318                                ReviewOutcome::Block(format!(
319                                    "review gate blocked after repeated failures: {reason}"
320                                ))
321                            }
322                            Ok(ReviewDecision::Failed(reason)) => {
323                                classify_review_failure(
324                                    &store_c, &repo_s, &tid, &reason, "review agent",
325                                )
326                                .await
327                            }
328                            Err(e) => {
329                                let reason = format!("{e:#}");
330                                classify_review_failure(
331                                    &store_c, &repo_s, &tid, &reason, "review_and_merge",
332                                )
333                                .await
334                            }
335                            Ok(ReviewDecision::Approve { .. }) => {
336                                // On approval we clear transient failure counters
337                                // but preserve the `review_cycles` counter which
338                                // tracks how many times the PR has requested
339                                // changes and is used as a circuit-breaker.
340                                crate::store::store_reset_failure_counters(
341                                    &Some(std::sync::Arc::clone(&store_c)),
342                                    &repo_s,
343                                    &tid,
344                                )
345                                .await;
346                                // Also reset needs_review_refires so the sync backoff doesn't persist across a successful review
347                                if let Err(e) = crate::store::store_set_result(
348                                    &Some(std::sync::Arc::clone(&store_c)),
349                                    &repo_s,
350                                    &tid,
351                                    &[("needs_review_refires", serde_json::json!(0))],
352                                )
353                                .await
354                                {
355                                    tracing::warn!(task_id = %tid, err = %e, "failed to write needs_review_refires to store");
356                                }
357
358                                // Approval only means the code looks good, the merge is the
359                                // completion signal. review_open_prs merges Approved tasks and
360                                // moves them to Done
361                                ReviewOutcome::Approved
362                            }
363                            Ok(ReviewDecision::Rerouted) => {
364                                // Task was re-routed because there was no PR and no commits.
365                                // Do NOT reset failure counters — no_code_reroutes must accumulate
366                                // so the max_reroute_attempts safety limit can be reached.
367                                ReviewOutcome::Ok
368                            }
369                            Ok(ReviewDecision::Skipped) => {
370                                // Skipped indicates there's nothing to review (no PR etc.).
371                                // Clear transient failure counters but preserve review_cycles.
372                                crate::store::store_reset_failure_counters(
373                                    &Some(std::sync::Arc::clone(&store_c)),
374                                    &repo_s,
375                                    &tid,
376                                )
377                                .await;
378                                if let Err(e) = crate::store::store_set_result(
379                                    &Some(std::sync::Arc::clone(&store_c)),
380                                    &repo_s,
381                                    &tid,
382                                    &[("needs_review_refires", serde_json::json!(0))],
383                                )
384                                .await
385                                {
386                                    tracing::warn!(task_id = %tid, err = %e, "failed to write needs_review_refires to store");
387                                }
388                                ReviewOutcome::Ok
389                            }
390                            Ok(ReviewDecision::RequestChanges { .. }) => {
391                                // handle_review_changes already incremented review_cycles —
392                                // only reset transient per-attempt counters.
393                                crate::store::store_reset_failure_counters(
394                                    &Some(std::sync::Arc::clone(&store_c)),
395                                    &repo_s,
396                                    &tid,
397                                )
398                                .await;
399                                if let Err(e) = crate::store::store_set_result(
400                                    &Some(std::sync::Arc::clone(&store_c)),
401                                    &repo_s,
402                                    &tid,
403                                    &[("needs_review_refires", serde_json::json!(0))],
404                                )
405                                .await
406                                {
407                                    tracing::warn!(task_id = %tid, err = %e, "failed to write needs_review_refires to store");
408                                }
409                                ReviewOutcome::Ok
410                            }
411                        };
412                        // Freshness guard: re-fetch the task's current status before applying
413                        // any outcome transitions. A concurrent path (e.g. PR merge, manual
414                        // close, or another review cycle) may have already moved the task out
415                        // of `in_review`. If so, discard the stale outcome rather than
416                        // overwriting the newer state.
417                        //
418                        // IMPORTANT: Distinguish between DB errors and legitimate status changes.
419                        // A DB lookup failure should NOT be treated as "task moved out of in_review"
420                        // — we should retain the review outcome rather than discarding it on
421                        // transient errors.
422                        let fetched: Result<Option<(crate::store::TaskStatus, Option<String>)>, anyhow::Error> = async {
423                            let id = match store_c.resolve_task_id(&repo_s, &tid).await {
424                                Ok(Some(id)) => id,
425                                Ok(None) => return Ok(None), // Task doesn't exist — treat as missing
426                                Err(e) => return Err(e),      // DB error — propagate, don't discard
427                            };
428                            let task = store_c.get(id).await?;
429                            Ok(Some((task.status, task.block_reason)))
430                        }
431                        .await;
432
433                        let (current_status, current_block_reason) = match fetched {
434                            Ok(Some((status, block_reason))) => (Some(status), block_reason),
435                            Ok(None) => {
436                                // Task doesn't exist in store — treat as not in_review
437                                (None, None)
438                            }
439                            Err(e) => {
440                                // DB error — log and keep the review outcome instead of discarding
441                                tracing::error!(
442                                    task_id = tid,
443                                    error = %e,
444                                    "review outcome retained — DB lookup failed, not treating as stale"
445                                );
446                                // Skip the stale outcome check and proceed with the outcome
447                                // by treating it as if the task is still InReview
448                                (Some(crate::store::TaskStatus::InReview), None)
449                            }
450                        };
451
452                        if !matches!(current_status, Some(crate::store::TaskStatus::InReview)) {
453                            let ci_blocked = matches!(
454                                current_status,
455                                Some(crate::store::TaskStatus::Blocked)
456                            ) && current_block_reason
457                                .as_deref()
458                                .is_some_and(|r| r.contains("CI failure limit"));
459                            let expected = ci_blocked
460                                || matches!(
461                                    current_status,
462                                    Some(crate::store::TaskStatus::Done | crate::store::TaskStatus::Approved |
463crate::store::TaskStatus::Routed | crate::store::TaskStatus::New)
464                                );
465                            if expected {
466                                tracing::debug!(
467                                    task_id = tid,
468                                    current_status = current_status.as_ref().map_or("unknown", crate::store::tasks::TaskStatus::as_str),
469                                    "review outcome discarded — task transitioned away from in_review (expected)"
470                                );
471                            } else {
472                                tracing::warn!(
473                                    task_id = tid,
474                                    current_status = current_status.as_ref().map_or("unknown", crate::store::tasks::TaskStatus::as_str),
475                                    "review outcome discarded — task is no longer in_review (stale review agent result)"
476                                );
477                            }
478
479                            // Kill the orphaned review session
480                            let stale_session = tmux_c.session_name(&repo_s, &format!("{tid}-review"));
481                            if let Err(e) = tmux_c.kill_session(&stale_session).await {
482                                tracing::debug!(task_id = tid, error = %e, "failed to kill stale review session");
483                            }
484
485                            drop(permit);
486                            return;
487                        }
488
489                        match outcome {
490                            ReviewOutcome::Reset => {
491                                // Kill any stale tmux review session before resetting — the
492                                // session may still be alive if the agent hit a rate limit,
493                                // if two concurrent reviewers raced and the loser's spawn
494                                // failed with "duplicate session", or if review_and_merge
495                                // returned an error before reaching its own cleanup.
496                                // Killing a non-existent session is a harmless no-op.
497                                let stale_session =
498                                    tmux_c.session_name(&repo_s, &format!("{tid}-review"));
499                                tmux_c.kill_session(&stale_session).await.ok();
500                                // Backoff before re-queuing to prevent rapid spin
501                                // if the review agent keeps failing instantly.
502                                tokio::time::sleep(std::time::Duration::from_secs(10)).await;
503                                if let Err(e) = task_manager_c
504                                    .update_task_status(
505                                        &ExternalId(tid.clone()),
506                                        Status::NeedsReview,
507                                    )
508                                    .await
509                                {
510                                    tracing::error!(task_id = %tid, err = %e, "update_task_status(NeedsReview) failed — task may be stuck in InReview");
511                                }
512                            }
513                            ReviewOutcome::Reroute => {
514                                // Merge conflict or similar — route back to the
515                                // task agent so it can fix the issue.
516                                let stale_session =
517                                    tmux_c.session_name(&repo_s, &format!("{tid}-review"));
518                                tmux_c.kill_session(&stale_session).await.ok();
519                                tracing::info!(
520                                    task_id = tid,
521                                    "re-routing to task agent to resolve merge conflict"
522                                );
523                                if let Err(e) = task_manager_c
524                                    .update_task_status(
525                                        &ExternalId(tid.clone()),
526                                        Status::Routed,
527                                    )
528                                    .await
529                                {
530                                    tracing::error!(task_id = %tid, err = %e, "update_task_status(Routed) failed");
531                                }
532                            }
533                            ReviewOutcome::Approved | ReviewOutcome::RetryMerge => {
534                                // The review is over, so its session goes. The merge may have
535                                // landed already, or CI sent the task back, so only in_review moves
536                                let stale_session =
537                                    tmux_c.session_name(&repo_s, &format!("{tid}-review"));
538                                tmux_c.kill_session(&stale_session).await.ok();
539                                match task_manager_c
540                                    .update_task_status_if(
541                                        &ExternalId(tid.clone()),
542                                        Status::Approved,
543                                        Status::InReview,
544                                    )
545                                    .await
546                                {
547                                    Ok(true) => tracing::info!(
548                                        task_id = tid,
549                                        "approval received, task approved until the PR is merged"
550                                    ),
551                                    Ok(false) => tracing::debug!(
552                                        task_id = tid,
553                                        "approval received after the task left in_review"
554                                    ),
555                                    Err(e) => tracing::error!(
556                                        task_id = %tid,
557                                        err = %e,
558                                        "update_task_status_if(Approved) failed, task may be stuck in in_review"
559                                    ),
560                                }
561                            }
562                            ReviewOutcome::RateLimited => {
563                                // Kill stale tmux session — same as Reset.
564                                let stale_session =
565                                    tmux_c.session_name(&repo_s, &format!("{tid}-review"));
566                                tmux_c.kill_session(&stale_session).await.ok();
567
568                                // Compute wait duration: if any agent is available right now,
569                                // use a short backoff (30 s). If every agent is cooled, sleep
570                                // until the earliest cooldown expires so we don't spin.
571                                let wait_secs = {
572                                    let r = router_c.read().await;
573                                    if r.healthy_agent_count("review") > 0 {
574                                        30u64
575                                    } else {
576                                        let now = chrono::Utc::now().timestamp();
577                                        r.earliest_cooldown_until(Some("review"))
578                                            .map_or(AGENT_COOLDOWN_SECS as u64, |until| u64::try_from((until - now).max(1)).unwrap_or(1))
579                                    }
580                                };
581                                tracing::info!(
582                                    task_id = tid,
583                                    wait_secs,
584                                    "deferring review retry due to agent rate limit cooldown"
585                                );
586                                tokio::time::sleep(std::time::Duration::from_secs(wait_secs)).await;
587                                if let Err(e) = task_manager_c
588                                    .update_task_status(
589                                        &ExternalId(tid.clone()),
590                                        Status::NeedsReview,
591                                    )
592                                    .await
593                                {
594                                    tracing::error!(task_id = %tid, err = %e, "update_task_status(NeedsReview) failed after rate limit backoff — task may be stuck in InReview");
595                                }
596                            }
597                             ReviewOutcome::Block(reason) => {
598                                 // Persist block_reason BEFORE transitioning to Blocked to avoid
599                                 // a race where auto_unblock sees a blocked task without a reason
600                                 // and immediately unblocks it.
601                                 let fields = [
602                                     (
603                                         "block_reason",
604                                         serde_json::json!("review agent blocked — exceeded failure threshold"),
605                                     ),
606                                     ("last_error", serde_json::json!(reason)),
607                                 ];
608                                 if let Err(e) = task_manager_c
609                                     .update_task_status_and_result(
610                                         &ExternalId(tid.clone()),
611                                         Status::Blocked,
612                                         &fields,
613                                     )
614                                     .await
615                                 {
616                                     tracing::error!(task_id = %tid, err = %e, "update_task_status_and_result(Blocked) failed — task may be stuck in InReview");
617                                 }
618                             }
619                            ReviewOutcome::Ok => {}
620                        }
621
622                        drop(permit);
623                        // _dispatch_guard dropped here — releases the per-task lock.
624                    }));
625
626                    // Small delay between dispatches to allow cooldowns from
627                    // early failures to propagate before subsequent reviews are
628                    // dispatched. This prevents batch dispatch of N reviews to
629                    // the same rate-limited agent when events arrive concurrently.
630                    tokio::time::sleep(std::time::Duration::from_secs(2)).await;
631                }
632                Ok(_) => {} // Not a needs_review event or different repo
633                Err(broadcast::error::RecvError::Lagged(n)) => {
634                    tracing::debug!(
635                        n,
636                        "review subscriber lagged; sync catch-up will re-fire missed NeedsReview tasks"
637                    );
638                }
639                Err(_) => break, // Channel closed
640            }
641        }
642    });
643}
644
645/**
646 * Record a failed review attempt on the task's activity timeline and hand the outcome
647 * back to `classify_review_failure`, which returns it unchanged. Without this the only
648 * trace of a failed attempt was the final `last_error` written at escalation, so
649 * `orch task inspect` showed that a task gave up but never why each attempt failed.
650 *
651 * `external` marks a cause outside the review itself (agent rate limit, GitHub API
652 * limit, no review model). Those also clear `needs_review_refires`: the refire quota
653 * exists to catch a task the subscriber never picks up, and spending it on a cause the
654 * review agent cannot control escalated healthy tasks to Blocked during a rate-limit
655 * window. The retry itself is already handled by the `RateLimited` / `Reset` arms
656 **/
657/// The CI gate (#3904): true when the review may start. A task held for CI stays in needs_review
658/// and its refire count goes back to 0, so the sync sweep checks it again and never blocks it
659async fn ci_allows_review(
660    store: &Arc<TaskStore>,
661    task_manager: &Arc<TaskManager>,
662    repo: &str,
663    task: &ExternalTask,
664) -> bool {
665    let task_id = task.id.0.as_str();
666    let stored = opt_store_get_task(&Some(std::sync::Arc::clone(store)), repo, task_id).await;
667    let Some(pr_number) = stored
668        .as_ref()
669        .and_then(|t| t.pr_number)
670        .and_then(|n| u64::try_from(n).ok())
671    else {
672        return true;
673    };
674    let worktree = stored
675        .map(|t| std::path::PathBuf::from(t.worktree))
676        .filter(|p| !p.as_os_str().is_empty() && p.exists());
677    let gate = match GhHttp::new() {
678        Ok(gh) => {
679            crate::engine::review_gate::review_ci_gate(&gh, store, repo, task_id, pr_number).await
680        }
681        Err(e) => Err(e),
682    };
683
684    let hold = match gate {
685        Ok(CiGate::Start) => return true,
686        // The merge still waits for green CI, so a lookup error does not hold the review
687        Err(e) => {
688            tracing::warn!(task_id, pr_number, err = %e, "CI gate lookup failed, starting the review");
689            return true;
690        }
691        Ok(CiGate::Red(failed)) => {
692            route_back_on_red_ci(store, task_manager, repo, task, pr_number, &failed).await;
693            return false;
694        }
695        Ok(CiGate::Wait) => "CI still running on the PR head, review waits",
696        // The reviewer runs shipcheck when no check ran, so orch runs it first, outside the review's limit
697        Ok(CiGate::NotStarted { waited_out }) => match local_ci_worktree(repo, worktree).await {
698            Some(wt) => match crate::engine::review_shipcheck::prerun(&wt).await {
699                Prerun::Ready => return true,
700                Prerun::Running => "shipcheck running on the PR head, review waits",
701            },
702            None if waited_out => {
703                tracing::warn!(
704                    task_id,
705                    pr_number,
706                    "no CI ran on the PR head, starting the review anyway"
707                );
708                return true;
709            }
710            None => "no CI started on the PR head yet, review waits",
711        },
712    };
713
714    tracing::info!(task_id, pr_number, "{hold}");
715    if let Err(e) = crate::store::store_set_result(
716        &Some(std::sync::Arc::clone(store)),
717        repo,
718        task_id,
719        &[("needs_review_refires", serde_json::json!(0))],
720    )
721    .await
722    {
723        tracing::warn!(task_id, err = %e, "failed to reset needs_review_refires for a task held by the CI gate");
724    }
725
726    false
727}
728
729/// The task worktree, when the reviewer there would run `shipcheck run`
730async fn local_ci_worktree(
731    repo: &str,
732    worktree: Option<std::path::PathBuf>,
733) -> Option<std::path::PathBuf> {
734    let wt = worktree?;
735    let sandboxed = crate::engine::review::review_sandboxed(repo);
736    let shipcheck = crate::engine::runner::agent::Shipcheck::detect(&wt)
737        .await
738        .for_prompt(sandboxed);
739    (shipcheck == crate::engine::runner::agent::Shipcheck::Available).then_some(wt)
740}
741
742/// CI failed before the review, so the task agent fixes it without a review round
743async fn route_back_on_red_ci(
744    store: &Arc<TaskStore>,
745    task_manager: &Arc<TaskManager>,
746    repo: &str,
747    task: &ExternalTask,
748    pr_number: u64,
749    failed: &[String],
750) {
751    let task_id = task.id.0.as_str();
752    let failures = match crate::store::store_increment(
753        &Some(std::sync::Arc::clone(store)),
754        repo,
755        task_id,
756        "ci_merge_failures",
757    )
758    .await
759    {
760        Ok(v) => v,
761        Err(e) => {
762            tracing::warn!(task_id, err = %e, "failed to increment ci_merge_failures, review waits");
763            return;
764        }
765    };
766    let checks = failed.join(", ");
767    let max = crate::engine::auto_merge::MAX_CI_MERGE_FAILURES;
768    let (status, fields) = if failures >= max {
769        tracing::error!(
770            task_id,
771            pr_number,
772            failures,
773            checks,
774            "CI failed before review, CI failure limit reached"
775        );
776        (
777            Status::Blocked,
778            vec![(
779                "block_reason",
780                serde_json::json!(format!("CI failure limit ({max}) reached before review")),
781            )],
782        )
783    } else {
784        tracing::warn!(
785            task_id,
786            pr_number,
787            failures,
788            checks,
789            "CI failed before review, routing back to the task agent"
790        );
791        (
792            Status::Routed,
793            vec![
794                (
795                    "last_error",
796                    serde_json::json!(format!(
797                        "CI failed on PR #{pr_number} before review: {checks}. Fix the failing checks"
798                    )),
799                ),
800                (
801                    "route_reason",
802                    serde_json::json!("re-dispatch after CI failed before review"),
803                ),
804            ],
805        )
806    };
807    if let Err(e) = task_manager
808        .update_task_status_and_result(&task.id, status, &fields)
809        .await
810    {
811        tracing::error!(task_id, err = %e, "failed to move a task whose CI failed before review");
812    }
813}
814
815async fn recorded(
816    store: &Arc<TaskStore>,
817    repo: &str,
818    task_id: &str,
819    reason: &str,
820    cause: &str,
821    external: bool,
822    outcome: ReviewOutcome,
823) -> ReviewOutcome {
824    let store_id = match store.resolve_task_id(repo, task_id).await {
825        Ok(Some(id)) => id,
826        Ok(None) => {
827            tracing::warn!(
828                task_id,
829                "review attempt not recorded — task not found in store"
830            );
831            return outcome;
832        }
833        Err(e) => {
834            tracing::warn!(task_id, err = %e, "review attempt not recorded — task id lookup failed");
835            return outcome;
836        }
837    };
838
839    if let Err(e) = store
840        .append_activity(
841            store_id,
842            "review_attempt_failed",
843            None,
844            None,
845            None,
846            None,
847            Some(&serde_json::json!({
848                "cause": cause,
849                "external": external,
850                "reason": reason,
851            })),
852        )
853        .await
854    {
855        tracing::warn!(task_id, err = %e, "failed to record review attempt failure");
856    }
857
858    if external
859        && let Err(e) = store
860            .set_fields(store_id, &[("needs_review_refires", serde_json::json!(0))])
861            .await
862    {
863        tracing::warn!(task_id, err = %e, "failed to clear needs_review_refires after external review failure");
864    }
865
866    outcome
867}
868
869/// Resets that end with no decision and do not count as a failure, before the task blocks
870pub(crate) const MAX_REVIEW_RESETS: u64 = 6;
871
872/// A reset that does not count toward `MAX_REVIEW_AGENT_FAILURES` still raises `review_resets`,
873/// so a review that never produces a decision stops bouncing between needs_review and in_review
874async fn reset_or_block(
875    store: &Arc<TaskStore>,
876    repo: &str,
877    task_id: &str,
878    reason: &str,
879    cause: &str,
880    external: bool,
881    context: &str,
882) -> ReviewOutcome {
883    let resets = match crate::store::store_increment(
884        &Some(std::sync::Arc::clone(store)),
885        repo,
886        task_id,
887        "review_resets",
888    )
889    .await
890    {
891        Ok(v) => v,
892        Err(e) => {
893            tracing::warn!(task_id, err = %e, "failed to increment review_resets");
894            0
895        }
896    };
897    let outcome = if resets >= MAX_REVIEW_RESETS {
898        tracing::error!(
899            task_id,
900            reason,
901            resets,
902            "{context} ended {resets} times without a decision, blocking task"
903        );
904        ReviewOutcome::Block(format!(
905            "{context} ended {resets} times without a decision: {reason}"
906        ))
907    } else {
908        ReviewOutcome::Reset
909    };
910    recorded(store, repo, task_id, reason, cause, external, outcome).await
911}
912
913/// Classify a review failure and return the appropriate outcome.
914///
915/// Both the `Ok(ReviewDecision::Failed)` and `Err(e)` arms share the same
916/// logic; callers convert the error to a string before calling this helper.
917/// `context` is used in log messages to distinguish the two call sites
918/// ("review agent" vs "review_and_merge").
919async fn classify_review_failure(
920    store: &Arc<TaskStore>,
921    repo: &str,
922    task_id: &str,
923    reason: &str,
924    context: &str,
925) -> ReviewOutcome {
926    // Rate-limit and auth failures are not real review failures — don't count them
927    // toward MAX_REVIEW_AGENT_FAILURES. The cooldown system steers the next
928    // attempt to an available agent.
929    let lower_reason = reason.to_lowercase();
930
931    // Parse errors are orch's fault, not review feedback. Re-route for retry
932    // without counting toward the failure threshold, so parse errors don't
933    // incorrectly block tasks that have no actual review feedback.
934    if lower_reason.contains("parse error") {
935        tracing::warn!(
936            task_id,
937            reason,
938            "{context} hit parse error — re-routing for retry without counting as failure"
939        );
940        return reset_or_block(store, repo, task_id, reason, "parse_error", false, context).await;
941    }
942
943    // Session exited mid-thought with no result envelope (issue #3734). Same
944    // treatment as a parse error: retry without counting toward the failure cap
945    if lower_reason.contains("incomplete:") {
946        tracing::warn!(
947            task_id,
948            reason,
949            "{context} session incomplete — re-routing for retry without counting as failure"
950        );
951        return reset_or_block(
952            store,
953            repo,
954            task_id,
955            reason,
956            "incomplete_session",
957            false,
958            context,
959        )
960        .await;
961    }
962
963    // The review output was cut off by the model's output/reasoning token
964    // budget, not a genuine parse/format failure. Same treatment as a parse
965    // error: re-route without counting toward MAX_REVIEW_AGENT_FAILURES.
966    if lower_reason.contains("truncated") {
967        tracing::warn!(
968            task_id,
969            reason,
970            "{context} output truncated by token budget — re-routing for retry without counting as failure"
971        );
972        return reset_or_block(
973            store,
974            repo,
975            task_id,
976            reason,
977            "truncated_output",
978            false,
979            context,
980        )
981        .await;
982    }
983
984    // Transient mergeability check: GitHub hasn't finished computing yet.
985    // This is a normal, expected transient state for freshly-pushed PRs.
986    // Return RetryMerge (not Reset) so the task stays in InReview — avoids
987    // spawning a redundant review agent that would just re-approve anyway.
988    if lower_reason.contains("not yet computed") {
989        tracing::warn!(
990            task_id,
991            reason,
992            "{context} PR mergeability check still pending — will retry merge without re-reviewing"
993        );
994        return recorded(
995            store,
996            repo,
997            task_id,
998            reason,
999            "mergeability_pending",
1000            true,
1001            ReviewOutcome::RetryMerge,
1002        )
1003        .await;
1004    }
1005
1006    // Merge conflicts should be handled by handle_merge_conflict() in
1007    // auto_merge.rs (which routes to the task agent). If a conflict error
1008    // leaks here, route to the task agent instead of resetting to NeedsReview
1009    // (which would re-spawn the review agent in a loop).
1010    if lower_reason.contains("merge conflict") || lower_reason.contains("not mergeable") {
1011        tracing::warn!(
1012            task_id,
1013            reason,
1014            "{context} hit merge conflict — routing to task agent to resolve"
1015        );
1016        return recorded(
1017            store,
1018            repo,
1019            task_id,
1020            reason,
1021            "merge_conflict",
1022            false,
1023            ReviewOutcome::Reroute,
1024        )
1025        .await;
1026    }
1027
1028    // Any other merge failure on approved work (rate limit, 5xx, stale lease, a check
1029    // still pending) is orch's problem, not the agent's. Keep the task in InReview so
1030    // review_open_prs retries the merge once the cause clears, and never re-dispatch the
1031    // agent — that burned attempts until the task blocked on work that was already done
1032    if lower_reason.contains("merge failed") {
1033        tracing::warn!(
1034            task_id,
1035            reason,
1036            "{context} merge failed without a conflict — retrying the merge, not re-dispatching the agent"
1037        );
1038        return recorded(
1039            store,
1040            repo,
1041            task_id,
1042            reason,
1043            "merge_retry",
1044            true,
1045            ReviewOutcome::RetryMerge,
1046        )
1047        .await;
1048    }
1049
1050    // No configured review model for any available agent, or the selected agent
1051    // has no review model. This is a configuration/cooldown gap, not a review
1052    // failure. Defer without incrementing the failure counter so the task retries
1053    // once a review model is available.
1054    if lower_reason.contains("no review agent") || lower_reason.contains("review model missing") {
1055        tracing::warn!(
1056            task_id,
1057            reason,
1058            "{context}: no review model available — deferring retry until one is available"
1059        );
1060        return recorded(
1061            store,
1062            repo,
1063            task_id,
1064            reason,
1065            "no_review_model",
1066            true,
1067            ReviewOutcome::RateLimited,
1068        )
1069        .await;
1070    }
1071
1072    // GhHttp's own throttle says "rate-limited", and GitHub answers 429 or abuse detection
1073    // without the words "rate limit"
1074    if lower_reason.contains("rate limit")
1075        || lower_reason.contains("rate-limited")
1076        || lower_reason.contains("rate limited")
1077        || lower_reason.contains("too many requests")
1078        || lower_reason.contains("abuse detection")
1079        || lower_reason.contains("failed (429)")
1080        || lower_reason.contains("auth error")
1081    {
1082        tracing::warn!(
1083            task_id,
1084            reason,
1085            "{context} hit rate limit or auth error — deferring retry until cooldown expires"
1086        );
1087        return recorded(
1088            store,
1089            repo,
1090            task_id,
1091            reason,
1092            "rate_limit",
1093            true,
1094            ReviewOutcome::RateLimited,
1095        )
1096        .await;
1097    }
1098
1099    if crate::engine::runner::git_ops::is_transient_github_api_error(reason) {
1100        tracing::warn!(
1101            task_id,
1102            reason,
1103            "{context} hit transient GitHub error — deferring retry without counting as failure"
1104        );
1105        return reset_or_block(
1106            store,
1107            repo,
1108            task_id,
1109            reason,
1110            "github_api_error",
1111            true,
1112            context,
1113        )
1114        .await;
1115    }
1116
1117    // Provider-side network errors (ConnectionRefused, timeouts, etc.) are
1118    // transient infrastructure issues, not review feedback. Re-route without
1119    // counting toward MAX_REVIEW_AGENT_FAILURES.
1120    if crate::engine::runner::agents::patterns::detect_network_error(reason).is_some() {
1121        tracing::warn!(
1122            task_id,
1123            reason,
1124            "{context} hit transient provider network error — re-routing for retry without counting as failure"
1125        );
1126        return reset_or_block(
1127            store,
1128            repo,
1129            task_id,
1130            reason,
1131            "provider_network_error",
1132            true,
1133            context,
1134        )
1135        .await;
1136    }
1137
1138    let failures = match crate::store::store_increment(
1139        &Some(std::sync::Arc::clone(store)),
1140        repo,
1141        task_id,
1142        "review_agent_failures",
1143    )
1144    .await
1145    {
1146        Ok(v) => v,
1147        Err(e) => {
1148            tracing::warn!(task_id, err = %e, "failed to increment review_agent_failures — skipping failure escalation this tick");
1149            // Treat as 0 for now (no escalation)
1150            0
1151        }
1152    };
1153    let blocking = failures >= MAX_REVIEW_AGENT_FAILURES;
1154    // Post failure comment to the PR so the history is visible.
1155    post_review_failure_comment(store, repo, task_id, reason, failures, blocking).await;
1156    if blocking {
1157        tracing::error!(
1158            task_id,
1159            reason,
1160            failures,
1161            "{context} failed too many times — blocking task"
1162        );
1163        recorded(
1164            store,
1165            repo,
1166            task_id,
1167            reason,
1168            "review_failure",
1169            false,
1170            ReviewOutcome::Block(format!("{context} failed {failures} times: {reason}")),
1171        )
1172        .await
1173    } else {
1174        tracing::error!(
1175            task_id,
1176            reason,
1177            failures,
1178            "{context} failed — resetting to NeedsReview for retry"
1179        );
1180        recorded(
1181            store,
1182            repo,
1183            task_id,
1184            reason,
1185            "review_failure",
1186            false,
1187            ReviewOutcome::Reset,
1188        )
1189        .await
1190    }
1191}
1192
1193/// Post a comment on the PR explaining why the review agent failed.
1194/// Best-effort — if the PR number is missing or the API call fails, we just log.
1195async fn post_review_failure_comment(
1196    store: &Arc<TaskStore>,
1197    repo: &str,
1198    task_id: &str,
1199    reason: &str,
1200    failures: u64,
1201    blocking: bool,
1202) {
1203    let pr_number =
1204        match opt_store_get_task(&Some(std::sync::Arc::clone(store)), repo, task_id).await {
1205            Some(t) => match t.pr_number {
1206                Some(n) => n,
1207                None => return, // No PR to comment on
1208            },
1209            _ => return, // No PR to comment on
1210        };
1211
1212    let status = if blocking {
1213        "Task blocked for human review."
1214    } else {
1215        "Retrying with a different agent."
1216    };
1217
1218    let comment = format!(
1219        "⚠️ Review agent failed (attempt {}/{})\n\n**Reason:** {}\n\n{}{}",
1220        failures,
1221        MAX_REVIEW_AGENT_FAILURES,
1222        reason,
1223        status,
1224        crate::engine::orch_footer()
1225    );
1226
1227    let Ok(gh) = GhHttp::new() else { return };
1228    if let Err(e) = gh.add_comment(repo, &pr_number.to_string(), &comment).await {
1229        tracing::debug!(
1230            task_id,
1231            pr_number,
1232            error = %e,
1233            "failed to post review failure comment"
1234        );
1235    }
1236}
1237
1238#[cfg(test)]
1239mod tests {
1240    use super::*;
1241
1242    /// An incomplete mid-thought review (no result envelope) must reset for
1243    /// retry without counting toward MAX_REVIEW_AGENT_FAILURES (issue #3734).
1244    #[tokio::test]
1245    async fn classify_review_failure_incomplete_resets_without_counting() {
1246        let store = Arc::new(TaskStore::open_memory().await.unwrap());
1247        let repo = "owner/repo";
1248        let store_id = store
1249            .create_internal(
1250                repo,
1251                "Incomplete review test",
1252                "",
1253                "review",
1254                "review-3734",
1255                None,
1256            )
1257            .await
1258            .unwrap();
1259        let task_id = format!("internal:{store_id}");
1260
1261        for _ in 0..=MAX_REVIEW_AGENT_FAILURES {
1262            let outcome = classify_review_failure(
1263                &store,
1264                repo,
1265                &task_id,
1266                "incomplete: review session exited before producing a decision",
1267                "review agent",
1268            )
1269            .await;
1270            assert!(
1271                matches!(outcome, ReviewOutcome::Reset),
1272                "incomplete review failure must reset for retry, not block"
1273            );
1274        }
1275
1276        let task = opt_store_get_task(&Some(std::sync::Arc::clone(&store)), repo, &task_id)
1277            .await
1278            .expect("task should exist");
1279        assert_eq!(
1280            task.review_agent_failures, 0,
1281            "incomplete failures must not count toward review_agent_failures"
1282        );
1283    }
1284
1285    /// A truncated review output (opencode step_finish reason=length) must be
1286    /// treated like a parse error: reset for retry without counting toward
1287    /// MAX_REVIEW_AGENT_FAILURES, so a run of token-budget flukes never
1288    /// permanently blocks the task (issue #3473).
1289    #[tokio::test]
1290    async fn classify_review_failure_truncated_resets_without_counting() {
1291        let store = Arc::new(TaskStore::open_memory().await.unwrap());
1292        let repo = "owner/repo";
1293        let store_id = store
1294            .create_internal(
1295                repo,
1296                "Truncated review test",
1297                "",
1298                "review",
1299                "review-1",
1300                None,
1301            )
1302            .await
1303            .unwrap();
1304        let task_id = format!("internal:{store_id}");
1305
1306        for _ in 0..=MAX_REVIEW_AGENT_FAILURES {
1307            let outcome = classify_review_failure(
1308                &store,
1309                repo,
1310                &task_id,
1311                "truncated: review output was cut off before completion \
1312                 (output/reasoning token budget exceeded)",
1313                "review agent",
1314            )
1315            .await;
1316            assert!(
1317                matches!(outcome, ReviewOutcome::Reset),
1318                "truncated review failure must reset for retry, not block"
1319            );
1320        }
1321
1322        let task = opt_store_get_task(&Some(std::sync::Arc::clone(&store)), repo, &task_id)
1323            .await
1324            .expect("task should exist");
1325        assert_eq!(
1326            task.review_agent_failures, 0,
1327            "truncated failures must not count toward review_agent_failures"
1328        );
1329    }
1330
1331    /**
1332     * After an approved review, a merge that fails for anything other than a content
1333     * conflict must retry the merge with the task still in InReview. Re-dispatching the
1334     * agent re-ran finished work until the task hit the attempt limit (#3783)
1335     **/
1336    #[tokio::test]
1337    async fn classify_review_failure_non_conflict_merge_failure_retries_merge() {
1338        let store = Arc::new(TaskStore::open_memory().await.unwrap());
1339        let repo = "owner/repo";
1340        let store_id = store
1341            .create_internal(repo, "Merge retry", "", "review", "review-3783", None)
1342            .await
1343            .unwrap();
1344        let task_id = format!("internal:{store_id}");
1345
1346        let error_messages = [
1347            "merge failed: GitHub REST API rate-limited, skipping call for 187s",
1348            "merge failed: GitHub PR merge failed (502 Bad Gateway)",
1349            "merge failed: required status check still pending",
1350        ];
1351
1352        for _ in 0..=MAX_REVIEW_AGENT_FAILURES {
1353            for msg in &error_messages {
1354                let outcome =
1355                    classify_review_failure(&store, repo, &task_id, msg, "review agent").await;
1356                assert!(
1357                    matches!(outcome, ReviewOutcome::RetryMerge),
1358                    "non-conflict merge failure '{msg}' must retry the merge, not re-dispatch the agent"
1359                );
1360            }
1361        }
1362
1363        let task = opt_store_get_task(&Some(std::sync::Arc::clone(&store)), repo, &task_id)
1364            .await
1365            .expect("task should exist");
1366        assert_eq!(
1367            task.review_agent_failures, 0,
1368            "a stuck merge is not a review failure"
1369        );
1370    }
1371
1372    /// A genuine content conflict is the one merge failure the agent has to fix
1373    #[tokio::test]
1374    async fn classify_review_failure_merge_conflict_still_reroutes() {
1375        let store = Arc::new(TaskStore::open_memory().await.unwrap());
1376        let repo = "owner/repo";
1377        let store_id = store
1378            .create_internal(repo, "Merge conflict", "", "review", "review-3783b", None)
1379            .await
1380            .unwrap();
1381        let task_id = format!("internal:{store_id}");
1382
1383        for msg in [
1384            "merge failed: PR is not mergeable",
1385            "merge conflict detected on PR #12",
1386        ] {
1387            let outcome =
1388                classify_review_failure(&store, repo, &task_id, msg, "review agent").await;
1389            assert!(
1390                matches!(outcome, ReviewOutcome::Reroute),
1391                "content conflict '{msg}' must go back to the task agent"
1392            );
1393        }
1394    }
1395
1396    /// Transient provider network errors (ConnectionRefused, timeouts) must be
1397    /// treated like parse errors: reset for retry without counting toward
1398    /// MAX_REVIEW_AGENT_FAILURES (issue #3482).
1399    #[tokio::test]
1400    async fn classify_review_failure_network_error_resets_without_counting() {
1401        let store = Arc::new(TaskStore::open_memory().await.unwrap());
1402        let repo = "owner/repo";
1403        let store_id = store
1404            .create_internal(
1405                repo,
1406                "Network error review test",
1407                "",
1408                "review",
1409                "review-2",
1410                None,
1411            )
1412            .await
1413            .unwrap();
1414        let task_id = format!("internal:{store_id}");
1415
1416        let error_messages = [
1417            "API Error: Unable to connect to API (ConnectionRefused)",
1418            "connection refused",
1419            "ECONNREFUSED",
1420            "Upstream idle timeout exceeded",
1421            "connect ETIMEDOUT",
1422            "network request failed",
1423        ];
1424
1425        for msg in &error_messages {
1426            // Each message on its own, the reset cap has its own test
1427            store
1428                .set_fields(store_id, &[("review_resets", serde_json::json!(0))])
1429                .await
1430                .unwrap();
1431            let outcome =
1432                classify_review_failure(&store, repo, &task_id, msg, "review agent").await;
1433            assert!(
1434                matches!(outcome, ReviewOutcome::Reset),
1435                "network error '{msg}' must reset for retry, not block"
1436            );
1437        }
1438
1439        let task = opt_store_get_task(&Some(std::sync::Arc::clone(&store)), repo, &task_id)
1440            .await
1441            .expect("task should exist");
1442        assert_eq!(
1443            task.review_agent_failures, 0,
1444            "network errors must not count toward review_agent_failures"
1445        );
1446    }
1447
1448    /// When no agent has a configured review model (or the fallback agent has none),
1449    /// the review must be deferred, not counted as a review failure. Counting it
1450    /// would eventually block tasks for a configuration/cooldown gap (issue #3584).
1451    #[tokio::test]
1452    async fn classify_review_failure_missing_review_model_is_rate_limited() {
1453        let store = Arc::new(TaskStore::open_memory().await.unwrap());
1454        let repo = "owner/repo";
1455        let store_id = store
1456            .create_internal(
1457                repo,
1458                "Missing review model test",
1459                "",
1460                "review",
1461                "review-3",
1462                None,
1463            )
1464            .await
1465            .unwrap();
1466        let task_id = format!("internal:{store_id}");
1467
1468        let error_messages = [
1469            "no review agent with a configured review model is available for task internal:123",
1470            "review model missing for task internal:123 after agent selection — refusing to dispatch",
1471        ];
1472
1473        for _ in 0..=MAX_REVIEW_AGENT_FAILURES {
1474            for msg in &error_messages {
1475                let outcome =
1476                    classify_review_failure(&store, repo, &task_id, msg, "review_and_merge").await;
1477                assert!(
1478                    matches!(outcome, ReviewOutcome::RateLimited),
1479                    "missing review model error '{msg}' must defer, not block"
1480                );
1481            }
1482        }
1483
1484        let task = opt_store_get_task(&Some(std::sync::Arc::clone(&store)), repo, &task_id)
1485            .await
1486            .expect("task should exist");
1487        assert_eq!(
1488            task.review_agent_failures, 0,
1489            "missing review model errors must not count toward review_agent_failures"
1490        );
1491    }
1492
1493    /// Resets that do not count as failures still stop at `MAX_REVIEW_RESETS`
1494    #[tokio::test]
1495    async fn classify_review_failure_blocks_after_max_resets() {
1496        let store = Arc::new(TaskStore::open_memory().await.unwrap());
1497        let repo = "owner/repo";
1498        let store_id = store
1499            .create_internal(repo, "Looping review", "", "review", "review-resets", None)
1500            .await
1501            .unwrap();
1502        let task_id = format!("internal:{store_id}");
1503        let msg = "parse error: failed to parse review response";
1504
1505        for attempt in 1..MAX_REVIEW_RESETS {
1506            let outcome =
1507                classify_review_failure(&store, repo, &task_id, msg, "review agent").await;
1508            assert!(
1509                matches!(outcome, ReviewOutcome::Reset),
1510                "reset {attempt} must retry the review"
1511            );
1512        }
1513        let outcome = classify_review_failure(&store, repo, &task_id, msg, "review agent").await;
1514        assert!(
1515            matches!(outcome, ReviewOutcome::Block(ref r) if r.contains("without a decision")),
1516            "reset {MAX_REVIEW_RESETS} must block the task"
1517        );
1518
1519        let task = opt_store_get_task(&Some(std::sync::Arc::clone(&store)), repo, &task_id)
1520            .await
1521            .expect("task should exist");
1522        assert_eq!(
1523            u64::try_from(task.review_resets).unwrap(),
1524            MAX_REVIEW_RESETS
1525        );
1526        assert_eq!(
1527            task.review_agent_failures, 0,
1528            "a reset must not count as a review failure"
1529        );
1530    }
1531
1532    /**
1533     * A review that fails on a rate limit is retried once the cooldown clears, and must
1534     * never spend the rebroadcast quota: `needs_review_refires` is cleared so the sync
1535     * sweep cannot escalate the task to Blocked for a cause outside the review (#3767)
1536     **/
1537    #[tokio::test]
1538    async fn classify_review_failure_rate_limit_clears_refire_quota() {
1539        let store = Arc::new(TaskStore::open_memory().await.unwrap());
1540        let repo = "owner/repo";
1541        let store_id = store
1542            .create_internal(
1543                repo,
1544                "Rate limited review",
1545                "",
1546                "review",
1547                "review-3767",
1548                None,
1549            )
1550            .await
1551            .unwrap();
1552        let task_id = format!("internal:{store_id}");
1553
1554        let error_messages = [
1555            "claude failed: rate limit reached for model sonnet",
1556            "GitHub API GET https://api.github.com/repos/owner/repo/pulls/65 failed (403): API rate limit exceeded",
1557            "GitHub core API rate-limited, skipping call for 1800s",
1558            "GitHub API GET https://api.github.com/repos/owner/repo/pulls/65 failed (429): Too Many Requests",
1559        ];
1560
1561        // The sweep has already burned most of the quota before the rate-limit window
1562        store
1563            .set_fields(store_id, &[("needs_review_refires", serde_json::json!(5))])
1564            .await
1565            .unwrap();
1566
1567        for _ in 0..=MAX_REVIEW_AGENT_FAILURES {
1568            for msg in &error_messages {
1569                let outcome =
1570                    classify_review_failure(&store, repo, &task_id, msg, "review agent").await;
1571                assert!(
1572                    matches!(outcome, ReviewOutcome::RateLimited | ReviewOutcome::Reset),
1573                    "rate limit '{msg}' must defer for retry, not block"
1574                );
1575            }
1576        }
1577
1578        let task = opt_store_get_task(&Some(std::sync::Arc::clone(&store)), repo, &task_id)
1579            .await
1580            .expect("task should exist");
1581        assert_eq!(
1582            task.needs_review_refires, 0,
1583            "a rate limit must not consume the rebroadcast quota"
1584        );
1585        assert_eq!(
1586            task.review_agent_failures, 0,
1587            "rate limits must not count toward review_agent_failures"
1588        );
1589        assert_ne!(
1590            task.status,
1591            crate::store::TaskStatus::Blocked,
1592            "a rate-limited review must leave the task retryable"
1593        );
1594
1595        let attempts = store
1596            .get_activity(store_id, None)
1597            .await
1598            .unwrap()
1599            .into_iter()
1600            .filter(|a| a.event_type == "review_attempt_failed")
1601            .collect::<Vec<_>>();
1602        assert_eq!(
1603            attempts.len(),
1604            (usize::try_from(MAX_REVIEW_AGENT_FAILURES).unwrap() + 1) * error_messages.len(),
1605            "every failed attempt must be recorded for `orch task inspect`"
1606        );
1607        assert!(
1608            attempts.iter().all(|a| a
1609                .details
1610                .get("external")
1611                .and_then(serde_json::Value::as_bool)
1612                == Some(true)),
1613            "rate limits must be recorded as an external cause"
1614        );
1615        assert!(
1616            attempts.iter().any(
1617                |a| a.details.get("reason").and_then(|v| v.as_str()) == Some(error_messages[0])
1618            ),
1619            "the recorded attempt must carry that attempt's error"
1620        );
1621    }
1622}