1use crate::backends::{ExternalBackend, ExternalId, ExternalTask, Status};
4use crate::engine::cooldown::AGENT_COOLDOWN_SECS;
5use crate::engine::dispatch_guard::DispatchGuard;
6use crate::engine::events::TaskEvent;
7use crate::engine::review::{MAX_REVIEW_AGENT_FAILURES, ReviewDecision, review_and_merge};
8use crate::engine::review_gate::CiGate;
9use crate::engine::review_shipcheck::Prerun;
10use crate::engine::router::Router;
11use crate::engine::tasks::TaskManager;
12use crate::github::http::GhHttp;
13use crate::repo_context::REPO_CONTEXT;
14use crate::store::{TaskStore, opt_store_get_task};
15use crate::tmux::TmuxManager;
16use dashmap::DashMap;
17use std::sync::Arc;
18use tokio::sync::broadcast;
19use tokio::sync::{RwLock, Semaphore};
20
21enum ReviewOutcome {
22 Reset,
23 RateLimited,
24 RetryMerge,
29 Approved,
32 Reroute,
34 Block(String),
35 Ok,
36}
37
38#[expect(
44 clippy::too_many_arguments,
45 reason = "the arguments have no natural grouping"
46)]
47pub fn spawn(
48 mut rx: broadcast::Receiver<TaskEvent>,
49 backend: Arc<dyn ExternalBackend>,
50 tmux: Arc<TmuxManager>,
51 semaphore: Arc<Semaphore>,
52 task_manager: Arc<TaskManager>,
53 router: Arc<RwLock<Router>>,
54 dispatching: Arc<DashMap<String, String>>,
55 store: Arc<TaskStore>,
56 repo: String,
57) {
58 tokio::spawn(async move {
59 loop {
60 match rx.recv().await {
61 Ok(event) if event.new_status == "needs_review" && event.repo == repo => {
62 let enable_review = crate::config::get("workflow.enable_review_agent")
64 .map_or(true, |v| v != "false");
65 if !enable_review {
66 continue;
67 }
68
69 let task_id = &event.task_id;
70 let dispatch_key = format!("{repo}/{task_id}");
71
72 {
77 use dashmap::mapref::entry::Entry;
78 match dispatching.entry(dispatch_key.clone()) {
79 Entry::Occupied(existing) => {
80 let existing_id = existing.get().clone();
81 drop(existing);
82 if existing_id == *task_id {
83 tracing::debug!(
84 task_id,
85 "task locked by dispatch flow, skipping event-driven review"
86 );
87 } else {
88 tracing::warn!(
89 task_id,
90 existing_task_id = existing_id,
91 dispatch_key,
92 "dispatch key collision: unexpected task already holds this key"
93 );
94 }
95 continue;
96 }
97 Entry::Vacant(slot) => {
98 slot.insert(task_id.clone());
99 }
100 }
101 }
102
103 let dispatch_guard = DispatchGuard::new(
105 std::sync::Arc::clone(&dispatching),
106 dispatch_key.clone(),
107 );
108
109 tracing::info!(task_id, "event-driven review triggered");
110
111 let task = {
113 if crate::engine::tasks::is_internal_id(task_id)
114 || match store.has_external_tasks(&repo).await {
115 Ok(has_external) => has_external,
116 Err(e) => {
117 tracing::warn!(
118 repo = %repo,
119 task_id,
120 error = %e,
121 "failed to check external task sentinel; keeping store-first review lookup"
122 );
123 true
124 }
125 }
126 {
127 let result = if let Some(numeric_id) =
130 crate::engine::tasks::parse_internal_id(task_id)
131 {
132 store.get(numeric_id).await.map(Some)
133 } else {
134 store.get_by_external_id(&repo, task_id).await
135 };
136 match result {
137 Ok(Some(t))
138 if t.status == crate::store::TaskStatus::NeedsReview =>
139 {
140 Some(crate::engine::tasks::store_task_to_external(&t))
141 }
142 Ok(Some(t)) => {
143 tracing::debug!(
144 task_id,
145 status = t.status.as_str(),
146 "task found but not in needs_review status, skipping"
147 );
148 None
149 }
150 Ok(None) => None,
151 Err(e) => {
152 tracing::warn!(task_id, err = %e, "store lookup failed for review");
153 None
154 }
155 }
156 } else {
157 match backend.list_by_status(Status::NeedsReview).await {
159 Ok(tasks) => tasks.into_iter().find(|t| t.id.0 == *task_id),
160 Err(e) => {
161 tracing::warn!(task_id, err = %e, "backend lookup failed for review");
162 None
163 }
164 }
165 }
166 };
167
168 let Some(task) = task else {
169 tracing::debug!(
170 task_id,
171 "task not found in needs_review list (may have already transitioned)"
172 );
173 continue;
174 };
175
176 if let Some(stored) =
177 opt_store_get_task(&Some(std::sync::Arc::clone(&store)), &repo, &task.id.0)
178 .await
179 {
180 let no_code_review = stored.worktree.trim().is_empty()
181 && stored.branch.trim().is_empty()
182 && stored.pr_number.is_none();
183 if no_code_review {
184 tracing::info!(
185 task_id,
186 "no worktree/branch/pr for needs_review task — skipping review dispatch and marking done"
187 );
188 crate::store::store_reset_failure_counters(
189 &Some(std::sync::Arc::clone(&store)),
190 &repo,
191 &task.id.0,
192 )
193 .await;
194 if let Err(e) = crate::store::store_set_result(
195 &Some(std::sync::Arc::clone(&store)),
196 &repo,
197 &task.id.0,
198 &[("needs_review_refires", serde_json::json!(0))],
199 )
200 .await
201 {
202 tracing::warn!(
203 task_id,
204 err = %e,
205 "failed to reset needs_review_refires while skipping no-code review"
206 );
207 }
208 if let Err(e) = task_manager
209 .update_task_status(&task.id, Status::Done)
210 .await
211 {
212 tracing::error!(
213 task_id,
214 err = %e,
215 "failed to mark no-code needs_review task as done"
216 );
217 }
218 continue;
219 }
220 }
221
222 {
229 let router_guard = router.read().await;
230 let no_routable = !router_guard.available_agents.is_empty()
231 && router_guard.healthy_agent_count("review") == 0;
232 drop(router_guard);
233 if no_routable {
234 tracing::info!(
235 task_id,
236 "no routable review agents — leaving task in needs_review for sync to re-fire"
237 );
238 continue;
239 }
240 }
241
242 if !ci_allows_review(&store, &task_manager, &repo, &task).await {
243 continue;
244 }
245
246 let Ok(permit) = std::sync::Arc::clone(&semaphore).try_acquire_owned() else {
248 tracing::debug!(
249 task_id,
250 "all parallel slots busy; sync catch-up will re-fire when a slot opens"
251 );
252 continue;
253 };
254
255 if let Err(e) = task_manager
257 .update_task_status(&task.id, Status::InReview)
258 .await
259 {
260 tracing::warn!(task_id, err = %e, "failed to transition to InReview");
261 drop(permit);
262 continue;
263 }
264
265 let backend_c = std::sync::Arc::clone(&backend);
266 let task_manager_c = std::sync::Arc::clone(&task_manager);
267 let tmux_c = std::sync::Arc::clone(&tmux);
268 let router_c = std::sync::Arc::clone(&router);
269 let store_c = std::sync::Arc::clone(&store);
270 let repo_s = repo.clone();
271 tokio::spawn(REPO_CONTEXT.scope(repo_s.clone(), async move {
272 let _dispatch_guard = dispatch_guard; let tid = task.id.0.clone();
274
275 let pre_check_all_cooled = {
280 let r = router_c.read().await;
281 !r.available_agents.is_empty() && r.healthy_agent_count("review") == 0
282 };
283 if pre_check_all_cooled {
284 tracing::info!(
285 task_id = tid,
286 "all agents cooled at review spawn time — resetting to needs_review for sync to re-fire"
287 );
288 if let Err(e) = task_manager_c
289 .update_task_status(
290 &ExternalId(tid.clone()),
291 Status::NeedsReview,
292 )
293 .await
294 {
295 tracing::error!(task_id = %tid, err = %e, "pre-check: update_task_status(NeedsReview) failed — task may be stuck in InReview");
296 }
297 drop(permit);
298 return;
299 }
300
301 let outcome = match Box::pin(review_and_merge(
302 &task,
303 &backend_c,
304 &tmux_c,
305 &repo_s,
306 &router_c,
307 &task_manager_c,
308 &store_c,
309 ))
310 .await
311 {
312 Ok(ReviewDecision::Blocked(reason)) => {
313 tracing::error!(
314 task_id = tid,
315 reason,
316 "review gate blocked after repeated failures — marking task blocked"
317 );
318 ReviewOutcome::Block(format!(
319 "review gate blocked after repeated failures: {reason}"
320 ))
321 }
322 Ok(ReviewDecision::Failed(reason)) => {
323 classify_review_failure(
324 &store_c, &repo_s, &tid, &reason, "review agent",
325 )
326 .await
327 }
328 Err(e) => {
329 let reason = format!("{e:#}");
330 classify_review_failure(
331 &store_c, &repo_s, &tid, &reason, "review_and_merge",
332 )
333 .await
334 }
335 Ok(ReviewDecision::Approve { .. }) => {
336 crate::store::store_reset_failure_counters(
341 &Some(std::sync::Arc::clone(&store_c)),
342 &repo_s,
343 &tid,
344 )
345 .await;
346 if let Err(e) = crate::store::store_set_result(
348 &Some(std::sync::Arc::clone(&store_c)),
349 &repo_s,
350 &tid,
351 &[("needs_review_refires", serde_json::json!(0))],
352 )
353 .await
354 {
355 tracing::warn!(task_id = %tid, err = %e, "failed to write needs_review_refires to store");
356 }
357
358 ReviewOutcome::Approved
362 }
363 Ok(ReviewDecision::Rerouted) => {
364 ReviewOutcome::Ok
368 }
369 Ok(ReviewDecision::Skipped) => {
370 crate::store::store_reset_failure_counters(
373 &Some(std::sync::Arc::clone(&store_c)),
374 &repo_s,
375 &tid,
376 )
377 .await;
378 if let Err(e) = crate::store::store_set_result(
379 &Some(std::sync::Arc::clone(&store_c)),
380 &repo_s,
381 &tid,
382 &[("needs_review_refires", serde_json::json!(0))],
383 )
384 .await
385 {
386 tracing::warn!(task_id = %tid, err = %e, "failed to write needs_review_refires to store");
387 }
388 ReviewOutcome::Ok
389 }
390 Ok(ReviewDecision::RequestChanges { .. }) => {
391 crate::store::store_reset_failure_counters(
394 &Some(std::sync::Arc::clone(&store_c)),
395 &repo_s,
396 &tid,
397 )
398 .await;
399 if let Err(e) = crate::store::store_set_result(
400 &Some(std::sync::Arc::clone(&store_c)),
401 &repo_s,
402 &tid,
403 &[("needs_review_refires", serde_json::json!(0))],
404 )
405 .await
406 {
407 tracing::warn!(task_id = %tid, err = %e, "failed to write needs_review_refires to store");
408 }
409 ReviewOutcome::Ok
410 }
411 };
412 let fetched: Result<Option<(crate::store::TaskStatus, Option<String>)>, anyhow::Error> = async {
423 let id = match store_c.resolve_task_id(&repo_s, &tid).await {
424 Ok(Some(id)) => id,
425 Ok(None) => return Ok(None), Err(e) => return Err(e), };
428 let task = store_c.get(id).await?;
429 Ok(Some((task.status, task.block_reason)))
430 }
431 .await;
432
433 let (current_status, current_block_reason) = match fetched {
434 Ok(Some((status, block_reason))) => (Some(status), block_reason),
435 Ok(None) => {
436 (None, None)
438 }
439 Err(e) => {
440 tracing::error!(
442 task_id = tid,
443 error = %e,
444 "review outcome retained — DB lookup failed, not treating as stale"
445 );
446 (Some(crate::store::TaskStatus::InReview), None)
449 }
450 };
451
452 if !matches!(current_status, Some(crate::store::TaskStatus::InReview)) {
453 let ci_blocked = matches!(
454 current_status,
455 Some(crate::store::TaskStatus::Blocked)
456 ) && current_block_reason
457 .as_deref()
458 .is_some_and(|r| r.contains("CI failure limit"));
459 let expected = ci_blocked
460 || matches!(
461 current_status,
462 Some(crate::store::TaskStatus::Done | crate::store::TaskStatus::Approved |
463crate::store::TaskStatus::Routed | crate::store::TaskStatus::New)
464 );
465 if expected {
466 tracing::debug!(
467 task_id = tid,
468 current_status = current_status.as_ref().map_or("unknown", crate::store::tasks::TaskStatus::as_str),
469 "review outcome discarded — task transitioned away from in_review (expected)"
470 );
471 } else {
472 tracing::warn!(
473 task_id = tid,
474 current_status = current_status.as_ref().map_or("unknown", crate::store::tasks::TaskStatus::as_str),
475 "review outcome discarded — task is no longer in_review (stale review agent result)"
476 );
477 }
478
479 let stale_session = tmux_c.session_name(&repo_s, &format!("{tid}-review"));
481 if let Err(e) = tmux_c.kill_session(&stale_session).await {
482 tracing::debug!(task_id = tid, error = %e, "failed to kill stale review session");
483 }
484
485 drop(permit);
486 return;
487 }
488
489 match outcome {
490 ReviewOutcome::Reset => {
491 let stale_session =
498 tmux_c.session_name(&repo_s, &format!("{tid}-review"));
499 tmux_c.kill_session(&stale_session).await.ok();
500 tokio::time::sleep(std::time::Duration::from_secs(10)).await;
503 if let Err(e) = task_manager_c
504 .update_task_status(
505 &ExternalId(tid.clone()),
506 Status::NeedsReview,
507 )
508 .await
509 {
510 tracing::error!(task_id = %tid, err = %e, "update_task_status(NeedsReview) failed — task may be stuck in InReview");
511 }
512 }
513 ReviewOutcome::Reroute => {
514 let stale_session =
517 tmux_c.session_name(&repo_s, &format!("{tid}-review"));
518 tmux_c.kill_session(&stale_session).await.ok();
519 tracing::info!(
520 task_id = tid,
521 "re-routing to task agent to resolve merge conflict"
522 );
523 if let Err(e) = task_manager_c
524 .update_task_status(
525 &ExternalId(tid.clone()),
526 Status::Routed,
527 )
528 .await
529 {
530 tracing::error!(task_id = %tid, err = %e, "update_task_status(Routed) failed");
531 }
532 }
533 ReviewOutcome::Approved | ReviewOutcome::RetryMerge => {
534 let stale_session =
537 tmux_c.session_name(&repo_s, &format!("{tid}-review"));
538 tmux_c.kill_session(&stale_session).await.ok();
539 match task_manager_c
540 .update_task_status_if(
541 &ExternalId(tid.clone()),
542 Status::Approved,
543 Status::InReview,
544 )
545 .await
546 {
547 Ok(true) => tracing::info!(
548 task_id = tid,
549 "approval received, task approved until the PR is merged"
550 ),
551 Ok(false) => tracing::debug!(
552 task_id = tid,
553 "approval received after the task left in_review"
554 ),
555 Err(e) => tracing::error!(
556 task_id = %tid,
557 err = %e,
558 "update_task_status_if(Approved) failed, task may be stuck in in_review"
559 ),
560 }
561 }
562 ReviewOutcome::RateLimited => {
563 let stale_session =
565 tmux_c.session_name(&repo_s, &format!("{tid}-review"));
566 tmux_c.kill_session(&stale_session).await.ok();
567
568 let wait_secs = {
572 let r = router_c.read().await;
573 if r.healthy_agent_count("review") > 0 {
574 30u64
575 } else {
576 let now = chrono::Utc::now().timestamp();
577 r.earliest_cooldown_until(Some("review"))
578 .map_or(AGENT_COOLDOWN_SECS as u64, |until| u64::try_from((until - now).max(1)).unwrap_or(1))
579 }
580 };
581 tracing::info!(
582 task_id = tid,
583 wait_secs,
584 "deferring review retry due to agent rate limit cooldown"
585 );
586 tokio::time::sleep(std::time::Duration::from_secs(wait_secs)).await;
587 if let Err(e) = task_manager_c
588 .update_task_status(
589 &ExternalId(tid.clone()),
590 Status::NeedsReview,
591 )
592 .await
593 {
594 tracing::error!(task_id = %tid, err = %e, "update_task_status(NeedsReview) failed after rate limit backoff — task may be stuck in InReview");
595 }
596 }
597 ReviewOutcome::Block(reason) => {
598 let fields = [
602 (
603 "block_reason",
604 serde_json::json!("review agent blocked — exceeded failure threshold"),
605 ),
606 ("last_error", serde_json::json!(reason)),
607 ];
608 if let Err(e) = task_manager_c
609 .update_task_status_and_result(
610 &ExternalId(tid.clone()),
611 Status::Blocked,
612 &fields,
613 )
614 .await
615 {
616 tracing::error!(task_id = %tid, err = %e, "update_task_status_and_result(Blocked) failed — task may be stuck in InReview");
617 }
618 }
619 ReviewOutcome::Ok => {}
620 }
621
622 drop(permit);
623 }));
625
626 tokio::time::sleep(std::time::Duration::from_secs(2)).await;
631 }
632 Ok(_) => {} Err(broadcast::error::RecvError::Lagged(n)) => {
634 tracing::debug!(
635 n,
636 "review subscriber lagged; sync catch-up will re-fire missed NeedsReview tasks"
637 );
638 }
639 Err(_) => break, }
641 }
642 });
643}
644
645async fn ci_allows_review(
660 store: &Arc<TaskStore>,
661 task_manager: &Arc<TaskManager>,
662 repo: &str,
663 task: &ExternalTask,
664) -> bool {
665 let task_id = task.id.0.as_str();
666 let stored = opt_store_get_task(&Some(std::sync::Arc::clone(store)), repo, task_id).await;
667 let Some(pr_number) = stored
668 .as_ref()
669 .and_then(|t| t.pr_number)
670 .and_then(|n| u64::try_from(n).ok())
671 else {
672 return true;
673 };
674 let worktree = stored
675 .map(|t| std::path::PathBuf::from(t.worktree))
676 .filter(|p| !p.as_os_str().is_empty() && p.exists());
677 let gate = match GhHttp::new() {
678 Ok(gh) => {
679 crate::engine::review_gate::review_ci_gate(&gh, store, repo, task_id, pr_number).await
680 }
681 Err(e) => Err(e),
682 };
683
684 let hold = match gate {
685 Ok(CiGate::Start) => return true,
686 Err(e) => {
688 tracing::warn!(task_id, pr_number, err = %e, "CI gate lookup failed, starting the review");
689 return true;
690 }
691 Ok(CiGate::Red(failed)) => {
692 route_back_on_red_ci(store, task_manager, repo, task, pr_number, &failed).await;
693 return false;
694 }
695 Ok(CiGate::Wait) => "CI still running on the PR head, review waits",
696 Ok(CiGate::NotStarted { waited_out }) => match local_ci_worktree(repo, worktree).await {
698 Some(wt) => match crate::engine::review_shipcheck::prerun(&wt).await {
699 Prerun::Ready => return true,
700 Prerun::Running => "shipcheck running on the PR head, review waits",
701 },
702 None if waited_out => {
703 tracing::warn!(
704 task_id,
705 pr_number,
706 "no CI ran on the PR head, starting the review anyway"
707 );
708 return true;
709 }
710 None => "no CI started on the PR head yet, review waits",
711 },
712 };
713
714 tracing::info!(task_id, pr_number, "{hold}");
715 if let Err(e) = crate::store::store_set_result(
716 &Some(std::sync::Arc::clone(store)),
717 repo,
718 task_id,
719 &[("needs_review_refires", serde_json::json!(0))],
720 )
721 .await
722 {
723 tracing::warn!(task_id, err = %e, "failed to reset needs_review_refires for a task held by the CI gate");
724 }
725
726 false
727}
728
729async fn local_ci_worktree(
731 repo: &str,
732 worktree: Option<std::path::PathBuf>,
733) -> Option<std::path::PathBuf> {
734 let wt = worktree?;
735 let sandboxed = crate::engine::review::review_sandboxed(repo);
736 let shipcheck = crate::engine::runner::agent::Shipcheck::detect(&wt)
737 .await
738 .for_prompt(sandboxed);
739 (shipcheck == crate::engine::runner::agent::Shipcheck::Available).then_some(wt)
740}
741
742async fn route_back_on_red_ci(
744 store: &Arc<TaskStore>,
745 task_manager: &Arc<TaskManager>,
746 repo: &str,
747 task: &ExternalTask,
748 pr_number: u64,
749 failed: &[String],
750) {
751 let task_id = task.id.0.as_str();
752 let failures = match crate::store::store_increment(
753 &Some(std::sync::Arc::clone(store)),
754 repo,
755 task_id,
756 "ci_merge_failures",
757 )
758 .await
759 {
760 Ok(v) => v,
761 Err(e) => {
762 tracing::warn!(task_id, err = %e, "failed to increment ci_merge_failures, review waits");
763 return;
764 }
765 };
766 let checks = failed.join(", ");
767 let max = crate::engine::auto_merge::MAX_CI_MERGE_FAILURES;
768 let (status, fields) = if failures >= max {
769 tracing::error!(
770 task_id,
771 pr_number,
772 failures,
773 checks,
774 "CI failed before review, CI failure limit reached"
775 );
776 (
777 Status::Blocked,
778 vec![(
779 "block_reason",
780 serde_json::json!(format!("CI failure limit ({max}) reached before review")),
781 )],
782 )
783 } else {
784 tracing::warn!(
785 task_id,
786 pr_number,
787 failures,
788 checks,
789 "CI failed before review, routing back to the task agent"
790 );
791 (
792 Status::Routed,
793 vec![
794 (
795 "last_error",
796 serde_json::json!(format!(
797 "CI failed on PR #{pr_number} before review: {checks}. Fix the failing checks"
798 )),
799 ),
800 (
801 "route_reason",
802 serde_json::json!("re-dispatch after CI failed before review"),
803 ),
804 ],
805 )
806 };
807 if let Err(e) = task_manager
808 .update_task_status_and_result(&task.id, status, &fields)
809 .await
810 {
811 tracing::error!(task_id, err = %e, "failed to move a task whose CI failed before review");
812 }
813}
814
815async fn recorded(
816 store: &Arc<TaskStore>,
817 repo: &str,
818 task_id: &str,
819 reason: &str,
820 cause: &str,
821 external: bool,
822 outcome: ReviewOutcome,
823) -> ReviewOutcome {
824 let store_id = match store.resolve_task_id(repo, task_id).await {
825 Ok(Some(id)) => id,
826 Ok(None) => {
827 tracing::warn!(
828 task_id,
829 "review attempt not recorded — task not found in store"
830 );
831 return outcome;
832 }
833 Err(e) => {
834 tracing::warn!(task_id, err = %e, "review attempt not recorded — task id lookup failed");
835 return outcome;
836 }
837 };
838
839 if let Err(e) = store
840 .append_activity(
841 store_id,
842 "review_attempt_failed",
843 None,
844 None,
845 None,
846 None,
847 Some(&serde_json::json!({
848 "cause": cause,
849 "external": external,
850 "reason": reason,
851 })),
852 )
853 .await
854 {
855 tracing::warn!(task_id, err = %e, "failed to record review attempt failure");
856 }
857
858 if external
859 && let Err(e) = store
860 .set_fields(store_id, &[("needs_review_refires", serde_json::json!(0))])
861 .await
862 {
863 tracing::warn!(task_id, err = %e, "failed to clear needs_review_refires after external review failure");
864 }
865
866 outcome
867}
868
869pub(crate) const MAX_REVIEW_RESETS: u64 = 6;
871
872async fn reset_or_block(
875 store: &Arc<TaskStore>,
876 repo: &str,
877 task_id: &str,
878 reason: &str,
879 cause: &str,
880 external: bool,
881 context: &str,
882) -> ReviewOutcome {
883 let resets = match crate::store::store_increment(
884 &Some(std::sync::Arc::clone(store)),
885 repo,
886 task_id,
887 "review_resets",
888 )
889 .await
890 {
891 Ok(v) => v,
892 Err(e) => {
893 tracing::warn!(task_id, err = %e, "failed to increment review_resets");
894 0
895 }
896 };
897 let outcome = if resets >= MAX_REVIEW_RESETS {
898 tracing::error!(
899 task_id,
900 reason,
901 resets,
902 "{context} ended {resets} times without a decision, blocking task"
903 );
904 ReviewOutcome::Block(format!(
905 "{context} ended {resets} times without a decision: {reason}"
906 ))
907 } else {
908 ReviewOutcome::Reset
909 };
910 recorded(store, repo, task_id, reason, cause, external, outcome).await
911}
912
913async fn classify_review_failure(
920 store: &Arc<TaskStore>,
921 repo: &str,
922 task_id: &str,
923 reason: &str,
924 context: &str,
925) -> ReviewOutcome {
926 let lower_reason = reason.to_lowercase();
930
931 if lower_reason.contains("parse error") {
935 tracing::warn!(
936 task_id,
937 reason,
938 "{context} hit parse error — re-routing for retry without counting as failure"
939 );
940 return reset_or_block(store, repo, task_id, reason, "parse_error", false, context).await;
941 }
942
943 if lower_reason.contains("incomplete:") {
946 tracing::warn!(
947 task_id,
948 reason,
949 "{context} session incomplete — re-routing for retry without counting as failure"
950 );
951 return reset_or_block(
952 store,
953 repo,
954 task_id,
955 reason,
956 "incomplete_session",
957 false,
958 context,
959 )
960 .await;
961 }
962
963 if lower_reason.contains("truncated") {
967 tracing::warn!(
968 task_id,
969 reason,
970 "{context} output truncated by token budget — re-routing for retry without counting as failure"
971 );
972 return reset_or_block(
973 store,
974 repo,
975 task_id,
976 reason,
977 "truncated_output",
978 false,
979 context,
980 )
981 .await;
982 }
983
984 if lower_reason.contains("not yet computed") {
989 tracing::warn!(
990 task_id,
991 reason,
992 "{context} PR mergeability check still pending — will retry merge without re-reviewing"
993 );
994 return recorded(
995 store,
996 repo,
997 task_id,
998 reason,
999 "mergeability_pending",
1000 true,
1001 ReviewOutcome::RetryMerge,
1002 )
1003 .await;
1004 }
1005
1006 if lower_reason.contains("merge conflict") || lower_reason.contains("not mergeable") {
1011 tracing::warn!(
1012 task_id,
1013 reason,
1014 "{context} hit merge conflict — routing to task agent to resolve"
1015 );
1016 return recorded(
1017 store,
1018 repo,
1019 task_id,
1020 reason,
1021 "merge_conflict",
1022 false,
1023 ReviewOutcome::Reroute,
1024 )
1025 .await;
1026 }
1027
1028 if lower_reason.contains("merge failed") {
1033 tracing::warn!(
1034 task_id,
1035 reason,
1036 "{context} merge failed without a conflict — retrying the merge, not re-dispatching the agent"
1037 );
1038 return recorded(
1039 store,
1040 repo,
1041 task_id,
1042 reason,
1043 "merge_retry",
1044 true,
1045 ReviewOutcome::RetryMerge,
1046 )
1047 .await;
1048 }
1049
1050 if lower_reason.contains("no review agent") || lower_reason.contains("review model missing") {
1055 tracing::warn!(
1056 task_id,
1057 reason,
1058 "{context}: no review model available — deferring retry until one is available"
1059 );
1060 return recorded(
1061 store,
1062 repo,
1063 task_id,
1064 reason,
1065 "no_review_model",
1066 true,
1067 ReviewOutcome::RateLimited,
1068 )
1069 .await;
1070 }
1071
1072 if lower_reason.contains("rate limit")
1075 || lower_reason.contains("rate-limited")
1076 || lower_reason.contains("rate limited")
1077 || lower_reason.contains("too many requests")
1078 || lower_reason.contains("abuse detection")
1079 || lower_reason.contains("failed (429)")
1080 || lower_reason.contains("auth error")
1081 {
1082 tracing::warn!(
1083 task_id,
1084 reason,
1085 "{context} hit rate limit or auth error — deferring retry until cooldown expires"
1086 );
1087 return recorded(
1088 store,
1089 repo,
1090 task_id,
1091 reason,
1092 "rate_limit",
1093 true,
1094 ReviewOutcome::RateLimited,
1095 )
1096 .await;
1097 }
1098
1099 if crate::engine::runner::git_ops::is_transient_github_api_error(reason) {
1100 tracing::warn!(
1101 task_id,
1102 reason,
1103 "{context} hit transient GitHub error — deferring retry without counting as failure"
1104 );
1105 return reset_or_block(
1106 store,
1107 repo,
1108 task_id,
1109 reason,
1110 "github_api_error",
1111 true,
1112 context,
1113 )
1114 .await;
1115 }
1116
1117 if crate::engine::runner::agents::patterns::detect_network_error(reason).is_some() {
1121 tracing::warn!(
1122 task_id,
1123 reason,
1124 "{context} hit transient provider network error — re-routing for retry without counting as failure"
1125 );
1126 return reset_or_block(
1127 store,
1128 repo,
1129 task_id,
1130 reason,
1131 "provider_network_error",
1132 true,
1133 context,
1134 )
1135 .await;
1136 }
1137
1138 let failures = match crate::store::store_increment(
1139 &Some(std::sync::Arc::clone(store)),
1140 repo,
1141 task_id,
1142 "review_agent_failures",
1143 )
1144 .await
1145 {
1146 Ok(v) => v,
1147 Err(e) => {
1148 tracing::warn!(task_id, err = %e, "failed to increment review_agent_failures — skipping failure escalation this tick");
1149 0
1151 }
1152 };
1153 let blocking = failures >= MAX_REVIEW_AGENT_FAILURES;
1154 post_review_failure_comment(store, repo, task_id, reason, failures, blocking).await;
1156 if blocking {
1157 tracing::error!(
1158 task_id,
1159 reason,
1160 failures,
1161 "{context} failed too many times — blocking task"
1162 );
1163 recorded(
1164 store,
1165 repo,
1166 task_id,
1167 reason,
1168 "review_failure",
1169 false,
1170 ReviewOutcome::Block(format!("{context} failed {failures} times: {reason}")),
1171 )
1172 .await
1173 } else {
1174 tracing::error!(
1175 task_id,
1176 reason,
1177 failures,
1178 "{context} failed — resetting to NeedsReview for retry"
1179 );
1180 recorded(
1181 store,
1182 repo,
1183 task_id,
1184 reason,
1185 "review_failure",
1186 false,
1187 ReviewOutcome::Reset,
1188 )
1189 .await
1190 }
1191}
1192
1193async fn post_review_failure_comment(
1196 store: &Arc<TaskStore>,
1197 repo: &str,
1198 task_id: &str,
1199 reason: &str,
1200 failures: u64,
1201 blocking: bool,
1202) {
1203 let pr_number =
1204 match opt_store_get_task(&Some(std::sync::Arc::clone(store)), repo, task_id).await {
1205 Some(t) => match t.pr_number {
1206 Some(n) => n,
1207 None => return, },
1209 _ => return, };
1211
1212 let status = if blocking {
1213 "Task blocked for human review."
1214 } else {
1215 "Retrying with a different agent."
1216 };
1217
1218 let comment = format!(
1219 "⚠️ Review agent failed (attempt {}/{})\n\n**Reason:** {}\n\n{}{}",
1220 failures,
1221 MAX_REVIEW_AGENT_FAILURES,
1222 reason,
1223 status,
1224 crate::engine::orch_footer()
1225 );
1226
1227 let Ok(gh) = GhHttp::new() else { return };
1228 if let Err(e) = gh.add_comment(repo, &pr_number.to_string(), &comment).await {
1229 tracing::debug!(
1230 task_id,
1231 pr_number,
1232 error = %e,
1233 "failed to post review failure comment"
1234 );
1235 }
1236}
1237
1238#[cfg(test)]
1239mod tests {
1240 use super::*;
1241
1242 #[tokio::test]
1245 async fn classify_review_failure_incomplete_resets_without_counting() {
1246 let store = Arc::new(TaskStore::open_memory().await.unwrap());
1247 let repo = "owner/repo";
1248 let store_id = store
1249 .create_internal(
1250 repo,
1251 "Incomplete review test",
1252 "",
1253 "review",
1254 "review-3734",
1255 None,
1256 )
1257 .await
1258 .unwrap();
1259 let task_id = format!("internal:{store_id}");
1260
1261 for _ in 0..=MAX_REVIEW_AGENT_FAILURES {
1262 let outcome = classify_review_failure(
1263 &store,
1264 repo,
1265 &task_id,
1266 "incomplete: review session exited before producing a decision",
1267 "review agent",
1268 )
1269 .await;
1270 assert!(
1271 matches!(outcome, ReviewOutcome::Reset),
1272 "incomplete review failure must reset for retry, not block"
1273 );
1274 }
1275
1276 let task = opt_store_get_task(&Some(std::sync::Arc::clone(&store)), repo, &task_id)
1277 .await
1278 .expect("task should exist");
1279 assert_eq!(
1280 task.review_agent_failures, 0,
1281 "incomplete failures must not count toward review_agent_failures"
1282 );
1283 }
1284
1285 #[tokio::test]
1290 async fn classify_review_failure_truncated_resets_without_counting() {
1291 let store = Arc::new(TaskStore::open_memory().await.unwrap());
1292 let repo = "owner/repo";
1293 let store_id = store
1294 .create_internal(
1295 repo,
1296 "Truncated review test",
1297 "",
1298 "review",
1299 "review-1",
1300 None,
1301 )
1302 .await
1303 .unwrap();
1304 let task_id = format!("internal:{store_id}");
1305
1306 for _ in 0..=MAX_REVIEW_AGENT_FAILURES {
1307 let outcome = classify_review_failure(
1308 &store,
1309 repo,
1310 &task_id,
1311 "truncated: review output was cut off before completion \
1312 (output/reasoning token budget exceeded)",
1313 "review agent",
1314 )
1315 .await;
1316 assert!(
1317 matches!(outcome, ReviewOutcome::Reset),
1318 "truncated review failure must reset for retry, not block"
1319 );
1320 }
1321
1322 let task = opt_store_get_task(&Some(std::sync::Arc::clone(&store)), repo, &task_id)
1323 .await
1324 .expect("task should exist");
1325 assert_eq!(
1326 task.review_agent_failures, 0,
1327 "truncated failures must not count toward review_agent_failures"
1328 );
1329 }
1330
1331 #[tokio::test]
1337 async fn classify_review_failure_non_conflict_merge_failure_retries_merge() {
1338 let store = Arc::new(TaskStore::open_memory().await.unwrap());
1339 let repo = "owner/repo";
1340 let store_id = store
1341 .create_internal(repo, "Merge retry", "", "review", "review-3783", None)
1342 .await
1343 .unwrap();
1344 let task_id = format!("internal:{store_id}");
1345
1346 let error_messages = [
1347 "merge failed: GitHub REST API rate-limited, skipping call for 187s",
1348 "merge failed: GitHub PR merge failed (502 Bad Gateway)",
1349 "merge failed: required status check still pending",
1350 ];
1351
1352 for _ in 0..=MAX_REVIEW_AGENT_FAILURES {
1353 for msg in &error_messages {
1354 let outcome =
1355 classify_review_failure(&store, repo, &task_id, msg, "review agent").await;
1356 assert!(
1357 matches!(outcome, ReviewOutcome::RetryMerge),
1358 "non-conflict merge failure '{msg}' must retry the merge, not re-dispatch the agent"
1359 );
1360 }
1361 }
1362
1363 let task = opt_store_get_task(&Some(std::sync::Arc::clone(&store)), repo, &task_id)
1364 .await
1365 .expect("task should exist");
1366 assert_eq!(
1367 task.review_agent_failures, 0,
1368 "a stuck merge is not a review failure"
1369 );
1370 }
1371
1372 #[tokio::test]
1374 async fn classify_review_failure_merge_conflict_still_reroutes() {
1375 let store = Arc::new(TaskStore::open_memory().await.unwrap());
1376 let repo = "owner/repo";
1377 let store_id = store
1378 .create_internal(repo, "Merge conflict", "", "review", "review-3783b", None)
1379 .await
1380 .unwrap();
1381 let task_id = format!("internal:{store_id}");
1382
1383 for msg in [
1384 "merge failed: PR is not mergeable",
1385 "merge conflict detected on PR #12",
1386 ] {
1387 let outcome =
1388 classify_review_failure(&store, repo, &task_id, msg, "review agent").await;
1389 assert!(
1390 matches!(outcome, ReviewOutcome::Reroute),
1391 "content conflict '{msg}' must go back to the task agent"
1392 );
1393 }
1394 }
1395
1396 #[tokio::test]
1400 async fn classify_review_failure_network_error_resets_without_counting() {
1401 let store = Arc::new(TaskStore::open_memory().await.unwrap());
1402 let repo = "owner/repo";
1403 let store_id = store
1404 .create_internal(
1405 repo,
1406 "Network error review test",
1407 "",
1408 "review",
1409 "review-2",
1410 None,
1411 )
1412 .await
1413 .unwrap();
1414 let task_id = format!("internal:{store_id}");
1415
1416 let error_messages = [
1417 "API Error: Unable to connect to API (ConnectionRefused)",
1418 "connection refused",
1419 "ECONNREFUSED",
1420 "Upstream idle timeout exceeded",
1421 "connect ETIMEDOUT",
1422 "network request failed",
1423 ];
1424
1425 for msg in &error_messages {
1426 store
1428 .set_fields(store_id, &[("review_resets", serde_json::json!(0))])
1429 .await
1430 .unwrap();
1431 let outcome =
1432 classify_review_failure(&store, repo, &task_id, msg, "review agent").await;
1433 assert!(
1434 matches!(outcome, ReviewOutcome::Reset),
1435 "network error '{msg}' must reset for retry, not block"
1436 );
1437 }
1438
1439 let task = opt_store_get_task(&Some(std::sync::Arc::clone(&store)), repo, &task_id)
1440 .await
1441 .expect("task should exist");
1442 assert_eq!(
1443 task.review_agent_failures, 0,
1444 "network errors must not count toward review_agent_failures"
1445 );
1446 }
1447
1448 #[tokio::test]
1452 async fn classify_review_failure_missing_review_model_is_rate_limited() {
1453 let store = Arc::new(TaskStore::open_memory().await.unwrap());
1454 let repo = "owner/repo";
1455 let store_id = store
1456 .create_internal(
1457 repo,
1458 "Missing review model test",
1459 "",
1460 "review",
1461 "review-3",
1462 None,
1463 )
1464 .await
1465 .unwrap();
1466 let task_id = format!("internal:{store_id}");
1467
1468 let error_messages = [
1469 "no review agent with a configured review model is available for task internal:123",
1470 "review model missing for task internal:123 after agent selection — refusing to dispatch",
1471 ];
1472
1473 for _ in 0..=MAX_REVIEW_AGENT_FAILURES {
1474 for msg in &error_messages {
1475 let outcome =
1476 classify_review_failure(&store, repo, &task_id, msg, "review_and_merge").await;
1477 assert!(
1478 matches!(outcome, ReviewOutcome::RateLimited),
1479 "missing review model error '{msg}' must defer, not block"
1480 );
1481 }
1482 }
1483
1484 let task = opt_store_get_task(&Some(std::sync::Arc::clone(&store)), repo, &task_id)
1485 .await
1486 .expect("task should exist");
1487 assert_eq!(
1488 task.review_agent_failures, 0,
1489 "missing review model errors must not count toward review_agent_failures"
1490 );
1491 }
1492
1493 #[tokio::test]
1495 async fn classify_review_failure_blocks_after_max_resets() {
1496 let store = Arc::new(TaskStore::open_memory().await.unwrap());
1497 let repo = "owner/repo";
1498 let store_id = store
1499 .create_internal(repo, "Looping review", "", "review", "review-resets", None)
1500 .await
1501 .unwrap();
1502 let task_id = format!("internal:{store_id}");
1503 let msg = "parse error: failed to parse review response";
1504
1505 for attempt in 1..MAX_REVIEW_RESETS {
1506 let outcome =
1507 classify_review_failure(&store, repo, &task_id, msg, "review agent").await;
1508 assert!(
1509 matches!(outcome, ReviewOutcome::Reset),
1510 "reset {attempt} must retry the review"
1511 );
1512 }
1513 let outcome = classify_review_failure(&store, repo, &task_id, msg, "review agent").await;
1514 assert!(
1515 matches!(outcome, ReviewOutcome::Block(ref r) if r.contains("without a decision")),
1516 "reset {MAX_REVIEW_RESETS} must block the task"
1517 );
1518
1519 let task = opt_store_get_task(&Some(std::sync::Arc::clone(&store)), repo, &task_id)
1520 .await
1521 .expect("task should exist");
1522 assert_eq!(
1523 u64::try_from(task.review_resets).unwrap(),
1524 MAX_REVIEW_RESETS
1525 );
1526 assert_eq!(
1527 task.review_agent_failures, 0,
1528 "a reset must not count as a review failure"
1529 );
1530 }
1531
1532 #[tokio::test]
1538 async fn classify_review_failure_rate_limit_clears_refire_quota() {
1539 let store = Arc::new(TaskStore::open_memory().await.unwrap());
1540 let repo = "owner/repo";
1541 let store_id = store
1542 .create_internal(
1543 repo,
1544 "Rate limited review",
1545 "",
1546 "review",
1547 "review-3767",
1548 None,
1549 )
1550 .await
1551 .unwrap();
1552 let task_id = format!("internal:{store_id}");
1553
1554 let error_messages = [
1555 "claude failed: rate limit reached for model sonnet",
1556 "GitHub API GET https://api.github.com/repos/owner/repo/pulls/65 failed (403): API rate limit exceeded",
1557 "GitHub core API rate-limited, skipping call for 1800s",
1558 "GitHub API GET https://api.github.com/repos/owner/repo/pulls/65 failed (429): Too Many Requests",
1559 ];
1560
1561 store
1563 .set_fields(store_id, &[("needs_review_refires", serde_json::json!(5))])
1564 .await
1565 .unwrap();
1566
1567 for _ in 0..=MAX_REVIEW_AGENT_FAILURES {
1568 for msg in &error_messages {
1569 let outcome =
1570 classify_review_failure(&store, repo, &task_id, msg, "review agent").await;
1571 assert!(
1572 matches!(outcome, ReviewOutcome::RateLimited | ReviewOutcome::Reset),
1573 "rate limit '{msg}' must defer for retry, not block"
1574 );
1575 }
1576 }
1577
1578 let task = opt_store_get_task(&Some(std::sync::Arc::clone(&store)), repo, &task_id)
1579 .await
1580 .expect("task should exist");
1581 assert_eq!(
1582 task.needs_review_refires, 0,
1583 "a rate limit must not consume the rebroadcast quota"
1584 );
1585 assert_eq!(
1586 task.review_agent_failures, 0,
1587 "rate limits must not count toward review_agent_failures"
1588 );
1589 assert_ne!(
1590 task.status,
1591 crate::store::TaskStatus::Blocked,
1592 "a rate-limited review must leave the task retryable"
1593 );
1594
1595 let attempts = store
1596 .get_activity(store_id, None)
1597 .await
1598 .unwrap()
1599 .into_iter()
1600 .filter(|a| a.event_type == "review_attempt_failed")
1601 .collect::<Vec<_>>();
1602 assert_eq!(
1603 attempts.len(),
1604 (usize::try_from(MAX_REVIEW_AGENT_FAILURES).unwrap() + 1) * error_messages.len(),
1605 "every failed attempt must be recorded for `orch task inspect`"
1606 );
1607 assert!(
1608 attempts.iter().all(|a| a
1609 .details
1610 .get("external")
1611 .and_then(serde_json::Value::as_bool)
1612 == Some(true)),
1613 "rate limits must be recorded as an external cause"
1614 );
1615 assert!(
1616 attempts.iter().any(
1617 |a| a.details.get("reason").and_then(|v| v.as_str()) == Some(error_messages[0])
1618 ),
1619 "the recorded attempt must carry that attempt's error"
1620 );
1621 }
1622}