1use crate::backends::{ExternalBackend, ExternalId, ExternalTask, Status};
4use crate::config;
5use crate::engine;
6use crate::engine::auto_merge;
7use crate::engine::cooldown::AGENT_COOLDOWN_SECS;
8use crate::engine::dispatch_guard::DispatchGuard;
9use crate::engine::events::TaskEvent;
10use crate::engine::review;
11use crate::engine::review::{MAX_REVIEW_AGENT_FAILURES, ReviewDecision, review_and_merge};
12use crate::engine::review_gate;
13use crate::engine::review_gate::CiGate;
14use crate::engine::review_shipcheck;
15use crate::engine::review_shipcheck::Prerun;
16use crate::engine::router::Router;
17use crate::engine::runner::agent::Shipcheck;
18use crate::engine::runner::agents::patterns;
19use crate::engine::runner::git_ops;
20use crate::engine::tasks;
21use crate::engine::tasks::TaskManager;
22use crate::github::http::GhHttp;
23use crate::repo_context::REPO_CONTEXT;
24use crate::store;
25use crate::store::TaskStatus;
26use crate::store::tasks as store_tasks;
27use crate::store::{TaskStore, opt_store_get_task};
28use crate::tmux::TmuxManager;
29use dashmap::DashMap;
30use dashmap::mapref::entry::Entry;
31use std::path::PathBuf;
32use std::sync::Arc;
33use std::time::Duration;
34use tokio::sync::broadcast;
35use tokio::sync::{RwLock, Semaphore};
36use tokio::time;
37
38enum ReviewOutcome {
39 Reset,
40 RateLimited,
41 RetryMerge,
46 Approved,
49 Reroute,
51 Block(String),
52 Ok,
53}
54
55#[expect(
61 clippy::too_many_arguments,
62 reason = "the arguments have no natural grouping"
63)]
64pub fn spawn(
65 mut rx: broadcast::Receiver<TaskEvent>,
66 backend: Arc<dyn ExternalBackend>,
67 tmux: Arc<TmuxManager>,
68 semaphore: Arc<Semaphore>,
69 task_manager: Arc<TaskManager>,
70 router: Arc<RwLock<Router>>,
71 dispatching: Arc<DashMap<String, String>>,
72 store: Arc<TaskStore>,
73 repo: String,
74) {
75 tokio::spawn(async move {
76 loop {
77 match rx.recv().await {
78 Ok(event) if event.new_status == "needs_review" && event.repo == repo => {
79 let enable_review =
81 config::get("workflow.enable_review_agent").map_or(true, |v| v != "false");
82 if !enable_review {
83 continue;
84 }
85
86 let task_id = &event.task_id;
87 let dispatch_key = format!("{repo}/{task_id}");
88
89 {
94 match dispatching.entry(dispatch_key.clone()) {
95 Entry::Occupied(existing) => {
96 let existing_id = existing.get().clone();
97 drop(existing);
98 if existing_id == *task_id {
99 tracing::debug!(
100 task_id,
101 "task locked by dispatch flow, skipping event-driven review"
102 );
103 } else {
104 tracing::warn!(
105 task_id,
106 existing_task_id = existing_id,
107 dispatch_key,
108 "dispatch key collision: unexpected task already holds this key"
109 );
110 }
111 continue;
112 }
113 Entry::Vacant(slot) => {
114 slot.insert(task_id.clone());
115 }
116 }
117 }
118
119 let dispatch_guard =
121 DispatchGuard::new(Arc::clone(&dispatching), dispatch_key.clone());
122
123 tracing::info!(task_id, "event-driven review triggered");
124
125 let task = {
127 if tasks::is_internal_id(task_id)
128 || match store.has_external_tasks(&repo).await {
129 Ok(has_external) => has_external,
130 Err(e) => {
131 tracing::warn!(
132 repo = %repo,
133 task_id,
134 error = %e,
135 "failed to check external task sentinel; keeping store-first review lookup"
136 );
137 true
138 }
139 }
140 {
141 let result = if let Some(numeric_id) = tasks::parse_internal_id(task_id)
144 {
145 store.get(numeric_id).await.map(Some)
146 } else {
147 store.get_by_external_id(&repo, task_id).await
148 };
149 match result {
150 Ok(Some(t)) if t.status == TaskStatus::NeedsReview => {
151 Some(tasks::store_task_to_external(&t))
152 }
153 Ok(Some(t)) => {
154 tracing::debug!(
155 task_id,
156 status = t.status.as_str(),
157 "task found but not in needs_review status, skipping"
158 );
159 None
160 }
161 Ok(None) => None,
162 Err(e) => {
163 tracing::warn!(task_id, err = %e, "store lookup failed for review");
164 None
165 }
166 }
167 } else {
168 match backend.list_by_status(Status::NeedsReview).await {
170 Ok(tasks) => tasks.into_iter().find(|t| t.id.0 == *task_id),
171 Err(e) => {
172 tracing::warn!(task_id, err = %e, "backend lookup failed for review");
173 None
174 }
175 }
176 }
177 };
178
179 let Some(task) = task else {
180 tracing::debug!(
181 task_id,
182 "task not found in needs_review list (may have already transitioned)"
183 );
184 continue;
185 };
186
187 if let Some(stored) =
188 opt_store_get_task(&Some(Arc::clone(&store)), &repo, &task.id.0).await
189 {
190 let no_code_review = stored.worktree.trim().is_empty()
191 && stored.branch.trim().is_empty()
192 && stored.pr_number.is_none();
193 if no_code_review {
194 tracing::info!(
195 task_id,
196 "no worktree/branch/pr for needs_review task — skipping review dispatch and marking done"
197 );
198 store::store_reset_failure_counters(
199 &Some(Arc::clone(&store)),
200 &repo,
201 &task.id.0,
202 )
203 .await;
204 if let Err(e) = store::store_set_result(
205 &Some(Arc::clone(&store)),
206 &repo,
207 &task.id.0,
208 &[("needs_review_refires", serde_json::json!(0))],
209 )
210 .await
211 {
212 tracing::warn!(
213 task_id,
214 err = %e,
215 "failed to reset needs_review_refires while skipping no-code review"
216 );
217 }
218 if let Err(e) = task_manager
219 .update_task_status(&task.id, Status::Done)
220 .await
221 {
222 tracing::error!(
223 task_id,
224 err = %e,
225 "failed to mark no-code needs_review task as done"
226 );
227 }
228 continue;
229 }
230 }
231
232 {
239 let router_guard = router.read().await;
240 let no_routable = !router_guard.available_agents.is_empty()
241 && router_guard.healthy_agent_count("review") == 0;
242 drop(router_guard);
243 if no_routable {
244 tracing::info!(
245 task_id,
246 "no routable review agents — leaving task in needs_review for sync to re-fire"
247 );
248 continue;
249 }
250 }
251
252 if !ci_allows_review(&store, &task_manager, &repo, &task).await {
253 continue;
254 }
255
256 let Ok(permit) = Arc::clone(&semaphore).try_acquire_owned() else {
258 tracing::debug!(
259 task_id,
260 "all review slots busy; sync catch-up will re-fire when a slot opens"
261 );
262 continue;
263 };
264
265 if let Err(e) = task_manager
267 .update_task_status(&task.id, Status::InReview)
268 .await
269 {
270 tracing::warn!(task_id, err = %e, "failed to transition to InReview");
271 drop(permit);
272 continue;
273 }
274
275 let backend_c = Arc::clone(&backend);
276 let task_manager_c = Arc::clone(&task_manager);
277 let tmux_c = Arc::clone(&tmux);
278 let router_c = Arc::clone(&router);
279 let store_c = Arc::clone(&store);
280 let repo_s = repo.clone();
281 tokio::spawn(REPO_CONTEXT.scope(repo_s.clone(), async move {
282 let _dispatch_guard = dispatch_guard; let tid = task.id.0.clone();
284
285 let pre_check_all_cooled = {
290 let r = router_c.read().await;
291 !r.available_agents.is_empty() && r.healthy_agent_count("review") == 0
292 };
293 if pre_check_all_cooled {
294 tracing::info!(
295 task_id = tid,
296 "all agents cooled at review spawn time — resetting to needs_review for sync to re-fire"
297 );
298 if let Err(e) = task_manager_c
299 .update_task_status(
300 &ExternalId(tid.clone()),
301 Status::NeedsReview,
302 )
303 .await
304 {
305 tracing::error!(task_id = %tid, err = %e, "pre-check: update_task_status(NeedsReview) failed — task may be stuck in InReview");
306 }
307 drop(permit);
308 return;
309 }
310
311 let outcome = match Box::pin(review_and_merge(
312 &task,
313 &backend_c,
314 &tmux_c,
315 &repo_s,
316 &router_c,
317 &task_manager_c,
318 &store_c,
319 ))
320 .await
321 {
322 Ok(ReviewDecision::Blocked(reason)) => {
323 tracing::error!(
324 task_id = tid,
325 reason,
326 "review gate blocked after repeated failures — marking task blocked"
327 );
328 ReviewOutcome::Block(format!(
329 "review gate blocked after repeated failures: {reason}"
330 ))
331 }
332 Ok(ReviewDecision::Failed(reason)) => {
333 classify_review_failure(
334 &store_c, &repo_s, &tid, &reason, "review agent",
335 )
336 .await
337 }
338 Err(e) => {
339 let reason = format!("{e:#}");
340 classify_review_failure(
341 &store_c, &repo_s, &tid, &reason, "review_and_merge",
342 )
343 .await
344 }
345 Ok(ReviewDecision::Approve { .. }) => {
346 store::store_reset_failure_counters(
351 &Some(Arc::clone(&store_c)),
352 &repo_s,
353 &tid,
354 )
355 .await;
356 if let Err(e) = store::store_set_result(
358 &Some(Arc::clone(&store_c)),
359 &repo_s,
360 &tid,
361 &[("needs_review_refires", serde_json::json!(0))],
362 )
363 .await
364 {
365 tracing::warn!(task_id = %tid, err = %e, "failed to write needs_review_refires to store");
366 }
367
368 ReviewOutcome::Approved
372 }
373 Ok(ReviewDecision::Rerouted) => {
374 ReviewOutcome::Ok
378 }
379 Ok(ReviewDecision::Skipped) => {
380 store::store_reset_failure_counters(
383 &Some(Arc::clone(&store_c)),
384 &repo_s,
385 &tid,
386 )
387 .await;
388 if let Err(e) = store::store_set_result(
389 &Some(Arc::clone(&store_c)),
390 &repo_s,
391 &tid,
392 &[("needs_review_refires", serde_json::json!(0))],
393 )
394 .await
395 {
396 tracing::warn!(task_id = %tid, err = %e, "failed to write needs_review_refires to store");
397 }
398 ReviewOutcome::Ok
399 }
400 Ok(ReviewDecision::RequestChanges { .. }) => {
401 store::store_reset_failure_counters(
404 &Some(Arc::clone(&store_c)),
405 &repo_s,
406 &tid,
407 )
408 .await;
409 if let Err(e) = store::store_set_result(
410 &Some(Arc::clone(&store_c)),
411 &repo_s,
412 &tid,
413 &[("needs_review_refires", serde_json::json!(0))],
414 )
415 .await
416 {
417 tracing::warn!(task_id = %tid, err = %e, "failed to write needs_review_refires to store");
418 }
419 ReviewOutcome::Ok
420 }
421 };
422 let fetched: Result<Option<(TaskStatus, Option<String>)>, anyhow::Error> = async {
433 let id = match store_c.resolve_task_id(&repo_s, &tid).await {
434 Ok(Some(id)) => id,
435 Ok(None) => return Ok(None), Err(e) => return Err(e), };
438 let task = store_c.get(id).await?;
439 Ok(Some((task.status, task.block_reason)))
440 }
441 .await;
442
443 let (current_status, current_block_reason) = match fetched {
444 Ok(Some((status, block_reason))) => (Some(status), block_reason),
445 Ok(None) => {
446 (None, None)
448 }
449 Err(e) => {
450 tracing::error!(
452 task_id = tid,
453 error = %e,
454 "review outcome retained — DB lookup failed, not treating as stale"
455 );
456 (Some(TaskStatus::InReview), None)
459 }
460 };
461
462 if !matches!(current_status, Some(TaskStatus::InReview)) {
463 let ci_blocked = matches!(
464 current_status,
465 Some(TaskStatus::Blocked)
466 ) && current_block_reason
467 .as_deref()
468 .is_some_and(|r| r.contains("CI failure limit"));
469 let expected = ci_blocked
470 || matches!(
471 current_status,
472 Some(TaskStatus::Done | TaskStatus::Approved |
473TaskStatus::Routed | TaskStatus::New)
474 );
475 if expected {
476 tracing::debug!(
477 task_id = tid,
478 current_status = current_status.as_ref().map_or("unknown", store_tasks::TaskStatus::as_str),
479 "review outcome discarded — task transitioned away from in_review (expected)"
480 );
481 } else {
482 tracing::warn!(
483 task_id = tid,
484 current_status = current_status.as_ref().map_or("unknown", store_tasks::TaskStatus::as_str),
485 "review outcome discarded — task is no longer in_review (stale review agent result)"
486 );
487 }
488
489 let stale_session = tmux_c.session_name(&repo_s, &format!("{tid}-review"));
491 if let Err(e) = tmux_c.kill_session(&stale_session).await {
492 tracing::debug!(task_id = tid, error = %e, "failed to kill stale review session");
493 }
494
495 drop(permit);
496 return;
497 }
498
499 match outcome {
500 ReviewOutcome::Reset => {
501 let stale_session =
508 tmux_c.session_name(&repo_s, &format!("{tid}-review"));
509 tmux_c.kill_session(&stale_session).await.ok();
510 time::sleep(Duration::from_secs(10)).await;
513 if let Err(e) = task_manager_c
514 .update_task_status(
515 &ExternalId(tid.clone()),
516 Status::NeedsReview,
517 )
518 .await
519 {
520 tracing::error!(task_id = %tid, err = %e, "update_task_status(NeedsReview) failed — task may be stuck in InReview");
521 }
522 }
523 ReviewOutcome::Reroute => {
524 let stale_session =
527 tmux_c.session_name(&repo_s, &format!("{tid}-review"));
528 tmux_c.kill_session(&stale_session).await.ok();
529 tracing::info!(
530 task_id = tid,
531 "re-routing to task agent to resolve merge conflict"
532 );
533 if let Err(e) = task_manager_c
534 .update_task_status(
535 &ExternalId(tid.clone()),
536 Status::Routed,
537 )
538 .await
539 {
540 tracing::error!(task_id = %tid, err = %e, "update_task_status(Routed) failed");
541 }
542 }
543 ReviewOutcome::Approved | ReviewOutcome::RetryMerge => {
544 let stale_session =
547 tmux_c.session_name(&repo_s, &format!("{tid}-review"));
548 tmux_c.kill_session(&stale_session).await.ok();
549 match task_manager_c
550 .update_task_status_if(
551 &ExternalId(tid.clone()),
552 Status::Approved,
553 Status::InReview,
554 )
555 .await
556 {
557 Ok(true) => tracing::info!(
558 task_id = tid,
559 "approval received, task approved until the PR is merged"
560 ),
561 Ok(false) => tracing::debug!(
562 task_id = tid,
563 "approval received after the task left in_review"
564 ),
565 Err(e) => tracing::error!(
566 task_id = %tid,
567 err = %e,
568 "update_task_status_if(Approved) failed, task may be stuck in in_review"
569 ),
570 }
571 }
572 ReviewOutcome::RateLimited => {
573 let stale_session =
575 tmux_c.session_name(&repo_s, &format!("{tid}-review"));
576 tmux_c.kill_session(&stale_session).await.ok();
577
578 let wait_secs = {
582 let r = router_c.read().await;
583 if r.healthy_agent_count("review") > 0 {
584 30u64
585 } else {
586 let now = chrono::Utc::now().timestamp();
587 r.earliest_cooldown_until(Some("review"))
588 .map_or(AGENT_COOLDOWN_SECS as u64, |until| u64::try_from((until - now).max(1)).unwrap_or(1))
589 }
590 };
591 tracing::info!(
592 task_id = tid,
593 wait_secs,
594 "deferring review retry due to agent rate limit cooldown"
595 );
596 time::sleep(Duration::from_secs(wait_secs)).await;
597 if let Err(e) = task_manager_c
598 .update_task_status(
599 &ExternalId(tid.clone()),
600 Status::NeedsReview,
601 )
602 .await
603 {
604 tracing::error!(task_id = %tid, err = %e, "update_task_status(NeedsReview) failed after rate limit backoff — task may be stuck in InReview");
605 }
606 }
607 ReviewOutcome::Block(reason) => {
608 let fields = [
612 (
613 "block_reason",
614 serde_json::json!("review agent blocked — exceeded failure threshold"),
615 ),
616 ("last_error", serde_json::json!(reason)),
617 ];
618 if let Err(e) = task_manager_c
619 .update_task_status_and_result(
620 &ExternalId(tid.clone()),
621 Status::Blocked,
622 &fields,
623 )
624 .await
625 {
626 tracing::error!(task_id = %tid, err = %e, "update_task_status_and_result(Blocked) failed — task may be stuck in InReview");
627 }
628 }
629 ReviewOutcome::Ok => {}
630 }
631
632 drop(permit);
633 }));
635
636 time::sleep(Duration::from_secs(2)).await;
641 }
642 Ok(_) => {} Err(broadcast::error::RecvError::Lagged(n)) => {
644 tracing::debug!(
645 n,
646 "review subscriber lagged; sync catch-up will re-fire missed NeedsReview tasks"
647 );
648 }
649 Err(_) => break, }
651 }
652 });
653}
654
655async fn ci_allows_review(
670 store: &Arc<TaskStore>,
671 task_manager: &Arc<TaskManager>,
672 repo: &str,
673 task: &ExternalTask,
674) -> bool {
675 let task_id = task.id.0.as_str();
676 let stored = opt_store_get_task(&Some(Arc::clone(store)), repo, task_id).await;
677 let Some(pr_number) = stored
678 .as_ref()
679 .and_then(|t| t.pr_number)
680 .and_then(|n| u64::try_from(n).ok())
681 else {
682 return true;
683 };
684 let worktree = stored
685 .map(|t| PathBuf::from(t.worktree))
686 .filter(|p| !p.as_os_str().is_empty() && p.exists());
687 let gh = GhHttp::new();
688 if let Ok(gh) = &gh
690 && !auto_merge::prepare_branch_for_review(task, repo, pr_number, gh, task_manager, store)
691 .await
692 {
693 reset_refires(store, repo, task_id).await;
694 return false;
695 }
696 let gate = match gh {
697 Ok(gh) => review_gate::review_ci_gate(&gh, store, repo, task_id, pr_number).await,
698 Err(e) => Err(e),
699 };
700
701 let hold = match gate {
702 Ok(CiGate::Start) => return true,
703 Err(e) => {
705 tracing::warn!(task_id, pr_number, err = %e, "CI gate lookup failed, starting the review");
706 return true;
707 }
708 Ok(CiGate::Red(failed)) => {
709 route_back_on_red_ci(store, task_manager, repo, task, pr_number, &failed).await;
710 return false;
711 }
712 Ok(CiGate::Wait) => "CI still running on the PR head, review waits",
713 Ok(CiGate::NotStarted { waited_out }) => match local_ci_worktree(repo, worktree).await {
715 Some(wt) => match review_shipcheck::prerun(&wt).await {
716 Prerun::Ready => return true,
717 Prerun::Running => "shipcheck running on the PR head, review waits",
718 },
719 None if waited_out => {
720 tracing::warn!(
721 task_id,
722 pr_number,
723 "no CI ran on the PR head, starting the review anyway"
724 );
725 return true;
726 }
727 None => "no CI started on the PR head yet, review waits",
728 },
729 };
730
731 tracing::info!(task_id, pr_number, "{hold}");
732 reset_refires(store, repo, task_id).await;
733
734 false
735}
736
737async fn reset_refires(store: &Arc<TaskStore>, repo: &str, task_id: &str) {
739 if let Err(e) = store::store_set_result(
740 &Some(Arc::clone(store)),
741 repo,
742 task_id,
743 &[("needs_review_refires", serde_json::json!(0))],
744 )
745 .await
746 {
747 tracing::warn!(task_id, err = %e, "failed to reset needs_review_refires for a task held before its review");
748 }
749}
750
751async fn local_ci_worktree(repo: &str, worktree: Option<PathBuf>) -> Option<PathBuf> {
753 let wt = worktree?;
754 let sandboxed = review::review_sandboxed(repo);
755 let shipcheck = Shipcheck::detect(&wt).await.for_prompt(sandboxed);
756 (shipcheck == Shipcheck::Available).then_some(wt)
757}
758
759async fn route_back_on_red_ci(
761 store: &Arc<TaskStore>,
762 task_manager: &Arc<TaskManager>,
763 repo: &str,
764 task: &ExternalTask,
765 pr_number: u64,
766 failed: &[String],
767) {
768 let task_id = task.id.0.as_str();
769 let failures = match store::store_increment(
770 &Some(Arc::clone(store)),
771 repo,
772 task_id,
773 "ci_merge_failures",
774 )
775 .await
776 {
777 Ok(v) => v,
778 Err(e) => {
779 tracing::warn!(task_id, err = %e, "failed to increment ci_merge_failures, review waits");
780 return;
781 }
782 };
783 let checks = failed.join(", ");
784 let max = auto_merge::MAX_CI_MERGE_FAILURES;
785 let (status, fields) = if failures >= max {
786 tracing::error!(
787 task_id,
788 pr_number,
789 failures,
790 checks,
791 "CI failed before review, CI failure limit reached"
792 );
793 (
794 Status::Blocked,
795 vec![(
796 "block_reason",
797 serde_json::json!(format!("CI failure limit ({max}) reached before review")),
798 )],
799 )
800 } else {
801 tracing::warn!(
802 task_id,
803 pr_number,
804 failures,
805 checks,
806 "CI failed before review, routing back to the task agent"
807 );
808 (
809 Status::Routed,
810 vec![
811 (
812 "last_error",
813 serde_json::json!(format!(
814 "CI failed on PR #{pr_number} before review: {checks}. Fix the failing checks"
815 )),
816 ),
817 (
818 "route_reason",
819 serde_json::json!("re-dispatch after CI failed before review"),
820 ),
821 ],
822 )
823 };
824 if let Err(e) = task_manager
825 .update_task_status_and_result(&task.id, status, &fields)
826 .await
827 {
828 tracing::error!(task_id, err = %e, "failed to move a task whose CI failed before review");
829 }
830}
831
832async fn recorded(
833 store: &Arc<TaskStore>,
834 repo: &str,
835 task_id: &str,
836 reason: &str,
837 cause: &str,
838 external: bool,
839 outcome: ReviewOutcome,
840) -> ReviewOutcome {
841 let store_id = match store.resolve_task_id(repo, task_id).await {
842 Ok(Some(id)) => id,
843 Ok(None) => {
844 tracing::warn!(
845 task_id,
846 "review attempt not recorded — task not found in store"
847 );
848 return outcome;
849 }
850 Err(e) => {
851 tracing::warn!(task_id, err = %e, "review attempt not recorded — task id lookup failed");
852 return outcome;
853 }
854 };
855
856 if let Err(e) = store
857 .append_activity(
858 store_id,
859 "review_attempt_failed",
860 None,
861 None,
862 None,
863 None,
864 Some(&serde_json::json!({
865 "cause": cause,
866 "external": external,
867 "reason": reason,
868 })),
869 )
870 .await
871 {
872 tracing::warn!(task_id, err = %e, "failed to record review attempt failure");
873 }
874
875 if external
876 && let Err(e) = store
877 .set_fields(store_id, &[("needs_review_refires", serde_json::json!(0))])
878 .await
879 {
880 tracing::warn!(task_id, err = %e, "failed to clear needs_review_refires after external review failure");
881 }
882
883 outcome
884}
885
886pub(crate) const MAX_REVIEW_RESETS: u64 = 6;
888
889async fn reset_or_block(
892 store: &Arc<TaskStore>,
893 repo: &str,
894 task_id: &str,
895 reason: &str,
896 cause: &str,
897 external: bool,
898 context: &str,
899) -> ReviewOutcome {
900 let resets = match store::store_increment(
901 &Some(Arc::clone(store)),
902 repo,
903 task_id,
904 "review_resets",
905 )
906 .await
907 {
908 Ok(v) => v,
909 Err(e) => {
910 tracing::warn!(task_id, err = %e, "failed to increment review_resets");
911 0
912 }
913 };
914 let outcome = if resets >= MAX_REVIEW_RESETS {
915 tracing::error!(
916 task_id,
917 reason,
918 resets,
919 "{context} ended {resets} times without a decision, blocking task"
920 );
921 ReviewOutcome::Block(format!(
922 "{context} ended {resets} times without a decision: {reason}"
923 ))
924 } else {
925 ReviewOutcome::Reset
926 };
927 recorded(store, repo, task_id, reason, cause, external, outcome).await
928}
929
930async fn classify_review_failure(
937 store: &Arc<TaskStore>,
938 repo: &str,
939 task_id: &str,
940 reason: &str,
941 context: &str,
942) -> ReviewOutcome {
943 let lower_reason = reason.to_lowercase();
947
948 if lower_reason.contains("parse error") {
952 tracing::warn!(
953 task_id,
954 reason,
955 "{context} hit parse error — re-routing for retry without counting as failure"
956 );
957 return reset_or_block(store, repo, task_id, reason, "parse_error", false, context).await;
958 }
959
960 if lower_reason.contains("incomplete:") {
963 tracing::warn!(
964 task_id,
965 reason,
966 "{context} session incomplete — re-routing for retry without counting as failure"
967 );
968 return reset_or_block(
969 store,
970 repo,
971 task_id,
972 reason,
973 "incomplete_session",
974 false,
975 context,
976 )
977 .await;
978 }
979
980 if lower_reason.contains("truncated") {
984 tracing::warn!(
985 task_id,
986 reason,
987 "{context} output truncated by token budget — re-routing for retry without counting as failure"
988 );
989 return reset_or_block(
990 store,
991 repo,
992 task_id,
993 reason,
994 "truncated_output",
995 false,
996 context,
997 )
998 .await;
999 }
1000
1001 if lower_reason.contains("not yet computed") {
1006 tracing::warn!(
1007 task_id,
1008 reason,
1009 "{context} PR mergeability check still pending — will retry merge without re-reviewing"
1010 );
1011 return recorded(
1012 store,
1013 repo,
1014 task_id,
1015 reason,
1016 "mergeability_pending",
1017 true,
1018 ReviewOutcome::RetryMerge,
1019 )
1020 .await;
1021 }
1022
1023 if lower_reason.contains("merge conflict") || lower_reason.contains("not mergeable") {
1028 tracing::warn!(
1029 task_id,
1030 reason,
1031 "{context} hit merge conflict — routing to task agent to resolve"
1032 );
1033 return recorded(
1034 store,
1035 repo,
1036 task_id,
1037 reason,
1038 "merge_conflict",
1039 false,
1040 ReviewOutcome::Reroute,
1041 )
1042 .await;
1043 }
1044
1045 if lower_reason.contains("merge failed") {
1050 tracing::warn!(
1051 task_id,
1052 reason,
1053 "{context} merge failed without a conflict — retrying the merge, not re-dispatching the agent"
1054 );
1055 return recorded(
1056 store,
1057 repo,
1058 task_id,
1059 reason,
1060 "merge_retry",
1061 true,
1062 ReviewOutcome::RetryMerge,
1063 )
1064 .await;
1065 }
1066
1067 if lower_reason.contains("no review agent") || lower_reason.contains("review model missing") {
1072 tracing::warn!(
1073 task_id,
1074 reason,
1075 "{context}: no review model available — deferring retry until one is available"
1076 );
1077 return recorded(
1078 store,
1079 repo,
1080 task_id,
1081 reason,
1082 "no_review_model",
1083 true,
1084 ReviewOutcome::RateLimited,
1085 )
1086 .await;
1087 }
1088
1089 if lower_reason.contains("rate limit")
1092 || lower_reason.contains("rate-limited")
1093 || lower_reason.contains("rate limited")
1094 || lower_reason.contains("too many requests")
1095 || lower_reason.contains("abuse detection")
1096 || lower_reason.contains("failed (429)")
1097 || lower_reason.contains("auth error")
1098 {
1099 tracing::warn!(
1100 task_id,
1101 reason,
1102 "{context} hit rate limit or auth error — deferring retry until cooldown expires"
1103 );
1104 return recorded(
1105 store,
1106 repo,
1107 task_id,
1108 reason,
1109 "rate_limit",
1110 true,
1111 ReviewOutcome::RateLimited,
1112 )
1113 .await;
1114 }
1115
1116 if git_ops::is_transient_github_api_error(reason) {
1117 tracing::warn!(
1118 task_id,
1119 reason,
1120 "{context} hit transient GitHub error — deferring retry without counting as failure"
1121 );
1122 return reset_or_block(
1123 store,
1124 repo,
1125 task_id,
1126 reason,
1127 "github_api_error",
1128 true,
1129 context,
1130 )
1131 .await;
1132 }
1133
1134 if patterns::detect_network_error(reason).is_some() {
1138 tracing::warn!(
1139 task_id,
1140 reason,
1141 "{context} hit transient provider network error — re-routing for retry without counting as failure"
1142 );
1143 return reset_or_block(
1144 store,
1145 repo,
1146 task_id,
1147 reason,
1148 "provider_network_error",
1149 true,
1150 context,
1151 )
1152 .await;
1153 }
1154
1155 let failures = match store::store_increment(
1156 &Some(Arc::clone(store)),
1157 repo,
1158 task_id,
1159 "review_agent_failures",
1160 )
1161 .await
1162 {
1163 Ok(v) => v,
1164 Err(e) => {
1165 tracing::warn!(task_id, err = %e, "failed to increment review_agent_failures — skipping failure escalation this tick");
1166 0
1168 }
1169 };
1170 let blocking = failures >= MAX_REVIEW_AGENT_FAILURES;
1171 post_review_failure_comment(store, repo, task_id, reason, failures, blocking).await;
1173 if blocking {
1174 tracing::error!(
1175 task_id,
1176 reason,
1177 failures,
1178 "{context} failed too many times — blocking task"
1179 );
1180 recorded(
1181 store,
1182 repo,
1183 task_id,
1184 reason,
1185 "review_failure",
1186 false,
1187 ReviewOutcome::Block(format!("{context} failed {failures} times: {reason}")),
1188 )
1189 .await
1190 } else {
1191 tracing::error!(
1192 task_id,
1193 reason,
1194 failures,
1195 "{context} failed — resetting to NeedsReview for retry"
1196 );
1197 recorded(
1198 store,
1199 repo,
1200 task_id,
1201 reason,
1202 "review_failure",
1203 false,
1204 ReviewOutcome::Reset,
1205 )
1206 .await
1207 }
1208}
1209
1210async fn post_review_failure_comment(
1213 store: &Arc<TaskStore>,
1214 repo: &str,
1215 task_id: &str,
1216 reason: &str,
1217 failures: u64,
1218 blocking: bool,
1219) {
1220 let pr_number = match opt_store_get_task(&Some(Arc::clone(store)), repo, task_id).await {
1221 Some(t) => match t.pr_number {
1222 Some(n) => n,
1223 None => return, },
1225 _ => return, };
1227
1228 let status = if blocking {
1229 "Task blocked for human review."
1230 } else {
1231 "Retrying with a different agent."
1232 };
1233
1234 let comment = format!(
1235 "⚠️ Review agent failed (attempt {}/{})\n\n**Reason:** {}\n\n{}{}",
1236 failures,
1237 MAX_REVIEW_AGENT_FAILURES,
1238 reason,
1239 status,
1240 engine::orch_footer()
1241 );
1242
1243 let Ok(gh) = GhHttp::new() else { return };
1244 if let Err(e) = gh.add_comment(repo, &pr_number.to_string(), &comment).await {
1245 tracing::debug!(
1246 task_id,
1247 pr_number,
1248 error = %e,
1249 "failed to post review failure comment"
1250 );
1251 }
1252}
1253
1254#[cfg(test)]
1255mod tests {
1256 use super::*;
1257
1258 #[tokio::test]
1261 async fn classify_review_failure_incomplete_resets_without_counting() {
1262 let store = Arc::new(TaskStore::open_memory().await.unwrap());
1263 let repo = "owner/repo";
1264 let store_id = store
1265 .create_internal(
1266 repo,
1267 "Incomplete review test",
1268 "",
1269 "review",
1270 "review-3734",
1271 None,
1272 )
1273 .await
1274 .unwrap();
1275 let task_id = format!("internal:{store_id}");
1276
1277 for _ in 0..=MAX_REVIEW_AGENT_FAILURES {
1278 let outcome = classify_review_failure(
1279 &store,
1280 repo,
1281 &task_id,
1282 "incomplete: review session exited before producing a decision",
1283 "review agent",
1284 )
1285 .await;
1286 assert!(
1287 matches!(outcome, ReviewOutcome::Reset),
1288 "incomplete review failure must reset for retry, not block"
1289 );
1290 }
1291
1292 let task = opt_store_get_task(&Some(Arc::clone(&store)), repo, &task_id)
1293 .await
1294 .expect("task should exist");
1295 assert_eq!(
1296 task.review_agent_failures, 0,
1297 "incomplete failures must not count toward review_agent_failures"
1298 );
1299 }
1300
1301 #[tokio::test]
1306 async fn classify_review_failure_truncated_resets_without_counting() {
1307 let store = Arc::new(TaskStore::open_memory().await.unwrap());
1308 let repo = "owner/repo";
1309 let store_id = store
1310 .create_internal(
1311 repo,
1312 "Truncated review test",
1313 "",
1314 "review",
1315 "review-1",
1316 None,
1317 )
1318 .await
1319 .unwrap();
1320 let task_id = format!("internal:{store_id}");
1321
1322 for _ in 0..=MAX_REVIEW_AGENT_FAILURES {
1323 let outcome = classify_review_failure(
1324 &store,
1325 repo,
1326 &task_id,
1327 "truncated: review output was cut off before completion \
1328 (output/reasoning token budget exceeded)",
1329 "review agent",
1330 )
1331 .await;
1332 assert!(
1333 matches!(outcome, ReviewOutcome::Reset),
1334 "truncated review failure must reset for retry, not block"
1335 );
1336 }
1337
1338 let task = opt_store_get_task(&Some(Arc::clone(&store)), repo, &task_id)
1339 .await
1340 .expect("task should exist");
1341 assert_eq!(
1342 task.review_agent_failures, 0,
1343 "truncated failures must not count toward review_agent_failures"
1344 );
1345 }
1346
1347 #[tokio::test]
1353 async fn classify_review_failure_non_conflict_merge_failure_retries_merge() {
1354 let store = Arc::new(TaskStore::open_memory().await.unwrap());
1355 let repo = "owner/repo";
1356 let store_id = store
1357 .create_internal(repo, "Merge retry", "", "review", "review-3783", None)
1358 .await
1359 .unwrap();
1360 let task_id = format!("internal:{store_id}");
1361
1362 let error_messages = [
1363 "merge failed: GitHub REST API rate-limited, skipping call for 187s",
1364 "merge failed: GitHub PR merge failed (502 Bad Gateway)",
1365 "merge failed: required status check still pending",
1366 ];
1367
1368 for _ in 0..=MAX_REVIEW_AGENT_FAILURES {
1369 for msg in &error_messages {
1370 let outcome =
1371 classify_review_failure(&store, repo, &task_id, msg, "review agent").await;
1372 assert!(
1373 matches!(outcome, ReviewOutcome::RetryMerge),
1374 "non-conflict merge failure '{msg}' must retry the merge, not re-dispatch the agent"
1375 );
1376 }
1377 }
1378
1379 let task = opt_store_get_task(&Some(Arc::clone(&store)), repo, &task_id)
1380 .await
1381 .expect("task should exist");
1382 assert_eq!(
1383 task.review_agent_failures, 0,
1384 "a stuck merge is not a review failure"
1385 );
1386 }
1387
1388 #[tokio::test]
1390 async fn classify_review_failure_merge_conflict_still_reroutes() {
1391 let store = Arc::new(TaskStore::open_memory().await.unwrap());
1392 let repo = "owner/repo";
1393 let store_id = store
1394 .create_internal(repo, "Merge conflict", "", "review", "review-3783b", None)
1395 .await
1396 .unwrap();
1397 let task_id = format!("internal:{store_id}");
1398
1399 for msg in [
1400 "merge failed: PR is not mergeable",
1401 "merge conflict detected on PR #12",
1402 ] {
1403 let outcome =
1404 classify_review_failure(&store, repo, &task_id, msg, "review agent").await;
1405 assert!(
1406 matches!(outcome, ReviewOutcome::Reroute),
1407 "content conflict '{msg}' must go back to the task agent"
1408 );
1409 }
1410 }
1411
1412 #[tokio::test]
1416 async fn classify_review_failure_network_error_resets_without_counting() {
1417 let store = Arc::new(TaskStore::open_memory().await.unwrap());
1418 let repo = "owner/repo";
1419 let store_id = store
1420 .create_internal(
1421 repo,
1422 "Network error review test",
1423 "",
1424 "review",
1425 "review-2",
1426 None,
1427 )
1428 .await
1429 .unwrap();
1430 let task_id = format!("internal:{store_id}");
1431
1432 let error_messages = [
1433 "API Error: Unable to connect to API (ConnectionRefused)",
1434 "connection refused",
1435 "ECONNREFUSED",
1436 "Upstream idle timeout exceeded",
1437 "connect ETIMEDOUT",
1438 "network request failed",
1439 ];
1440
1441 for msg in &error_messages {
1442 store
1444 .set_fields(store_id, &[("review_resets", serde_json::json!(0))])
1445 .await
1446 .unwrap();
1447 let outcome =
1448 classify_review_failure(&store, repo, &task_id, msg, "review agent").await;
1449 assert!(
1450 matches!(outcome, ReviewOutcome::Reset),
1451 "network error '{msg}' must reset for retry, not block"
1452 );
1453 }
1454
1455 let task = opt_store_get_task(&Some(Arc::clone(&store)), repo, &task_id)
1456 .await
1457 .expect("task should exist");
1458 assert_eq!(
1459 task.review_agent_failures, 0,
1460 "network errors must not count toward review_agent_failures"
1461 );
1462 }
1463
1464 #[tokio::test]
1468 async fn classify_review_failure_missing_review_model_is_rate_limited() {
1469 let store = Arc::new(TaskStore::open_memory().await.unwrap());
1470 let repo = "owner/repo";
1471 let store_id = store
1472 .create_internal(
1473 repo,
1474 "Missing review model test",
1475 "",
1476 "review",
1477 "review-3",
1478 None,
1479 )
1480 .await
1481 .unwrap();
1482 let task_id = format!("internal:{store_id}");
1483
1484 let error_messages = [
1485 "no review agent with a configured review model is available for task internal:123",
1486 "review model missing for task internal:123 after agent selection — refusing to dispatch",
1487 ];
1488
1489 for _ in 0..=MAX_REVIEW_AGENT_FAILURES {
1490 for msg in &error_messages {
1491 let outcome =
1492 classify_review_failure(&store, repo, &task_id, msg, "review_and_merge").await;
1493 assert!(
1494 matches!(outcome, ReviewOutcome::RateLimited),
1495 "missing review model error '{msg}' must defer, not block"
1496 );
1497 }
1498 }
1499
1500 let task = opt_store_get_task(&Some(Arc::clone(&store)), repo, &task_id)
1501 .await
1502 .expect("task should exist");
1503 assert_eq!(
1504 task.review_agent_failures, 0,
1505 "missing review model errors must not count toward review_agent_failures"
1506 );
1507 }
1508
1509 #[tokio::test]
1511 async fn classify_review_failure_blocks_after_max_resets() {
1512 let store = Arc::new(TaskStore::open_memory().await.unwrap());
1513 let repo = "owner/repo";
1514 let store_id = store
1515 .create_internal(repo, "Looping review", "", "review", "review-resets", None)
1516 .await
1517 .unwrap();
1518 let task_id = format!("internal:{store_id}");
1519 let msg = "parse error: failed to parse review response";
1520
1521 for attempt in 1..MAX_REVIEW_RESETS {
1522 let outcome =
1523 classify_review_failure(&store, repo, &task_id, msg, "review agent").await;
1524 assert!(
1525 matches!(outcome, ReviewOutcome::Reset),
1526 "reset {attempt} must retry the review"
1527 );
1528 }
1529 let outcome = classify_review_failure(&store, repo, &task_id, msg, "review agent").await;
1530 assert!(
1531 matches!(outcome, ReviewOutcome::Block(ref r) if r.contains("without a decision")),
1532 "reset {MAX_REVIEW_RESETS} must block the task"
1533 );
1534
1535 let task = opt_store_get_task(&Some(Arc::clone(&store)), repo, &task_id)
1536 .await
1537 .expect("task should exist");
1538 assert_eq!(
1539 u64::try_from(task.review_resets).unwrap(),
1540 MAX_REVIEW_RESETS
1541 );
1542 assert_eq!(
1543 task.review_agent_failures, 0,
1544 "a reset must not count as a review failure"
1545 );
1546 }
1547
1548 #[tokio::test]
1554 async fn classify_review_failure_rate_limit_clears_refire_quota() {
1555 let store = Arc::new(TaskStore::open_memory().await.unwrap());
1556 let repo = "owner/repo";
1557 let store_id = store
1558 .create_internal(
1559 repo,
1560 "Rate limited review",
1561 "",
1562 "review",
1563 "review-3767",
1564 None,
1565 )
1566 .await
1567 .unwrap();
1568 let task_id = format!("internal:{store_id}");
1569
1570 let error_messages = [
1571 "claude failed: rate limit reached for model sonnet",
1572 "GitHub API GET https://api.github.com/repos/owner/repo/pulls/65 failed (403): API rate limit exceeded",
1573 "GitHub core API rate-limited, skipping call for 1800s",
1574 "GitHub API GET https://api.github.com/repos/owner/repo/pulls/65 failed (429): Too Many Requests",
1575 ];
1576
1577 store
1579 .set_fields(store_id, &[("needs_review_refires", serde_json::json!(5))])
1580 .await
1581 .unwrap();
1582
1583 for _ in 0..=MAX_REVIEW_AGENT_FAILURES {
1584 for msg in &error_messages {
1585 let outcome =
1586 classify_review_failure(&store, repo, &task_id, msg, "review agent").await;
1587 assert!(
1588 matches!(outcome, ReviewOutcome::RateLimited | ReviewOutcome::Reset),
1589 "rate limit '{msg}' must defer for retry, not block"
1590 );
1591 }
1592 }
1593
1594 let task = opt_store_get_task(&Some(Arc::clone(&store)), repo, &task_id)
1595 .await
1596 .expect("task should exist");
1597 assert_eq!(
1598 task.needs_review_refires, 0,
1599 "a rate limit must not consume the rebroadcast quota"
1600 );
1601 assert_eq!(
1602 task.review_agent_failures, 0,
1603 "rate limits must not count toward review_agent_failures"
1604 );
1605 assert_ne!(
1606 task.status,
1607 TaskStatus::Blocked,
1608 "a rate-limited review must leave the task retryable"
1609 );
1610
1611 let attempts = store
1612 .get_activity(store_id, None)
1613 .await
1614 .unwrap()
1615 .into_iter()
1616 .filter(|a| a.event_type == "review_attempt_failed")
1617 .collect::<Vec<_>>();
1618 assert_eq!(
1619 attempts.len(),
1620 (usize::try_from(MAX_REVIEW_AGENT_FAILURES).unwrap() + 1) * error_messages.len(),
1621 "every failed attempt must be recorded for `orch task inspect`"
1622 );
1623 assert!(
1624 attempts.iter().all(|a| a
1625 .details
1626 .get("external")
1627 .and_then(serde_json::Value::as_bool)
1628 == Some(true)),
1629 "rate limits must be recorded as an external cause"
1630 );
1631 assert!(
1632 attempts.iter().any(
1633 |a| a.details.get("reason").and_then(|v| v.as_str()) == Some(error_messages[0])
1634 ),
1635 "the recorded attempt must carry that attempt's error"
1636 );
1637 }
1638}