1pub(crate) const MAX_REVIEW_AGENT_FAILURES: u64 = 3;
13
14const MAX_PR_CREATE_FAILURES: u64 = 3;
16
17fn outcome_for_agent_error(err: &runner::agents::AgentError) -> &'static str {
22 match err {
23 runner::agents::AgentError::Timeout { .. } => "timeout",
24 runner::agents::AgentError::RateLimit { message } => {
29 crate::engine::cooldown::detect_credit_exhaustion_outcome(message)
30 .unwrap_or("rate_limit")
31 }
32 runner::agents::AgentError::Auth { message } => {
33 crate::engine::cooldown::detect_credit_exhaustion_outcome(message)
34 .unwrap_or("auth_error")
35 }
36 runner::agents::AgentError::InvalidResponse { .. } => "parse_error",
37 _ => "failed",
38 }
39}
40
41fn format_review_error(agent: &str, err: &runner::agents::AgentError) -> String {
50 match err {
51 runner::agents::AgentError::RateLimit { message } => format!(
52 "{agent} rate limit: {}",
53 runner::fallback::summarize_rate_limit_error(message)
54 ),
55 runner::agents::AgentError::Auth { message } => format!("{agent} auth error: {message}"),
56 runner::agents::AgentError::Timeout { elapsed } => {
57 format!("{agent} timed out after {}s", elapsed.as_secs())
58 }
59 other => format!("{agent} agent error: {other}"),
60 }
61}
62
63pub(crate) fn parse_pr_number_from_url(url: &str) -> Result<u64, String> {
75 let url = url.trim();
76 if url.is_empty() {
77 return Err("URL is empty".to_string());
78 }
79 if !url.starts_with("https://github.com/") {
80 return Err(format!(
81 "URL does not start with 'https://github.com/': {url}"
82 ));
83 }
84 if !url.contains("/pull/") {
85 return Err(format!("URL missing '/pull/' segment: {url}"));
86 }
87 let last_segment = url
88 .trim_end_matches('/')
89 .rsplit('/')
90 .next()
91 .ok_or_else(|| format!("No segment after '/pull/' in URL: {url}"))?;
92 let number_part = last_segment
93 .split(['?', '#'])
94 .next()
95 .filter(|s| !s.is_empty())
96 .ok_or_else(|| format!("PR number segment is empty in URL: {url}"))?;
97 let pr_num: u64 = number_part.parse().map_err(|_| {
98 format!("PR number is not a valid positive integer: '{number_part}' in URL: {url}")
99 })?;
100 if pr_num == 0 {
101 return Err(format!("PR number cannot be zero in URL: {url}"));
102 }
103 Ok(pr_num)
104}
105
106fn review_started_comment(review_agent: &str, review_model: &str) -> String {
107 let footer = attribution_footer("Review started", review_agent, Some(review_model));
108 format!("🔍 Automated review started{}", footer)
109}
110
111fn should_skip_no_code_reroute_increment(last_error: &str) -> bool {
112 crate::engine::runner::git_ops::is_transient_github_api_error(last_error)
113}
114
115use crate::backends::{ExternalBackend, ExternalTask, Status};
116use crate::cmd::CommandErrorContext;
117use crate::config;
118use crate::engine::attribution_footer;
119use crate::engine::auto_merge::{auto_merge_pr, handle_review_changes};
120use crate::engine::runner;
121use crate::engine::runner::worktree;
122use crate::engine::tasks::TaskManager;
123use crate::github::http::GhHttp;
124use crate::store::store_log_activity;
125use crate::store::TaskStore;
126use crate::store::{opt_store_get_task, set_review_session_expected, store_increment};
127use crate::store::{store_set, store_set_result};
128use crate::store::{CompleteRun, RunTokenUsage, StartRun};
129use crate::tmux::TmuxManager;
130use anyhow::Context;
131use std::collections::HashSet;
132use std::sync::Arc;
133use tokio::process::Command;
134use tokio::sync::RwLock;
135
136use super::router::Router;
137
138#[derive(Debug, Clone)]
140pub(crate) enum ReviewDecision {
141 Approve,
143 RequestChanges {
145 notes: String,
146 issues: Vec<crate::engine::runner::response::ReviewIssue>,
147 },
148 Failed(String),
150 Blocked(String),
152 Skipped,
154 Rerouted,
156}
157
158enum EnsurePrResult {
160 Ready(u64),
162 EarlyReturn(ReviewDecision),
164}
165
166enum ReviewPhase<T> {
167 Ready(T),
168 EarlyReturn(ReviewDecision),
169}
170
171struct ReviewContext {
172 store_id: Option<i64>,
173 had_prev_push_failure: bool,
174 review_attempt: u32,
175 worktree_path: std::path::PathBuf,
176 branch_name: String,
177 pr_number: u64,
178 default_branch: String,
179 review_agent: String,
180 review_model: Option<String>,
181 review_task_id: String,
182 review_attempt_dir: std::path::PathBuf,
183 output_file: std::path::PathBuf,
184 invocation: runner::agent::AgentInvocation,
185 same_agent_fallback: bool,
186}
187
188impl ReviewContext {
189 fn review_model_str(&self) -> &str {
190 self.review_model.as_deref().unwrap_or("")
191 }
192}
193
194struct ReviewRun {
195 run_id: Option<i64>,
196}
197
198struct ParsedReview {
199 run_id: Option<i64>,
200 exit_code: i32,
201 stderr: String,
202 raw_output: String,
203 text_for_review: String,
204 token_usage: RunTokenUsage,
205 review_notes_for_comment: String,
206 decision: ReviewDecision,
207}
208
209fn build_review_disallowed_tools(worktree_path: &std::path::Path) -> Vec<String> {
210 let mut tools = vec![
211 "Bash(rm *)".to_string(),
212 "Bash(rm -*)".to_string(),
213 "Bash(git push*)".to_string(),
214 ];
215 let orch_yml = worktree_path.join(".orch.yml");
216 let orch_yml_str = orch_yml.to_string_lossy();
217 tools.extend([
218 format!("Write({orch_yml_str})"),
219 format!("Edit({orch_yml_str})"),
220 ]);
221 if let Ok(orch_home) = crate::home::orch_home() {
222 for path in [
223 orch_home.join("config.yml"),
224 orch_home.join("config.example.yml"),
225 ] {
226 let path_str = path.to_string_lossy();
227 tools.extend([
228 format!("Read({path_str})"),
229 format!("Write({path_str})"),
230 format!("Edit({path_str})"),
231 ]);
232 }
233 }
234 tools
235}
236
237async fn select_review_agent(
238 task_id: &str,
239 task_agent: &str,
240 store_id: Option<i64>,
241 router: &Arc<RwLock<Router>>,
242 store: &Arc<TaskStore>,
243) -> anyhow::Result<(String, Option<String>, bool)> {
244 let mut exclude_set: HashSet<String> = HashSet::new();
245 if !task_agent.is_empty() {
246 exclude_set.insert(task_agent.to_string());
247 }
248 if let Some(store_id) = store_id {
249 if let Ok(failed) = store.previous_review_agents(store_id).await {
250 for agent in failed {
251 exclude_set.insert(agent);
252 }
253 }
254 }
255
256 let mut r = router.write().await;
257 let mut tried_agents: HashSet<String> = HashSet::new();
258 let mut chosen_agent: Option<String> = None;
259 let mut chosen_model: Option<String> = None;
260 let available_count = r.available_agents.len().max(1);
261
262 loop {
263 let tmp_exclude_refs: Vec<&str> = exclude_set.iter().map(|s| s.as_str()).collect();
264 let agent = match r.next_round_robin_agent(&tmp_exclude_refs, "review") {
265 Some(a) => a,
266 None => break,
267 };
268
269 if tried_agents.contains(&agent) {
270 break;
271 }
272 tried_agents.insert(agent.clone());
273
274 let model = r.config.model_for_complexity(&agent, "review", task_id);
275 if model.is_some() {
276 chosen_agent = Some(agent);
277 chosen_model = model;
278 break;
279 }
280
281 exclude_set.insert(agent);
282 if exclude_set.len() >= available_count {
283 break;
284 }
285 }
286
287 if let Some(agent) = chosen_agent {
288 let same = !task_agent.is_empty() && agent == task_agent;
289 Ok((agent, chosen_model, same))
290 } else {
291 let final_exclude_refs: Vec<&str> = exclude_set.iter().map(|s| s.as_str()).collect();
292 let fallback_agent = r
293 .next_round_robin_agent(&final_exclude_refs, "review")
294 .unwrap_or_else(|| "claude".to_string());
295 let fallback_model = r
296 .config
297 .model_for_complexity(&fallback_agent, "review", task_id);
298 if fallback_model.is_none() {
299 anyhow::bail!(
300 "no review agent with a configured review model is available for task {}",
301 task_id
302 );
303 }
304 let same = !task_agent.is_empty() && fallback_agent == task_agent;
305 Ok((fallback_agent, fallback_model, same))
306 }
307}
308
309#[allow(clippy::too_many_arguments)]
310async fn build_review_context(
311 task: &ExternalTask,
312 repo: &str,
313 router: &Arc<RwLock<Router>>,
314 task_manager: &Arc<TaskManager>,
315 store: &Arc<TaskStore>,
316) -> anyhow::Result<ReviewPhase<ReviewContext>> {
317 let stored_task = store
318 .get_by_external_id(repo, &task.id.0)
319 .await
320 .with_context(|| format!("store lookup failed for task {}", task.id.0))?;
321
322 let store_id: Option<i64> = stored_task.as_ref().map(|t| t.id);
323 let had_prev_push_failure = stored_task
324 .as_ref()
325 .is_some_and(|t| t.last_error.contains("push failed"));
326 let review_attempt: u32 = match store_increment(
327 &Some(Arc::clone(store)),
328 repo,
329 &task.id.0,
330 "review_invocations",
331 )
332 .await
333 {
334 Ok(v) => v as u32,
335 Err(e) => {
336 tracing::warn!(task_id = task.id.0, err = %e, "failed to increment review_invocations — using fallback attempt=1");
337 1
338 }
339 };
340
341 let no_code_review = stored_task.as_ref().is_some_and(|t| {
342 t.worktree.trim().is_empty() && t.branch.trim().is_empty() && t.pr_number.is_none()
343 });
344 if no_code_review {
345 if crate::engine::tasks::is_internal_id(&task.id.0) {
346 tracing::info!(
348 task_id = task.id.0,
349 "no worktree/branch/pr for internal needs_review task — skipping review and marking done"
350 );
351 if let Err(e) = task_manager
352 .update_task_status(&task.id, Status::Done)
353 .await
354 {
355 tracing::error!(
356 task_id = task.id.0,
357 err = %e,
358 "update_task_status(Done) failed for no-code review skip"
359 );
360 }
361 return Ok(ReviewPhase::EarlyReturn(ReviewDecision::Skipped));
362 }
363
364 let max_reroutes: u32 = config::get("workflow.max_reroute_attempts")
367 .ok()
368 .and_then(|s| s.parse().ok())
369 .or_else(|| {
370 config::get("workflow.max_attempts")
371 .ok()
372 .and_then(|s| s.parse().ok())
373 })
374 .unwrap_or(3);
375
376 let reroutes = match store_increment(
377 &Some(Arc::clone(store)),
378 repo,
379 &task.id.0,
380 "no_code_reroutes",
381 )
382 .await
383 {
384 Ok(v) => v,
385 Err(e) => {
386 tracing::warn!(task_id = task.id.0, err = %e, "failed to increment no_code_reroutes — skipping reroute/block decision this tick");
387 return Ok(ReviewPhase::EarlyReturn(ReviewDecision::Failed(format!(
388 "transient store error: {e}"
389 ))));
390 }
391 };
392
393 if reroutes as u32 >= max_reroutes {
394 tracing::error!(
395 task_id = task.id.0,
396 reroutes,
397 max_reroutes,
398 "reached max reroute attempts for no-worktree/branch/pr result — blocking for human review"
399 );
400 let msg = format!(
401 "no worktree/branch/pr after {}/{} reroute attempts",
402 reroutes, max_reroutes
403 );
404 let fields = [
405 (
406 "block_reason",
407 serde_json::json!(format!(
408 "max reroute attempts ({}) reached — no worktree/branch/pr produced",
409 max_reroutes
410 )),
411 ),
412 ("last_error", serde_json::json!(msg)),
413 ("agent", serde_json::json!(null)),
414 ("model", serde_json::json!(null)),
415 ];
416 if let Err(e) = task_manager
417 .update_task_status_and_result(&task.id, Status::Blocked, &fields)
418 .await
419 {
420 tracing::error!(task_id = task.id.0, err = %e, "update_task_status_and_result(Blocked) failed — skipping block to avoid silent auto-unblock loop");
421 }
422 return Ok(ReviewPhase::EarlyReturn(ReviewDecision::Blocked(msg)));
423 }
424
425 tracing::warn!(
426 task_id = task.id.0,
427 reroutes,
428 max_reroutes,
429 "no worktree/branch/pr for external needs_review task — re-routing to a different agent"
430 );
431 if let Err(e) = store_set_result(
432 &Some(Arc::clone(store)),
433 repo,
434 &task.id.0,
435 &[
436 ("agent", serde_json::json!(null)),
437 ("model", serde_json::json!(null)),
438 (
439 "last_error",
440 serde_json::json!("no worktree/branch/pr produced"),
441 ),
442 ],
443 )
444 .await
445 {
446 tracing::error!(task_id = task.id.0, err = %e, "failed to clear agent/model for reroute — skipping status transition");
447 } else if let Err(e) = task_manager.update_task_status(&task.id, Status::New).await {
448 tracing::error!(task_id = task.id.0, err = %e, "update_task_status(New) failed — task may be stuck in InReview");
449 }
450 return Ok(ReviewPhase::EarlyReturn(ReviewDecision::Rerouted));
451 }
452
453 let mut worktree_path = match stored_task.as_ref().map(|t| t.worktree.as_str()) {
454 Some(w) if !w.is_empty() => std::path::PathBuf::from(w),
455 _ => {
456 tracing::warn!(task_id = task.id.0, "no worktree found for review");
457 return Ok(ReviewPhase::EarlyReturn(ReviewDecision::Failed(
458 "no worktree found".to_string(),
459 )));
460 }
461 };
462
463 let mut branch_name = match stored_task.as_ref().map(|t| t.branch.as_str()) {
464 Some(b) if !b.is_empty() => b.to_string(),
465 _ => {
466 tracing::warn!(task_id = task.id.0, "no branch found for review");
467 return Ok(ReviewPhase::EarlyReturn(ReviewDecision::Failed(
468 "no branch found".to_string(),
469 )));
470 }
471 };
472
473 let mut missing_worktree_recovered = false;
474 if !worktree_path.exists() {
475 match recover_missing_review_worktree(task, repo, store).await {
476 Ok(recovered) => {
477 branch_name = recovered.branch;
478 worktree_path = recovered.work_dir;
479 missing_worktree_recovered = true;
480 }
481 Err(e) => {
482 let reason = format!("missing review worktree recovery failed: {e}");
483 tracing::error!(task_id = task.id.0, error = %reason, "cannot recover review worktree");
484 store_set(
485 &Some(Arc::clone(store)),
486 repo,
487 &task.id.0,
488 &[("last_error", serde_json::json!(reason.clone()))],
489 )
490 .await;
491 return Ok(ReviewPhase::EarlyReturn(ReviewDecision::Blocked(reason)));
492 }
493 }
494 }
495
496 if missing_worktree_recovered {
497 store_set(
498 &Some(Arc::clone(store)),
499 repo,
500 &task.id.0,
501 &[("last_error", serde_json::json!(""))],
502 )
503 .await;
504 }
505
506 let agent_summary = stored_task
507 .as_ref()
508 .map(|t| t.summary.clone())
509 .unwrap_or_default();
510 let stored_pr_number = stored_task
511 .as_ref()
512 .and_then(|t| t.pr_number)
513 .map(|n| n as u64)
514 .filter(|&n| n > 0);
515
516 let pr_number = match ensure_pr_exists(
517 task,
518 &branch_name,
519 &worktree_path,
520 repo,
521 &agent_summary,
522 store,
523 task_manager,
524 stored_pr_number,
525 )
526 .await?
527 {
528 EnsurePrResult::Ready(n) => n,
529 EnsurePrResult::EarlyReturn(decision) => return Ok(ReviewPhase::EarlyReturn(decision)),
530 };
531
532 {
536 let fetch_timeout = std::time::Duration::from_secs(60);
537 let fetch_future = async {
538 let mut cmd = Command::new("git");
539 cmd.args(["fetch", "origin", "--prune"])
540 .current_dir(&worktree_path);
541 cmd.kill_on_drop(true);
543 cmd.output_with_context().await
544 };
545
546 match tokio::time::timeout(fetch_timeout, fetch_future).await {
547 Ok(Ok(o)) if o.status.success() => {
548 }
550 Ok(Ok(o)) => {
551 tracing::warn!(
552 task_id = task.id.0,
553 exit_code = ?o.status.code(),
554 stdout = %String::from_utf8_lossy(&o.stdout),
555 stderr = %String::from_utf8_lossy(&o.stderr),
556 "review: git fetch failed (non-zero exit) — diff/log may use stale remote refs"
557 );
558 }
559 Ok(Err(e)) => {
560 tracing::warn!(
561 task_id = task.id.0,
562 error = %e,
563 "review: git fetch failed — diff/log may use stale remote refs"
564 );
565 }
566 Err(_) => {
567 tracing::warn!(
568 task_id = task.id.0,
569 "review: git fetch timed out after 60s — diff/log may use stale remote refs"
570 );
571 }
572 }
573 }
574
575 let default_branch = runner::worktree::detect_default_branch(&worktree_path).await;
576 let git_diff = runner::context::build_git_diff(&worktree_path, &default_branch).await;
577 let git_log = runner::context::build_git_log(&worktree_path, &default_branch).await;
578 let review_prompt = runner::agent::build_review_prompt(
579 task,
580 &agent_summary,
581 &git_diff,
582 &git_log,
583 &default_branch,
584 pr_number,
585 runner::agent::Shipcheck::detect(&worktree_path).await,
586 );
587
588 let task_agent = stored_task
589 .as_ref()
590 .and_then(|t| t.agent.clone())
591 .unwrap_or_default();
592 let (review_agent, review_model, same_agent_fallback) =
593 select_review_agent(&task.id.0, &task_agent, store_id, router, store).await?;
594
595 anyhow::ensure!(
596 review_model.is_some(),
597 "review model missing for task {} after agent selection — refusing to dispatch",
598 task.id.0
599 );
600
601 if same_agent_fallback {
602 tracing::warn!(
603 task_id = task.id.0,
604 agent = %review_agent,
605 "no other agent available for review — falling back to the agent that wrote the code"
606 );
607 store_log_activity(
608 &Some(Arc::clone(store)),
609 repo,
610 &task.id.0,
611 "review_agent_fallback",
612 None,
613 None,
614 Some(&review_agent),
615 review_model.as_deref(),
616 Some(&serde_json::json!({
617 "reason": "all available agents excluded (author + previous reviewers)",
618 })),
619 )
620 .await;
621 }
622
623 tracing::info!(
624 task_id = task.id.0,
625 agent = %review_agent,
626 model = ?review_model,
627 "spawning review agent"
628 );
629
630 let review_task_id = format!("{}-review", task.id.0);
631 let review_attempt_dir =
632 crate::home::task_attempt_dir_async(repo, &review_task_id, review_attempt).await?;
633 let output_file = review_attempt_dir.join("output.json");
634
635 let git_name =
636 crate::config::get("git.name").unwrap_or_else(|_| format!("{review_agent}[bot]"));
637 let git_email = crate::config::get("git.email")
638 .unwrap_or_else(|_| format!("{review_agent}[bot]@users.noreply.github.com"));
639 let review_timeout_secs: u64 = crate::config::get("workflow.review_timeout_seconds")
640 .ok()
641 .and_then(|s| s.parse().ok())
642 .unwrap_or_else(|| {
643 crate::config::get("workflow.timeout_seconds")
644 .ok()
645 .and_then(|s| s.parse().ok())
646 .unwrap_or(1800)
647 })
648 .max(1800);
649
650 let invocation = runner::agent::AgentInvocation {
651 agent: review_agent.clone(),
652 model: review_model.clone(),
653 work_dir: worktree_path.clone(),
654 system_prompt: runner::agent::review_system_prompt(),
655 agent_message: review_prompt,
656 task_id: review_task_id.clone(),
657 disallowed_tools: build_review_disallowed_tools(&worktree_path),
658 git_author_name: git_name,
659 git_author_email: git_email,
660 output_file: output_file.clone(),
661 timeout_seconds: review_timeout_secs,
662 repo: repo.to_string(),
663 attempt: review_attempt,
664 };
665
666 Ok(ReviewPhase::Ready(ReviewContext {
667 store_id,
668 had_prev_push_failure,
669 review_attempt,
670 worktree_path,
671 branch_name,
672 pr_number,
673 default_branch,
674 review_agent,
675 review_model,
676 review_task_id,
677 review_attempt_dir,
678 output_file,
679 invocation,
680 same_agent_fallback,
681 }))
682}
683
684#[allow(clippy::too_many_arguments)]
685async fn complete_review_run(
686 store: &Arc<TaskStore>,
687 run_id: Option<i64>,
688 exit_code: Option<i32>,
689 stdout: &str,
690 stderr: &str,
691 parsed: &str,
692 outcome: &str,
693 error: &str,
694 tokens: RunTokenUsage,
695) {
696 if let Some(run_id) = run_id {
697 if let Err(e) = store
698 .complete_run(&CompleteRun {
699 run_id,
700 exit_code,
701 stdout,
702 stderr,
703 parsed,
704 outcome,
705 error,
706 tokens,
707 })
708 .await
709 {
710 tracing::warn!(
711 run_id,
712 error = %e,
713 "failed to record review run completion in audit trail"
714 );
715 }
716 }
717}
718
719async fn invoke_review_agent(
720 task: &ExternalTask,
721 tmux: &Arc<TmuxManager>,
722 ctx: &ReviewContext,
723 repo: &str,
724 store: &Arc<TaskStore>,
725) -> ReviewPhase<ReviewRun> {
726 let run_id = if let Some(sid) = ctx.store_id {
727 match store
728 .start_run(&StartRun {
729 task_id: sid,
730 attempt: ctx.review_attempt as i32,
731 run_type: "review",
732 agent: &ctx.review_agent,
733 model: ctx.review_model.as_deref().unwrap_or(""),
734 command: "",
735 prompt: "",
736 })
737 .await
738 {
739 Ok(run_id) => Some(run_id),
740 Err(e) => {
741 tracing::warn!(
742 task_id = task.id.0,
743 error = %e,
744 "failed to record review run start in audit trail"
745 );
746 None
747 }
748 }
749 } else {
750 None
751 };
752
753 let session = match runner::agent::spawn_in_tmux(tmux, &ctx.invocation).await {
754 Ok(session) => session,
755 Err(e) => {
756 tracing::error!(task_id = task.id.0, error = %e, "failed to spawn review agent");
757 complete_review_run(
758 store,
759 run_id,
760 None,
761 "",
762 &e.to_string(),
763 "",
764 "failed",
765 &format!("spawn failed: {e}"),
766 RunTokenUsage::default(),
767 )
768 .await;
769 return ReviewPhase::EarlyReturn(ReviewDecision::Failed(format!("spawn failed: {e}")));
770 }
771 };
772
773 let start_comment = review_started_comment(&ctx.review_agent, ctx.review_model_str());
774 match GhHttp::new() {
775 Ok(gh) => {
776 if let Err(e) = gh
777 .add_comment(repo, &ctx.pr_number.to_string(), &start_comment)
778 .await
779 {
780 tracing::debug!(
781 task_id = task.id.0,
782 pr_number = ctx.pr_number,
783 error = %e,
784 "failed to post review-started comment"
785 );
786 }
787 }
788 Err(e) => {
789 tracing::debug!(
790 task_id = task.id.0,
791 pr_number = ctx.pr_number,
792 error = %e,
793 "failed to create GitHub client for review-started comment"
794 );
795 }
796 }
797
798 let poll_interval = std::time::Duration::from_secs(5);
799 let timeout_duration = std::time::Duration::from_secs(ctx.invocation.timeout_seconds);
800 let wait_result = tokio::time::timeout(
801 timeout_duration,
802 tmux.wait_for_completion(&session, poll_interval),
803 )
804 .await;
805
806 match wait_result {
807 Ok(Ok(_)) => {
808 tracing::info!(task_id = task.id.0, "review agent completed");
809 let _ = tmux.kill_session(&session).await;
810 ReviewPhase::Ready(ReviewRun { run_id })
811 }
812 Ok(Err(e)) => {
813 tracing::warn!(task_id = task.id.0, error = %e, "review agent process exited with error");
814 let _ = tmux.kill_session(&session).await;
815
816 let exit_code: i32 = tokio::task::spawn_blocking({
821 let review_attempt_dir = ctx.review_attempt_dir.clone();
822 move || {
823 std::fs::read_to_string(review_attempt_dir.join("exit.txt"))
824 .ok()
825 .and_then(|s| s.trim().parse().ok())
826 .unwrap_or(-1)
827 }
828 })
829 .await
830 .unwrap_or(-1);
831
832 if exit_code != 0 {
833 let raw_output =
838 runner::response::read_output_file(&ctx.review_task_id, &ctx.output_file, repo)
839 .await;
840 let agent_result =
841 runner::agents::find_agent_result(&ctx.review_agent, &raw_output);
842 if agent_result.as_ref().map(|r| !r.is_error).unwrap_or(false) {
844 tracing::info!(
845 task_id = task.id.0,
846 agent = %ctx.review_agent,
847 exit_code,
848 output_found = !raw_output.is_empty(),
849 "review agent succeeded despite non-zero exit — proceeding to parse output"
850 );
851 return ReviewPhase::Ready(ReviewRun { run_id });
852 }
853 }
854
855 tracing::error!(task_id = task.id.0, error = %e, "review agent failed");
856 complete_review_run(
857 store,
858 run_id,
859 Some(exit_code),
860 "",
861 &e.to_string(),
862 "",
863 "failed",
864 &format!("agent error: {e}"),
865 RunTokenUsage::default(),
866 )
867 .await;
868 ReviewPhase::EarlyReturn(ReviewDecision::Failed(format!("agent error: {e}")))
869 }
870 Err(_) => {
871 tracing::error!(task_id = task.id.0, "review agent timed out");
872 let _ = tmux.kill_session(&session).await;
873 if let Some(model) = ctx.review_model.as_deref() {
874 crate::engine::cooldown::record_persistent_model_failure(&ctx.review_agent, model)
879 .await;
880 }
881 complete_review_run(
882 store,
883 run_id,
884 Some(-1),
885 "",
886 "timeout",
887 "",
888 "timeout",
889 "timeout",
890 RunTokenUsage::default(),
891 )
892 .await;
893 ReviewPhase::EarlyReturn(ReviewDecision::Failed("timeout".to_string()))
894 }
895 }
896}
897
898async fn record_review_agent_failure(
899 task_id: &str,
900 review_agent: &str,
901 review_model: Option<&str>,
902 err: &runner::agents::AgentError,
903) {
904 match err {
905 runner::agents::AgentError::RateLimit { message }
906 | runner::agents::AgentError::Auth { message } => {
907 tracing::warn!(
908 task_id,
909 agent = %review_agent,
910 "review agent hit error — adding to cooldown"
911 );
912 if let Some(reason) = crate::engine::cooldown::detect_credit_exhaustion(message) {
913 if reason == crate::engine::cooldown::CreditExhaustionReason::BillingCycleExhausted
914 {
915 if let Some(model) = review_model {
916 crate::engine::cooldown::record_persistent_model_failure(
918 review_agent,
919 model,
920 )
921 .await;
922 } else {
923 crate::engine::cooldown::record_credit_exhaustion(review_agent, reason)
924 .await;
925 }
926 } else {
927 crate::engine::cooldown::record_credit_exhaustion(review_agent, reason).await;
928 }
929 } else {
930 runner::response::record_agent_failure_with_message(review_agent, &err.to_string())
931 .await;
932 }
933 }
934 runner::agents::AgentError::InvalidResponse { .. } => {
935 if let Some(model) = review_model {
940 crate::engine::cooldown::record_persistent_model_failure(review_agent, model).await;
941 }
942 }
943 runner::agents::AgentError::ModelUnavailable { .. } => {
944 if let Some(model) = review_model {
949 crate::engine::cooldown::record_persistent_model_failure(review_agent, model).await;
950 }
951 }
952 _ => {
953 if let Some(model) = review_model {
956 crate::engine::cooldown::record_model_failure(review_agent, model).await;
957 }
958 }
959 }
960}
961
962fn review_decision_from_response(
963 review_response: runner::response::ReviewResponse,
964) -> ReviewDecision {
965 match review_response.decision.as_str() {
966 "approve" => ReviewDecision::Approve,
967 "request_changes" => ReviewDecision::RequestChanges {
968 notes: review_response.notes,
969 issues: review_response.issues,
970 },
971 _ => ReviewDecision::Failed(format!("unknown decision: {}", review_response.decision)),
972 }
973}
974
975fn review_decision_name(decision: &ReviewDecision) -> &'static str {
976 match decision {
977 ReviewDecision::Approve => "approve",
978 ReviewDecision::RequestChanges { .. } => "request_changes",
979 ReviewDecision::Failed(_) => "failed",
980 ReviewDecision::Blocked(_) => "blocked",
981 ReviewDecision::Skipped => "skipped",
982 ReviewDecision::Rerouted => "rerouted",
983 }
984}
985
986async fn parse_review_output(
987 task: &ExternalTask,
988 repo: &str,
989 ctx: &ReviewContext,
990 run: ReviewRun,
991 store: &Arc<TaskStore>,
992) -> ReviewPhase<ParsedReview> {
993 let (file_exists, file_size, exit_code, stderr) = {
994 let output_file_clone = ctx.output_file.clone();
995 let review_attempt_dir_clone = ctx.review_attempt_dir.clone();
996 match tokio::task::spawn_blocking(move || {
997 let file_exists = output_file_clone.exists();
998 let file_size = std::fs::metadata(&output_file_clone)
999 .map(|m| m.len())
1000 .unwrap_or(0);
1001 let exit_code = std::fs::read_to_string(review_attempt_dir_clone.join("exit.txt"))
1002 .ok()
1003 .and_then(|s| s.trim().parse::<i32>().ok())
1004 .unwrap_or(-1);
1005 let stderr = std::fs::read_to_string(review_attempt_dir_clone.join("stderr.txt"))
1006 .unwrap_or_default();
1007 (file_exists, file_size, exit_code, stderr)
1008 })
1009 .await
1010 {
1011 Ok(tuple) => tuple,
1012 Err(e) => {
1013 tracing::error!(
1014 task_id = task.id.0,
1015 error = %e,
1016 "spawn_blocking panicked reading review output metadata"
1017 );
1018 (false, 0, -1, format!("spawn_blocking failed: {e}"))
1019 }
1020 }
1021 };
1022
1023 tracing::info!(
1024 task_id = task.id.0,
1025 path = %ctx.output_file.display(),
1026 file_exists,
1027 file_size,
1028 "reading review agent output"
1029 );
1030
1031 let raw_output =
1032 runner::response::read_output_file(&ctx.review_task_id, &ctx.output_file, repo).await;
1033 let raw_output = runner::agents::claude::strip_tty_control_lines(&raw_output);
1034 let has_result_envelope =
1035 runner::agents::claude::find_ndjson_result_line(&raw_output).is_some();
1036 let agent_result = runner::agents::find_agent_result(&ctx.review_agent, &raw_output);
1037 let token_usage = RunTokenUsage {
1038 input_tokens: agent_result
1039 .as_ref()
1040 .and_then(|r| r.input_tokens)
1041 .unwrap_or(0),
1042 output_tokens: agent_result
1043 .as_ref()
1044 .and_then(|r| r.output_tokens)
1045 .unwrap_or(0),
1046 total_cost_usd: agent_result
1047 .as_ref()
1048 .and_then(|r| r.cost_usd)
1049 .unwrap_or(0.0),
1050 duration_secs: 0.0,
1051 };
1052
1053 let agent_result_is_error = agent_result.as_ref().map(|r| r.is_error).unwrap_or(false);
1054 let ndjson_completed_no_error = raw_output.contains("\"terminal_reason\":\"completed\"")
1055 && !raw_output.contains("\"is_error\":true");
1056 let is_hard_failure =
1058 !ndjson_completed_no_error && (raw_output.is_empty() || agent_result_is_error);
1059
1060 if is_hard_failure {
1061 tracing::warn!(
1062 task_id = task.id.0,
1063 exit_code,
1064 raw_output_len = raw_output.len(),
1065 stderr_len = stderr.len(),
1066 "review agent: entering error path"
1067 );
1068 let agent_runner = runner::agents::get_runner(&ctx.review_agent);
1069 let err = agent_runner.classify_error(exit_code, &raw_output, &stderr);
1070 record_review_agent_failure(
1071 &task.id.0,
1072 &ctx.review_agent,
1073 ctx.review_model.as_deref(),
1074 &err,
1075 )
1076 .await;
1077 tracing::error!(task_id = task.id.0, error = %err, "review agent failed");
1078 let stored_error = format_review_error(&ctx.review_agent, &err);
1079 complete_review_run(
1080 store,
1081 run.run_id,
1082 Some(exit_code),
1083 &raw_output,
1084 &stderr,
1085 "",
1086 outcome_for_agent_error(&err),
1087 &stored_error,
1088 token_usage,
1089 )
1090 .await;
1091 return ReviewPhase::EarlyReturn(ReviewDecision::Failed(stored_error));
1092 }
1093
1094 let text_for_review = agent_result
1095 .as_ref()
1096 .map(|r| r.result_text.clone())
1097 .filter(|text| !text.is_empty())
1098 .unwrap_or_else(|| {
1099 tracing::debug!(
1100 task_id = task.id.0,
1101 agent = %ctx.review_agent,
1102 "review agent: per-agent extractor returned no text, using raw output"
1103 );
1104 raw_output.clone()
1105 });
1106
1107 let review_response = match runner::response::parse_review_response(&text_for_review) {
1108 Ok(response) => response,
1109 Err(e) => {
1110 if let Some(response) = runner::response::infer_review_response(&text_for_review) {
1111 tracing::warn!(
1112 task_id = task.id.0,
1113 agent = %ctx.review_agent,
1114 "review response parsed via keyword fallback"
1115 );
1116 response
1117 } else {
1118 let ndjson_completed = ndjson_completed_no_error;
1124
1125 if ndjson_completed || !has_result_envelope {
1128 let assistant_text = runner::agents::collect_assistant_messages_text(
1129 &ctx.review_agent,
1130 &raw_output,
1131 );
1132 if !assistant_text.is_empty() {
1133 if let Ok(response) =
1134 runner::response::parse_review_response(&assistant_text)
1135 {
1136 tracing::debug!(
1137 task_id = task.id.0,
1138 agent = %ctx.review_agent,
1139 "found review response JSON in assistant message (terminal_reason:completed)"
1140 );
1141 let review_notes_for_comment = response.notes.clone();
1142 let decision = review_decision_from_response(response);
1143 return ReviewPhase::Ready(ParsedReview {
1144 run_id: run.run_id,
1145 exit_code,
1146 stderr,
1147 raw_output,
1148 text_for_review: assistant_text,
1149 token_usage,
1150 review_notes_for_comment,
1151 decision,
1152 });
1153 }
1154 if let Some(response) =
1155 runner::response::infer_review_response(&assistant_text)
1156 {
1157 tracing::warn!(
1158 task_id = task.id.0,
1159 agent = %ctx.review_agent,
1160 "inferred review response from assistant message (terminal_reason:completed)"
1161 );
1162 let review_notes_for_comment = response.notes.clone();
1163 let decision = review_decision_from_response(response);
1164 return ReviewPhase::Ready(ParsedReview {
1165 run_id: run.run_id,
1166 exit_code,
1167 stderr,
1168 raw_output,
1169 text_for_review: assistant_text,
1170 token_usage,
1171 review_notes_for_comment,
1172 decision,
1173 });
1174 }
1175 }
1176 }
1177
1178 let scan_head = runner::agents::patterns::safe_head(
1185 &text_for_review,
1186 runner::agents::patterns::RATE_LIMIT_SCAN_HEAD_BYTES,
1187 );
1188 let scan_tail = runner::agents::patterns::safe_tail(
1189 &text_for_review,
1190 runner::agents::patterns::RATE_LIMIT_SCAN_TAIL_BYTES,
1191 );
1192
1193 if let Some(err) = (!ndjson_completed)
1194 .then(|| {
1195 runner::agents::patterns::detect_rate_limit(scan_head)
1196 .or_else(|| runner::agents::patterns::detect_rate_limit(scan_tail))
1197 })
1198 .flatten()
1199 {
1200 let runner::agents::AgentError::RateLimit { message } = &err else {
1201 unreachable!("detect_rate_limit only returns AgentError::RateLimit")
1202 };
1203 tracing::warn!(
1204 task_id = task.id.0,
1205 agent = %ctx.review_agent,
1206 "review agent hit rate limit — adding to cooldown"
1207 );
1208 let summary = runner::fallback::summarize_rate_limit_error(message);
1215 let stored_error = format!("{} rate limit: {summary}", ctx.review_agent);
1216 record_review_agent_failure(
1217 &task.id.0,
1218 &ctx.review_agent,
1219 ctx.review_model.as_deref(),
1220 &err,
1221 )
1222 .await;
1223 complete_review_run(
1224 store,
1225 run.run_id,
1226 Some(exit_code),
1227 &raw_output,
1228 &stderr,
1229 &text_for_review,
1230 outcome_for_agent_error(&err),
1231 &stored_error,
1232 token_usage,
1233 )
1234 .await;
1235 return ReviewPhase::EarlyReturn(ReviewDecision::Failed(stored_error));
1236 }
1237
1238 if let Some(err) = runner::agents::patterns::detect_auth_error(scan_tail) {
1239 let runner::agents::AgentError::Auth { message } = &err else {
1240 unreachable!("detect_auth_error only returns AgentError::Auth")
1241 };
1242 tracing::warn!(
1243 task_id = task.id.0,
1244 agent = %ctx.review_agent,
1245 "review agent hit auth error — adding to cooldown"
1246 );
1247 let stored_error = format!("auth error: {message}");
1248 record_review_agent_failure(
1249 &task.id.0,
1250 &ctx.review_agent,
1251 ctx.review_model.as_deref(),
1252 &err,
1253 )
1254 .await;
1255 complete_review_run(
1256 store,
1257 run.run_id,
1258 Some(exit_code),
1259 &raw_output,
1260 &stderr,
1261 &text_for_review,
1262 outcome_for_agent_error(&err),
1263 &stored_error,
1264 token_usage,
1265 )
1266 .await;
1267 return ReviewPhase::EarlyReturn(ReviewDecision::Failed(stored_error));
1268 }
1269
1270 if agent_result
1277 .as_ref()
1278 .map(|r| r.truncated_by_length)
1279 .unwrap_or(false)
1280 {
1281 tracing::warn!(
1282 task_id = task.id.0,
1283 agent = %ctx.review_agent,
1284 model = ?ctx.review_model,
1285 "review agent output truncated: step_finish reason=length \
1286 (token budget exceeded before completion)"
1287 );
1288 if let Some(model) = ctx.review_model.as_deref() {
1289 crate::engine::cooldown::record_persistent_model_failure(
1290 &ctx.review_agent,
1291 model,
1292 )
1293 .await;
1294 }
1295 let stored_error = "truncated: review output was cut off before completion \
1296 (output/reasoning token budget exceeded)"
1297 .to_string();
1298 complete_review_run(
1299 store,
1300 run.run_id,
1301 Some(exit_code),
1302 &raw_output,
1303 &stderr,
1304 &text_for_review,
1305 "truncated",
1306 &stored_error,
1307 token_usage,
1308 )
1309 .await;
1310 return ReviewPhase::EarlyReturn(ReviewDecision::Failed(stored_error));
1311 }
1312
1313 if runner::agents::claude::review_session_exited_mid_thought(&raw_output) {
1316 tracing::warn!(
1317 task_id = task.id.0,
1318 agent = %ctx.review_agent,
1319 model = ?ctx.review_model,
1320 "review session exited mid-thought with no result envelope \
1321 — storing as incomplete and retrying without persistent cooldown"
1322 );
1323 let stored_error =
1324 "incomplete: review session exited before producing a decision".to_string();
1325 complete_review_run(
1326 store,
1327 run.run_id,
1328 Some(exit_code),
1329 &raw_output,
1330 &stderr,
1331 &text_for_review,
1332 "incomplete",
1333 &stored_error,
1334 token_usage,
1335 )
1336 .await;
1337 return ReviewPhase::EarlyReturn(ReviewDecision::Failed(stored_error));
1338 }
1339
1340 tracing::error!(
1341 task_id = task.id.0,
1342 error = %e,
1343 agent = %ctx.review_agent,
1344 output = %text_for_review.chars().take(300).collect::<String>(),
1345 raw_output = %raw_output.chars().take(1000).collect::<String>(),
1346 "failed to parse review response"
1347 );
1348 if let Some(model) = ctx.review_model.as_deref() {
1349 crate::engine::cooldown::record_persistent_model_failure(
1352 &ctx.review_agent,
1353 model,
1354 )
1355 .await;
1356 }
1357 complete_review_run(
1358 store,
1359 run.run_id,
1360 Some(exit_code),
1361 &raw_output,
1362 &stderr,
1363 &text_for_review,
1364 "parse_error",
1365 &format!("parse error: {e}"),
1366 token_usage,
1367 )
1368 .await;
1369 return ReviewPhase::EarlyReturn(ReviewDecision::Failed(format!(
1370 "parse error: {e}"
1371 )));
1372 }
1373 }
1374 };
1375
1376 let review_notes_for_comment = review_response.notes.clone();
1377 let decision = review_decision_from_response(review_response);
1378
1379 tracing::info!(
1380 task_id = task.id.0,
1381 pr_number = ctx.pr_number,
1382 decision = ?decision,
1383 "review agent decision received"
1384 );
1385 store_log_activity(
1386 &Some(Arc::clone(store)),
1387 repo,
1388 &task.id.0,
1389 "review_decision",
1390 None,
1391 None,
1392 Some(&ctx.review_agent),
1393 ctx.review_model.as_deref(),
1394 Some(&serde_json::json!({
1395 "decision": review_decision_name(&decision),
1396 "pr_number": ctx.pr_number,
1397 })),
1398 )
1399 .await;
1400
1401 ReviewPhase::Ready(ParsedReview {
1402 run_id: run.run_id,
1403 exit_code,
1404 stderr,
1405 raw_output,
1406 text_for_review,
1407 token_usage,
1408 review_notes_for_comment,
1409 decision,
1410 })
1411}
1412
1413async fn finalize_review_run(
1414 task: &ExternalTask,
1415 ctx: &ReviewContext,
1416 parsed: &ParsedReview,
1417 store: &Arc<TaskStore>,
1418) {
1419 let outcome = match &parsed.decision {
1420 ReviewDecision::Approve
1421 | ReviewDecision::RequestChanges { .. }
1422 | ReviewDecision::Skipped
1423 | ReviewDecision::Rerouted => "success",
1424 ReviewDecision::Failed(_) | ReviewDecision::Blocked(_) => "failed",
1425 };
1426 let error = match &parsed.decision {
1427 ReviewDecision::Failed(error) | ReviewDecision::Blocked(error) => error.as_str(),
1428 _ => "",
1429 };
1430
1431 if outcome == "success" {
1432 crate::engine::cooldown::record_agent_success(&ctx.review_agent, ctx.review_model_str())
1435 .await;
1436 }
1437
1438 complete_review_run(
1439 store,
1440 parsed.run_id,
1441 Some(parsed.exit_code),
1442 &parsed.raw_output,
1443 &parsed.stderr,
1444 &parsed.text_for_review,
1445 outcome,
1446 error,
1447 parsed.token_usage,
1448 )
1449 .await;
1450
1451 if let Some(store_id) = ctx.store_id {
1452 if parsed.token_usage.input_tokens > 0 || parsed.token_usage.output_tokens > 0 {
1453 let model = if ctx.review_model_str().is_empty() {
1454 "unknown"
1455 } else {
1456 ctx.review_model_str()
1457 };
1458 if let Err(e) = store
1459 .store_tokens(
1460 store_id,
1461 parsed.token_usage.input_tokens,
1462 parsed.token_usage.output_tokens,
1463 model,
1464 )
1465 .await
1466 {
1467 tracing::warn!(
1468 task_id = task.id.0,
1469 ?e,
1470 "failed to store review agent token usage"
1471 );
1472 }
1473 }
1474 }
1475}
1476
1477async fn restore_review_config_if_needed(
1478 task_id: &str,
1479 worktree_path: &std::path::Path,
1480 default_branch: &str,
1481) {
1482 let _ = Command::new("git")
1483 .args(["checkout", "HEAD", "--", ".orch.yml"])
1484 .current_dir(worktree_path)
1485 .status()
1486 .await;
1487
1488 let merge_base = Command::new("git")
1489 .args(["merge-base", "HEAD", default_branch])
1490 .current_dir(worktree_path)
1491 .output()
1492 .await
1493 .ok()
1494 .filter(|output| output.status.success())
1495 .and_then(|output| String::from_utf8(output.stdout).ok())
1496 .map(|s| s.trim().to_string())
1497 .unwrap_or_default();
1498
1499 if merge_base.is_empty() {
1500 return;
1501 }
1502
1503 let orch_yml_changed = Command::new("git")
1504 .args([
1505 "diff",
1506 "--name-only",
1507 &merge_base,
1508 "HEAD",
1509 "--",
1510 ".orch.yml",
1511 ])
1512 .current_dir(worktree_path)
1513 .output()
1514 .await
1515 .ok()
1516 .filter(|output| output.status.success())
1517 .map(|output| !output.stdout.is_empty())
1518 .unwrap_or(false);
1519
1520 if !orch_yml_changed {
1521 return;
1522 }
1523
1524 tracing::warn!(
1525 task_id,
1526 "review agent modified .orch.yml (forbidden); restoring to merge-base version"
1527 );
1528 let restore_ok = Command::new("git")
1529 .args(["checkout", &merge_base, "--", ".orch.yml"])
1530 .current_dir(worktree_path)
1531 .status()
1532 .await
1533 .map(|status| status.success())
1534 .unwrap_or(false);
1535
1536 if !restore_ok {
1537 return;
1538 }
1539
1540 let staged = Command::new("git")
1541 .args(["diff", "--cached", "--name-only", "--", ".orch.yml"])
1542 .current_dir(worktree_path)
1543 .output()
1544 .await
1545 .ok()
1546 .filter(|output| output.status.success())
1547 .map(|output| !output.stdout.is_empty())
1548 .unwrap_or(false);
1549
1550 if staged {
1551 let _ = Command::new("git")
1552 .args([
1553 "-c",
1554 "commit.gpgsign=false",
1555 "commit",
1556 "-m",
1557 "revert: restore .orch.yml (review agent must not modify project config)",
1558 ])
1559 .current_dir(worktree_path)
1560 .status()
1561 .await;
1562 }
1563}
1564
1565fn is_missing_path_error_message(err: &str) -> bool {
1566 err.to_ascii_lowercase()
1567 .contains("no such file or directory")
1568}
1569
1570async fn push_review_branch(
1571 task: &ExternalTask,
1572 repo: &str,
1573 ctx: &ReviewContext,
1574 store: &Arc<TaskStore>,
1575) -> Result<(), String> {
1576 let mut worktree_path = ctx.worktree_path.clone();
1577 let mut branch_name = ctx.branch_name.clone();
1578 let mut retried_missing_worktree = false;
1579
1580 let push_result = loop {
1581 match runner::git_ops::push_branch(&worktree_path, &branch_name, &ctx.default_branch).await
1582 {
1583 Ok(ok) => break Ok(ok),
1584 Err(e)
1585 if !retried_missing_worktree
1586 && !worktree_path.exists()
1587 && is_missing_path_error_message(&e.to_string()) =>
1588 {
1589 retried_missing_worktree = true;
1590 tracing::warn!(
1591 task_id = task.id.0,
1592 worktree = %worktree_path.display(),
1593 branch = %branch_name,
1594 "review push failed because worktree disappeared; attempting one-time recovery"
1595 );
1596 match recover_missing_review_worktree(task, repo, store).await {
1597 Ok(recovered) => {
1598 worktree_path = recovered.work_dir;
1599 branch_name = recovered.branch;
1600 continue;
1601 }
1602 Err(recover_err) => {
1603 break Err(anyhow::anyhow!(
1604 "missing review worktree recovery failed after push error: {recover_err}"
1605 ));
1606 }
1607 }
1608 }
1609 Err(e) => break Err(e),
1610 }
1611 };
1612
1613 match push_result {
1614 Ok(_) => {
1615 store_log_activity(
1616 &Some(Arc::clone(store)),
1617 repo,
1618 &task.id.0,
1619 "push",
1620 None,
1621 None,
1622 Some(&ctx.review_agent),
1623 ctx.review_model.as_deref(),
1624 Some(&serde_json::json!({
1625 "status": "ok",
1626 "branch": branch_name.clone(),
1627 "default_branch": ctx.default_branch.clone(),
1628 "phase": "review",
1629 })),
1630 )
1631 .await;
1632 if ctx.had_prev_push_failure {
1633 if let Err(e) = store_set_result(
1634 &Some(Arc::clone(store)),
1635 repo,
1636 &task.id.0,
1637 &[
1638 ("last_error", serde_json::json!("")),
1639 ("push_failures", serde_json::json!(0)),
1640 ],
1641 )
1642 .await
1643 {
1644 tracing::warn!(task_id = task.id.0, err = %e, "failed to reset push_failures — counter may drift");
1645 }
1646 }
1647 Ok(())
1648 }
1649 Err(e) => {
1650 store_log_activity(
1651 &Some(Arc::clone(store)),
1652 repo,
1653 &task.id.0,
1654 "push",
1655 None,
1656 None,
1657 Some(&ctx.review_agent),
1658 ctx.review_model.as_deref(),
1659 Some(&serde_json::json!({
1660 "status": "error",
1661 "branch": branch_name.clone(),
1662 "default_branch": ctx.default_branch.clone(),
1663 "phase": "review",
1664 "error": e.to_string(),
1665 })),
1666 )
1667 .await;
1668 tracing::error!(
1669 task_id = task.id.0,
1670 branch = %ctx.branch_name,
1671 error = %e,
1672 "review push failed"
1673 );
1674 store_set(
1675 &Some(Arc::clone(store)),
1676 repo,
1677 &task.id.0,
1678 &[("last_error", serde_json::json!(format!("push failed: {e}")))],
1679 )
1680 .await;
1681 Err(e.to_string())
1682 }
1683 }
1684}
1685
1686fn build_pr_review_comment(decision: &ReviewDecision, review_notes_for_comment: &str) -> String {
1687 match decision {
1688 ReviewDecision::Approve => {
1689 format!(
1690 "## Automated Review \u{2014} Approve\n\n{}",
1691 review_notes_for_comment
1692 )
1693 }
1694 ReviewDecision::RequestChanges { notes, issues } => {
1695 let mut body = format!(
1696 "## Automated Review \u{2014} Changes Requested\n\n{}\n",
1697 notes
1698 );
1699 if !issues.is_empty() {
1700 body.push_str("\n**Issues Found:**\n");
1701 for issue in issues {
1702 body.push_str(&format!(
1703 "- `{}` line {}: {} [{}]\n",
1704 issue.file,
1705 issue
1706 .line
1707 .map(|line| line.to_string())
1708 .unwrap_or_else(|| "?".to_string()),
1709 issue.description,
1710 issue.severity
1711 ));
1712 }
1713 }
1714 body
1715 }
1716 _ => String::new(),
1717 }
1718}
1719
1720async fn post_review_comment(
1721 task: &ExternalTask,
1722 repo: &str,
1723 ctx: &ReviewContext,
1724 decision: &ReviewDecision,
1725 review_notes_for_comment: &str,
1726) -> anyhow::Result<()> {
1727 let gh = match GhHttp::new() {
1728 Ok(gh) => gh,
1729 Err(e) => {
1730 tracing::warn!(
1731 task_id = task.id.0,
1732 error = %e,
1733 "failed to create GH client for review comment"
1734 );
1735 return Ok(());
1736 }
1737 };
1738 let mut pr_comment = build_pr_review_comment(decision, review_notes_for_comment);
1739 if ctx.same_agent_fallback && !pr_comment.is_empty() {
1740 pr_comment.push_str(
1741 "\n> Note: no other agent was available, so this PR was reviewed by the same agent that wrote the code.\n",
1742 );
1743 }
1744
1745 if !pr_comment.is_empty() {
1746 let footer =
1747 attribution_footer("Reviewed", &ctx.review_agent, Some(ctx.review_model_str()));
1748 let pr_comment_with_footer = format!("{}{}", pr_comment, footer);
1749 if let Err(e) = gh
1750 .add_comment(repo, &ctx.pr_number.to_string(), &pr_comment_with_footer)
1751 .await
1752 {
1753 tracing::warn!(
1754 task_id = task.id.0,
1755 pr_number = ctx.pr_number,
1756 error = %e,
1757 "failed to post automated review comment on PR"
1758 );
1759 }
1760 }
1761
1762 Ok(())
1763}
1764
1765#[allow(clippy::too_many_arguments)]
1766async fn apply_review_decision(
1767 task: &ExternalTask,
1768 backend: &Arc<dyn ExternalBackend>,
1769 repo: &str,
1770 task_manager: &Arc<TaskManager>,
1771 store: &Arc<TaskStore>,
1772 ctx: &ReviewContext,
1773 parsed: &ParsedReview,
1774 router_config: &crate::engine::router::RouterConfig,
1775) -> anyhow::Result<ReviewDecision> {
1776 restore_review_config_if_needed(&task.id.0, &ctx.worktree_path, &ctx.default_branch).await;
1777
1778 if let Err(e) = push_review_branch(task, repo, ctx, store).await {
1779 return Ok(ReviewDecision::Failed(format!("push failed: {e}")));
1780 }
1781
1782 post_review_comment(
1783 task,
1784 repo,
1785 ctx,
1786 &parsed.decision,
1787 &parsed.review_notes_for_comment,
1788 )
1789 .await?;
1790
1791 match &parsed.decision {
1792 ReviewDecision::Approve => {
1793 if let Err(e) = auto_merge_pr(
1798 task,
1799 &ctx.branch_name,
1800 backend,
1801 repo,
1802 &ctx.review_agent,
1803 ctx.review_model_str(),
1804 task_manager,
1805 store,
1806 )
1807 .await
1808 {
1809 let error_msg = e.to_string();
1810 if error_msg.contains("not yet computed") {
1811 tracing::warn!(
1812 task_id = task.id.0,
1813 pr_number = ctx.pr_number,
1814 branch = %ctx.branch_name,
1815 error = %e,
1816 "auto-merge deferred — PR mergeability not yet computed"
1817 );
1818 } else {
1819 tracing::error!(
1820 task_id = task.id.0,
1821 pr_number = ctx.pr_number,
1822 branch = %ctx.branch_name,
1823 error = %e,
1824 "auto-merge failed"
1825 );
1826 }
1827 return Ok(ReviewDecision::Failed(format!("merge failed: {e}")));
1828 }
1829 Ok(ReviewDecision::Approve)
1830 }
1831 ReviewDecision::RequestChanges { notes, issues } => {
1832 handle_review_changes(
1833 task,
1834 notes,
1835 issues,
1836 backend,
1837 repo,
1838 ctx.pr_number,
1839 &ctx.review_agent,
1840 ctx.review_model_str(),
1841 task_manager,
1842 store,
1843 router_config,
1844 )
1845 .await?;
1846 Ok(parsed.decision.clone())
1847 }
1848 _ => Ok(parsed.decision.clone()),
1849 }
1850}
1851
1852async fn recover_missing_review_worktree(
1853 task: &ExternalTask,
1854 repo: &str,
1855 store: &Arc<TaskStore>,
1856) -> anyhow::Result<worktree::WorktreeSetup> {
1857 let project_dir = crate::config::get_projects_with_paths()?
1858 .into_iter()
1859 .find(|(candidate_repo, _)| candidate_repo == repo)
1860 .map(|(_, project_dir)| project_dir)
1861 .ok_or_else(|| anyhow::anyhow!("no project directory configured for repo {repo}"))?;
1862
1863 tracing::warn!(
1864 task_id = task.id.0,
1865 repo,
1866 project_dir = %project_dir.display(),
1867 "review worktree missing, recreating via normal setup"
1868 );
1869
1870 let store_ref = Some(Arc::clone(store));
1871 let wt = worktree::setup_worktree(&task.id.0, &task.title, &project_dir, &store_ref, repo)
1872 .await
1873 .context("recreating missing review worktree")?;
1874
1875 if !wt.work_dir.exists() {
1876 anyhow::bail!(
1877 "recreated review worktree at {} for task {} is still missing",
1878 wt.work_dir.display(),
1879 task.id.0
1880 );
1881 }
1882
1883 Ok(wt)
1884}
1885
1886async fn count_ahead_commits(
1892 worktree_path: &std::path::Path,
1893 default_branch: &str,
1894) -> Result<u64, String> {
1895 let worktree_str = worktree_path.to_str().ok_or_else(|| {
1896 format!(
1897 "worktree path contains non-UTF-8 characters: {:?}",
1898 worktree_path
1899 )
1900 })?;
1901
1902 let output = tokio::process::Command::new("git")
1903 .args([
1904 "-C",
1905 worktree_str,
1906 "rev-list",
1907 "--count",
1908 &format!("origin/{default_branch}..HEAD"),
1909 ])
1910 .output()
1911 .await
1912 .map_err(|e| format!("git rev-list command failed to start: {e}"))?;
1913
1914 if !output.status.success() {
1915 let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string();
1916 return Err(format!(
1917 "git rev-list exited with {:?}: {}",
1918 output.status.code(),
1919 if stderr.is_empty() {
1920 String::from_utf8_lossy(&output.stdout).trim().to_string()
1921 } else {
1922 stderr
1923 }
1924 ));
1925 }
1926
1927 String::from_utf8_lossy(&output.stdout)
1928 .trim()
1929 .parse::<u64>()
1930 .map_err(|e| format!("could not parse git rev-list output as u64: {e}"))
1931}
1932
1933#[allow(clippy::too_many_arguments)]
1940async fn ensure_pr_exists(
1941 task: &ExternalTask,
1942 branch_name: &str,
1943 worktree_path: &std::path::Path,
1944 repo: &str,
1945 agent_summary: &str,
1946 store: &Arc<TaskStore>,
1947 task_manager: &Arc<TaskManager>,
1948 stored_pr_number: Option<u64>,
1949) -> anyhow::Result<EnsurePrResult> {
1950 let gh_check = GhHttp::new()?;
1951
1952 if let Some(n) = stored_pr_number {
1953 tracing::info!(
1954 task_id = task.id.0,
1955 pr_number = n,
1956 branch = %branch_name,
1957 "open PR found in store, proceeding with review"
1958 );
1959 return Ok(EnsurePrResult::Ready(n));
1960 }
1961
1962 match gh_check.get_pr_number(repo, branch_name).await {
1963 Ok(Some(n)) => {
1964 tracing::info!(
1965 task_id = task.id.0,
1966 pr_number = n,
1967 branch = %branch_name,
1968 "open PR found, proceeding with review"
1969 );
1970 Ok(EnsurePrResult::Ready(n))
1971 }
1972 Ok(None) => {
1973 let default_branch = worktree::detect_default_branch(worktree_path).await;
1975 let has_commits = match count_ahead_commits(worktree_path, &default_branch).await {
1976 Ok(count) => count > 0,
1977 Err(e) => {
1978 tracing::error!(
1979 task_id = task.id.0,
1980 branch = %branch_name,
1981 default_branch = %default_branch,
1982 err = %e,
1983 "could not determine if branch has commits ahead of default"
1984 );
1985 return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
1986 format!("git ancestry check failed: {e}"),
1987 )));
1988 }
1989 };
1990
1991 if has_commits {
1992 tracing::warn!(
1994 task_id = task.id.0,
1995 branch = %branch_name,
1996 "no open PR but branch has commits — attempting to create PR"
1997 );
1998 let worktree_str = worktree_path.to_str().ok_or_else(|| {
1999 anyhow::anyhow!(
2000 "worktree path contains non-UTF-8 characters: {:?}",
2001 worktree_path
2002 )
2003 })?;
2004 let auth_env = runner::git_ops::build_git_auth_env();
2006
2007 let push_timeout = std::time::Duration::from_secs(120);
2012 match tokio::time::timeout(
2013 push_timeout,
2014 tokio::process::Command::new("git")
2015 .args(["-C", worktree_str, "push", "-u", "origin", branch_name])
2016 .envs(auth_env.iter().map(|(k, v)| (k.as_str(), v.as_str())))
2017 .output(),
2018 )
2019 .await
2020 {
2021 Err(_elapsed) => {
2022 tracing::warn!(
2023 task_id = task.id.0,
2024 branch = %branch_name,
2025 timeout_secs = push_timeout.as_secs(),
2026 "review gate fallback push timed out"
2027 );
2028 }
2029 Ok(Ok(o)) => {
2030 if !o.status.success() {
2031 tracing::warn!(
2032 task_id = task.id.0,
2033 branch = %branch_name,
2034 exit_code = ?o.status.code(),
2035 stdout = %String::from_utf8_lossy(&o.stdout),
2036 stderr = %String::from_utf8_lossy(&o.stderr),
2037 "review gate fallback push failed"
2038 );
2039 } else {
2040 tracing::info!(task_id = task.id.0, branch = %branch_name, "review gate fallback push succeeded");
2041 }
2042 }
2043 Ok(Err(e)) => {
2044 tracing::error!(
2045 task_id = task.id.0,
2046 branch = %branch_name,
2047 error = %e,
2048 "review gate fallback push command failed to run"
2049 );
2050 }
2051 }
2052
2053 let task_ref = runner::git_ops::format_task_ref(&task.id.0);
2054 let pr_body = format!(
2055 "Resolves {task_ref}\n\nAuto-created by orch review gate (agent forgot to open PR)"
2056 );
2057 let gh = GhHttp::new()?;
2058 match gh
2059 .create_pr(repo, &task.title, &pr_body, branch_name, &default_branch)
2060 .await
2061 {
2062 Ok(url) => match parse_pr_number_from_url(&url) {
2063 Ok(pr_num) => {
2064 store_set_result(
2065 &Some(Arc::clone(store)),
2066 repo,
2067 &task.id.0,
2068 &[("pr_number", serde_json::json!(pr_num as i64))],
2069 )
2070 .await?;
2071 tracing::info!(
2072 task_id = task.id.0,
2073 branch = %branch_name,
2074 pr_url = %url,
2075 "created missing PR via GhHttp — continuing review"
2076 );
2077 Ok(EnsurePrResult::Ready(pr_num))
2078 }
2079 Err(e) => {
2080 tracing::warn!(
2081 task_id = task.id.0,
2082 branch = %branch_name,
2083 pr_url = %url,
2084 error = %e,
2085 "created missing PR via GhHttp, but failed to parse PR number"
2086 );
2087 Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2088 format!("created missing PR, but could not parse PR number: {e}"),
2089 )))
2090 }
2091 },
2092 Err(e) => {
2093 let e_str = format!("{e}");
2094 if e_str.contains("already exists") {
2095 let backoff_delays = [2u64, 4u64];
2099 let mut found_pr: Option<u64> = None;
2100 for delay_secs in backoff_delays {
2101 tokio::time::sleep(tokio::time::Duration::from_secs(delay_secs))
2102 .await;
2103 if let Ok(Some(n)) = gh_check.get_pr_number(repo, branch_name).await
2104 {
2105 found_pr = Some(n);
2106 break;
2107 }
2108 }
2109 if let Some(n) = found_pr {
2110 tracing::info!(
2111 task_id = task.id.0,
2112 pr_number = n,
2113 branch = %branch_name,
2114 "found existing PR after create_pr 422 — retrying review"
2115 );
2116 store_set_result(
2117 &Some(Arc::clone(store)),
2118 repo,
2119 &task.id.0,
2120 &[("pr_number", serde_json::json!(n as i64))],
2121 )
2122 .await?;
2123 return Ok(EnsurePrResult::Ready(n));
2124 }
2125 }
2126 tracing::warn!(
2127 task_id = task.id.0,
2128 branch = %branch_name,
2129 error = %e,
2130 "create_pr failed via GhHttp, falling back to CLI"
2131 );
2132 let pr_timeout = std::time::Duration::from_secs(120);
2135 let mut gh_cmd = tokio::process::Command::new("gh");
2136 gh_cmd.args([
2137 "pr",
2138 "create",
2139 "--repo",
2140 repo,
2141 "--head",
2142 branch_name,
2143 "--title",
2144 &task.title,
2145 "--body",
2146 &pr_body,
2147 ]);
2148 gh_cmd.current_dir(worktree_path);
2149 gh_cmd.kill_on_drop(true);
2150
2151 let pr_result = tokio::time::timeout(pr_timeout, gh_cmd.output()).await;
2152 match pr_result {
2153 Ok(Ok(o)) if o.status.success() => {
2154 let stdout = String::from_utf8_lossy(&o.stdout);
2155 match parse_pr_number_from_url(stdout.trim()) {
2156 Ok(pr_num) => {
2157 store_set_result(
2158 &Some(Arc::clone(store)),
2159 repo,
2160 &task.id.0,
2161 &[("pr_number", serde_json::json!(pr_num as i64))],
2162 )
2163 .await?;
2164 tracing::info!(
2165 task_id = task.id.0,
2166 branch = %branch_name,
2167 "created missing PR via CLI — continuing review"
2168 );
2169 Ok(EnsurePrResult::Ready(pr_num))
2170 }
2171 Err(e) => {
2172 tracing::warn!(
2173 task_id = task.id.0,
2174 branch = %branch_name,
2175 stdout = %stdout,
2176 error = %e,
2177 "created missing PR via CLI, but failed to parse PR number"
2178 );
2179 Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2180 format!("created missing PR via CLI, but could not parse PR number: {e}")
2181 )))
2182 }
2183 }
2184 }
2185 Ok(Ok(o)) => {
2186 let stderr = String::from_utf8_lossy(&o.stderr);
2187 if stderr.contains("already exists") {
2188 if let Some(pr_url) =
2189 stderr.lines().find(|l| l.trim().starts_with("https://"))
2190 {
2191 match parse_pr_number_from_url(pr_url.trim()) {
2192 Ok(pr_num) => {
2193 if let Err(e) = store_set_result(
2194 &Some(Arc::clone(store)),
2195 repo,
2196 &task.id.0,
2197 &[(
2198 "pr_number",
2199 serde_json::json!(pr_num as i64),
2200 )],
2201 )
2202 .await
2203 {
2204 tracing::warn!(
2205 task_id = task.id.0,
2206 pr_num,
2207 err = %e,
2208 "failed to persist pr_number from already-exists recovery — downstream may retry PR creation"
2209 );
2210 }
2211 tracing::info!(
2212 task_id = task.id.0,
2213 branch = %branch_name,
2214 pr_url = %pr_url,
2215 "PR already exists (from CLI stderr) — retrying review"
2216 );
2217 return Ok(EnsurePrResult::Ready(pr_num));
2218 }
2219 Err(e) => {
2220 tracing::warn!(
2221 task_id = task.id.0,
2222 branch = %branch_name,
2223 pr_url = %pr_url,
2224 error = %e,
2225 "found existing PR URL in CLI stderr, but failed to parse PR number"
2226 );
2227 }
2229 }
2230 }
2231 }
2232 tracing::error!(
2233 task_id = task.id.0,
2234 branch = %branch_name,
2235 stderr = %stderr,
2236 "failed to create missing PR — work may be stuck"
2237 );
2238
2239 if crate::engine::runner::git_ops::is_transient_github_api_error(
2246 &stderr,
2247 ) {
2248 tracing::warn!(
2249 task_id = task.id.0,
2250 branch = %branch_name,
2251 "transient GitHub error creating PR; will retry later without incrementing persistent failure counter"
2252 );
2253 return Ok(EnsurePrResult::EarlyReturn(
2254 ReviewDecision::Failed(format!(
2255 "transient github error creating PR: {stderr}"
2256 )),
2257 ));
2258 }
2259
2260 match store_increment(
2261 &Some(Arc::clone(store)),
2262 repo,
2263 &task.id.0,
2264 "pr_create_failures",
2265 )
2266 .await
2267 {
2268 Ok(failures) if failures >= MAX_PR_CREATE_FAILURES => {
2269 return Ok(EnsurePrResult::EarlyReturn(
2270 ReviewDecision::Blocked(format!(
2271 "no PR, create failed {failures} times: {stderr}"
2272 )),
2273 ));
2274 }
2275 Ok(_) => {}
2276 Err(e) => {
2277 tracing::warn!(task_id = task.id.0, err = %e, "failed to increment pr_create_failures — skipping blocking decision this tick");
2278 }
2279 }
2280 Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2281 format!("no PR, create failed: {stderr}"),
2282 )))
2283 }
2284 Ok(Err(e)) => {
2285 tracing::error!(
2286 task_id = task.id.0,
2287 error = %e,
2288 "failed to run gh pr create"
2289 );
2290 let e_str = format!("{e}");
2291 if crate::engine::runner::git_ops::is_transient_github_api_error(
2292 &e_str,
2293 ) {
2294 tracing::warn!(
2295 task_id = task.id.0,
2296 branch = %branch_name,
2297 "transient GitHub error from gh CLI fallback; will retry later without incrementing persistent failure counter"
2298 );
2299 return Ok(EnsurePrResult::EarlyReturn(
2300 ReviewDecision::Failed(format!("transient gh error: {e}")),
2301 ));
2302 }
2303
2304 match store_increment(
2305 &Some(Arc::clone(store)),
2306 repo,
2307 &task.id.0,
2308 "pr_create_failures",
2309 )
2310 .await
2311 {
2312 Ok(failures) if failures >= MAX_PR_CREATE_FAILURES => {
2313 return Ok(EnsurePrResult::EarlyReturn(
2314 ReviewDecision::Blocked(format!(
2315 "no PR, gh error {failures} times: {e}"
2316 )),
2317 ));
2318 }
2319 Ok(_) => {}
2320 Err(e) => {
2321 tracing::warn!(task_id = task.id.0, err = %e, "failed to increment pr_create_failures — skipping blocking decision this tick");
2322 }
2323 }
2324 Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2325 format!("no PR, gh error: {e}"),
2326 )))
2327 }
2328 Err(_elapsed) => {
2329 tracing::error!(
2331 task_id = task.id.0,
2332 timeout_secs = pr_timeout.as_secs(),
2333 "gh pr create CLI fallback timed out"
2334 );
2335 let e_str = format!(
2336 "gh pr create timed out after {}s",
2337 pr_timeout.as_secs()
2338 );
2339 if crate::engine::runner::git_ops::is_transient_github_api_error(
2340 &e_str,
2341 ) {
2342 tracing::warn!(
2343 task_id = task.id.0,
2344 branch = %branch_name,
2345 "transient GitHub timeout from gh CLI fallback; will retry later without incrementing persistent failure counter"
2346 );
2347 return Ok(EnsurePrResult::EarlyReturn(
2348 ReviewDecision::Failed(format!(
2349 "transient gh error: {e_str}"
2350 )),
2351 ));
2352 }
2353
2354 match store_increment(
2355 &Some(Arc::clone(store)),
2356 repo,
2357 &task.id.0,
2358 "pr_create_failures",
2359 )
2360 .await
2361 {
2362 Ok(failures) if failures >= MAX_PR_CREATE_FAILURES => {
2363 return Ok(EnsurePrResult::EarlyReturn(
2364 ReviewDecision::Blocked(format!(
2365 "no PR, gh timeout {failures} times: {e_str}"
2366 )),
2367 ));
2368 }
2369 Ok(_) => {}
2370 Err(e) => {
2371 tracing::warn!(task_id = task.id.0, err = %e, "failed to increment pr_create_failures — skipping blocking decision this tick");
2372 }
2373 }
2374 Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2375 format!("no PR, gh timeout: {e_str}"),
2376 )))
2377 }
2378 }
2379 }
2380 }
2381 } else {
2382 let merged = match gh_check.is_pr_merged(repo, branch_name).await {
2384 Ok(v) => v,
2385 Err(e) => {
2386 tracing::warn!(task_id = task.id.0, branch = %branch_name, err = %e, "merge check failed, skipping task this tick");
2387 return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2388 format!("merge check failed: {e}"),
2389 )));
2390 }
2391 };
2392 if merged {
2393 tracing::info!(
2394 task_id = task.id.0,
2395 branch = %branch_name,
2396 "PR already merged, marking done"
2397 );
2398 if let Err(e) = task_manager
2399 .update_task_status(&task.id, Status::Done)
2400 .await
2401 {
2402 tracing::error!(task_id = task.id.0, err = %e, "update_task_status(Done) failed — task may be stuck in InReview");
2403 }
2404 return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Skipped));
2405 }
2406
2407 let last_error = opt_store_get_task(&Some(Arc::clone(store)), repo, &task.id.0)
2408 .await
2409 .map(|t| t.last_error)
2410 .unwrap_or_default();
2411 let reason = if !agent_summary.is_empty() {
2412 agent_summary.to_string()
2413 } else {
2414 last_error.clone()
2415 };
2416
2417 if last_error.contains("exceeded max attempts") {
2418 tracing::warn!(
2419 task_id = task.id.0,
2420 branch = %branch_name,
2421 "no PR and no commits after max attempts — marking blocked to stop loop"
2422 );
2423 let fields = [
2427 (
2428 "block_reason",
2429 serde_json::json!("max attempts exceeded — no PR or commits produced"),
2430 ),
2431 ("last_error", serde_json::json!(last_error)),
2432 ];
2433 if let Err(e) = task_manager
2434 .update_task_status_and_result(&task.id, Status::Blocked, &fields)
2435 .await
2436 {
2437 tracing::error!(task_id = task.id.0, err = %e, "update_task_status_and_result(Blocked) failed — skipping block to avoid silent auto-unblock loop");
2438 return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2439 format!("failed to write block_reason: {e}"),
2440 )));
2441 }
2442 return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Skipped));
2443 }
2444
2445 if last_error.contains("422") && last_error.contains("head") {
2446 tracing::info!(
2447 task_id = task.id.0,
2448 branch = %branch_name,
2449 "no PR — 422/head-invalid means work already merged, marking done"
2450 );
2451 if let Err(e) = task_manager
2452 .update_task_status(&task.id, Status::Done)
2453 .await
2454 {
2455 tracing::error!(task_id = task.id.0, err = %e, "update_task_status(Done) failed");
2456 }
2457 return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Skipped));
2458 }
2459
2460 if last_error.contains("no fallback agents")
2461 || last_error.contains("all agents exhausted")
2462 {
2463 tracing::warn!(
2464 task_id = task.id.0,
2465 branch = %branch_name,
2466 last_error = %last_error,
2467 "no PR and no commits after failover exhaustion — marking blocked to stop loop"
2468 );
2469 let fields = [
2473 (
2474 "block_reason",
2475 serde_json::json!("all agents exhausted — no PR or commits produced"),
2476 ),
2477 ("last_error", serde_json::json!(last_error)),
2478 ];
2479 if let Err(e) = task_manager
2480 .update_task_status_and_result(&task.id, Status::Blocked, &fields)
2481 .await
2482 {
2483 tracing::error!(task_id = task.id.0, err = %e, "update_task_status_and_result(Blocked) failed — skipping block to avoid silent auto-unblock loop");
2484 return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2485 format!("failed to write block_reason: {e}"),
2486 )));
2487 }
2488 return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Skipped));
2489 }
2490
2491 let max_reroutes: u32 = config::get("workflow.max_reroute_attempts")
2498 .ok()
2499 .and_then(|s| s.parse().ok())
2500 .or_else(|| {
2501 config::get("workflow.max_attempts")
2502 .ok()
2503 .and_then(|s| s.parse().ok())
2504 })
2505 .unwrap_or(3);
2506
2507 tracing::warn!(
2508 task_id = task.id.0,
2509 branch = %branch_name,
2510 reason = %reason,
2511 "no PR and no commits — re-routing for retry"
2512 );
2513
2514 if should_skip_no_code_reroute_increment(&last_error) {
2518 tracing::warn!(
2519 task_id = task.id.0,
2520 last_error = %last_error,
2521 "transient GitHub error recorded; skipping persistent no_pr_reroutes increment"
2522 );
2523 return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2524 format!("transient github error: {last_error}"),
2525 )));
2526 }
2527
2528 let reroutes = match store_increment(
2530 &Some(Arc::clone(store)),
2531 repo,
2532 &task.id.0,
2533 "no_code_reroutes",
2534 )
2535 .await
2536 {
2537 Ok(v) => v,
2538 Err(e) => {
2539 tracing::warn!(task_id = task.id.0, err = %e, "failed to increment no_code_reroutes — skipping reroute/block decision this tick");
2540 return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2542 format!("transient store error: {e}"),
2543 )));
2544 }
2545 };
2546
2547 if reroutes as u32 >= max_reroutes {
2548 tracing::error!(
2549 task_id = task.id.0,
2550 reroutes,
2551 max_reroutes,
2552 "reached max reroute attempts for no-pr-result — blocking for human review"
2553 );
2554 let msg = format!(
2555 "no PR or code changes after {}/{} reroute attempts",
2556 reroutes, max_reroutes
2557 );
2558 let fields = [
2562 (
2563 "block_reason",
2564 serde_json::json!(format!(
2565 "max reroute attempts ({}) reached — no PR or code changes produced",
2566 max_reroutes
2567 )),
2568 ),
2569 ("last_error", serde_json::json!(msg)),
2570 ("agent", serde_json::json!(null)),
2571 ("model", serde_json::json!(null)),
2572 ];
2573 if let Err(e) = task_manager
2574 .update_task_status_and_result(&task.id, Status::Blocked, &fields)
2575 .await
2576 {
2577 tracing::error!(task_id = task.id.0, err = %e, "update_task_status_and_result(Blocked) failed — skipping block to avoid silent auto-unblock loop");
2578 }
2579 Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Blocked(msg)))
2580 } else if task.id.0.starts_with("internal:") {
2581 tracing::info!(
2588 task_id = task.id.0,
2589 "no PR and no commits for internal task — marking done"
2590 );
2591 if let Err(e) = task_manager
2592 .update_task_status(&task.id, Status::Done)
2593 .await
2594 {
2595 tracing::error!(task_id = task.id.0, err = %e, "update_task_status(Done) failed — task may be stuck in InReview");
2596 }
2597 Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Skipped))
2598 } else {
2599 if let Err(e) = store_set_result(
2604 &Some(Arc::clone(store)),
2605 repo,
2606 &task.id.0,
2607 &[
2608 ("agent", serde_json::json!(null)),
2609 ("model", serde_json::json!(null)),
2610 (
2611 "last_error",
2612 serde_json::json!("no PR or code changes produced"),
2613 ),
2614 ],
2615 )
2616 .await
2617 {
2618 tracing::error!(task_id = task.id.0, err = %e, "failed to clear agent/model for reroute — skipping status transition");
2619 } else if let Err(e) =
2621 task_manager.update_task_status(&task.id, Status::New).await
2622 {
2623 tracing::error!(task_id = task.id.0, err = %e, "update_task_status(New) failed — task may be stuck in InReview");
2624 }
2625 Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Rerouted))
2626 }
2627 }
2628 }
2629 Err(e) => {
2630 tracing::warn!(
2631 task_id = task.id.0,
2632 branch = %branch_name,
2633 error = %e,
2634 "failed to check PR status"
2635 );
2636 Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2637 format!("PR check failed: {e}"),
2638 )))
2639 }
2640 }
2641}
2642
2643pub(crate) async fn review_and_merge(
2649 task: &ExternalTask,
2650 backend: &Arc<dyn ExternalBackend>,
2651 tmux: &Arc<TmuxManager>,
2652 repo: &str,
2653 router: &Arc<RwLock<Router>>,
2654 task_manager: &Arc<TaskManager>,
2655 store: &Arc<TaskStore>,
2656) -> anyhow::Result<ReviewDecision> {
2657 set_review_session_expected(store, repo, &task.id.0, true).await;
2658
2659 let ctx = match build_review_context(task, repo, router, task_manager, store).await? {
2660 ReviewPhase::Ready(ctx) => ctx,
2661 ReviewPhase::EarlyReturn(decision) => return Ok(decision),
2662 };
2663
2664 let run = match invoke_review_agent(task, tmux, &ctx, repo, store).await {
2665 ReviewPhase::Ready(run) => run,
2666 ReviewPhase::EarlyReturn(decision) => return Ok(decision),
2667 };
2668
2669 let parsed = match parse_review_output(task, repo, &ctx, run, store).await {
2670 ReviewPhase::Ready(parsed) => parsed,
2671 ReviewPhase::EarlyReturn(decision) => return Ok(decision),
2672 };
2673
2674 finalize_review_run(task, &ctx, &parsed, store).await;
2675
2676 let router_config = router.read().await.config.clone();
2677 apply_review_decision(
2678 task,
2679 backend,
2680 repo,
2681 task_manager,
2682 store,
2683 &ctx,
2684 &parsed,
2685 &router_config,
2686 )
2687 .await
2688}
2689
2690#[cfg(test)]
2691mod tests {
2692 use super::*;
2693
2694 use crate::backends::ExternalTask;
2695 use crate::engine::router::RouterConfig;
2696 use crate::github::types::{GitHubReview, GitHubReviewComment, GitHubUser, PullRequestReview};
2697 use crate::store::{TaskStatus, TaskStore};
2698 use tempfile::TempDir;
2699
2700 #[test]
2703 fn outcome_for_agent_error_kimi_billing_cycle_is_not_rate_limit() {
2704 let err = runner::agents::AgentError::RateLimit {
2708 message: "Failed to authenticate. API Error: 403 You've reached your usage limit \
2709 for this billing cycle. Your quota will be refreshed soon."
2710 .to_string(),
2711 };
2712 assert_eq!(outcome_for_agent_error(&err), "billing_cycle_exhausted");
2713 }
2714
2715 #[test]
2716 fn outcome_for_agent_error_generic_rate_limit_unaffected() {
2717 let err = runner::agents::AgentError::RateLimit {
2718 message: "429 Too Many Requests".to_string(),
2719 };
2720 assert_eq!(outcome_for_agent_error(&err), "rate_limit");
2721 }
2722
2723 #[test]
2724 fn outcome_for_agent_error_generic_auth_unaffected() {
2725 let err = runner::agents::AgentError::Auth {
2726 message: "401 Unauthorized".to_string(),
2727 };
2728 assert_eq!(outcome_for_agent_error(&err), "auth_error");
2729 }
2730
2731 #[test]
2734 fn parse_pr_number_standard_url() {
2735 assert_eq!(
2736 parse_pr_number_from_url("https://github.com/owner/repo/pull/42"),
2737 Ok(42)
2738 );
2739 }
2740
2741 #[test]
2742 fn parse_pr_number_trailing_slash() {
2743 assert_eq!(
2744 parse_pr_number_from_url("https://github.com/owner/repo/pull/42/"),
2745 Ok(42)
2746 );
2747 }
2748
2749 #[test]
2750 fn parse_pr_number_with_query_string() {
2751 assert_eq!(
2752 parse_pr_number_from_url("https://github.com/owner/repo/pull/42?tab=files"),
2753 Ok(42)
2754 );
2755 }
2756
2757 #[test]
2758 fn parse_pr_number_with_hash_fragment() {
2759 assert_eq!(
2760 parse_pr_number_from_url("https://github.com/owner/repo/pull/42#issuecomment-123"),
2761 Ok(42)
2762 );
2763 }
2764
2765 #[test]
2766 fn parse_pr_number_with_query_and_hash() {
2767 assert_eq!(
2768 parse_pr_number_from_url("https://github.com/owner/repo/pull/42?quick_pull=1#top"),
2769 Ok(42)
2770 );
2771 }
2772
2773 #[test]
2774 fn parse_pr_number_with_leading_whitespace() {
2775 assert_eq!(
2776 parse_pr_number_from_url(" https://github.com/owner/repo/pull/42\n"),
2777 Ok(42)
2778 );
2779 }
2780
2781 #[test]
2782 fn parse_pr_number_wrong_domain() {
2783 let url = "https://gitlab.com/owner/repo/merge_requests/42";
2784 let result = parse_pr_number_from_url(url);
2785 assert!(result.is_err());
2786 assert!(result
2787 .unwrap_err()
2788 .contains("does not start with 'https://github.com/'"));
2789 }
2790
2791 #[test]
2792 fn parse_pr_number_missing_pull_segment() {
2793 let url = "https://github.com/owner/repo/issues/42";
2795 let result = parse_pr_number_from_url(url);
2796 assert!(result.is_err());
2797 assert!(result.unwrap_err().contains("missing '/pull/' segment"));
2798 }
2799
2800 #[test]
2801 fn parse_pr_number_zero_is_rejected() {
2802 let url = "https://github.com/owner/repo/pull/0";
2803 let result = parse_pr_number_from_url(url);
2804 assert!(result.is_err());
2805 assert!(result.unwrap_err().contains("cannot be zero"));
2806 }
2807
2808 #[test]
2809 fn parse_pr_number_non_numeric_segment() {
2810 let url = "https://github.com/owner/repo/pull/abc";
2811 let result = parse_pr_number_from_url(url);
2812 assert!(result.is_err());
2813 assert!(result.unwrap_err().contains("not a valid positive integer"));
2814 }
2815
2816 #[test]
2817 fn parse_pr_number_empty_string() {
2818 let result = parse_pr_number_from_url("");
2819 assert!(result.is_err());
2820 assert!(result.unwrap_err().contains("empty"));
2821 }
2822
2823 #[test]
2824 fn parse_pr_number_http_not_https() {
2825 let url = "http://github.com/owner/repo/pull/42";
2827 let result = parse_pr_number_from_url(url);
2828 assert!(result.is_err());
2829 assert!(result
2830 .unwrap_err()
2831 .contains("does not start with 'https://github.com/'"));
2832 }
2833
2834 #[test]
2842 fn format_review_error_summarises_rate_limit_fragment_with_api_retry() {
2843 let raw_fragment = "{\"attempt\":6,\"max_retries\":10,\"retry_delay_ms\":17266.789,\"error_status\":429,\"error\":\"rate_limit\",\"session_id\":\"5c7148ee\"}\n\
2847 {\"type\":\"system\",\"subtype\":\"api_retry\",\"attempt\":7,\"max_retries\":10,\"retry_delay_ms\":35162.66,\"error_status\":429,\"error\":\"rate_limit\",\"session_id\":\"5c7148ee\"}";
2848 let err = runner::agents::AgentError::RateLimit {
2849 message: raw_fragment.to_string(),
2850 };
2851
2852 let stored = format_review_error("minimax", &err);
2853
2854 assert!(
2857 !stored.contains("{\"type\""),
2858 "stored error must not contain raw NDJSON envelope, got: {stored}"
2859 );
2860 assert!(
2861 !stored.contains("subtype"),
2862 "stored error must not contain raw api_retry fragment, got: {stored}"
2863 );
2864 assert!(
2865 !stored.contains("session_id"),
2866 "stored error must not contain session_id from provider payload, got: {stored}"
2867 );
2868 assert_eq!(
2869 stored,
2870 "minimax rate limit: status=429 after 7 attempts (last delay 35s)"
2871 );
2872 }
2873
2874 #[test]
2875 fn format_review_error_summarises_truncated_fragment_without_subtype_marker() {
2876 let raw_fragment = ",\"attempt\":5,\"max_retries\":10,\"retry_delay_ms\":9034.81,\"error_status\":429,\"error\":\"rate_limit\",\"session_id\":\"992e17d3\"";
2881 let err = runner::agents::AgentError::RateLimit {
2882 message: raw_fragment.to_string(),
2883 };
2884
2885 let stored = format_review_error("glm", &err);
2886
2887 assert!(stored.starts_with("glm rate limit:"));
2888 assert!(
2889 !stored.contains("session_id"),
2890 "stored error must not leak session_id from truncated fragment, got: {stored}"
2891 );
2892 assert!(
2895 stored.len() <= 200,
2896 "stored error should be compact (<=200 chars), got {} chars: {stored}",
2897 stored.len()
2898 );
2899 }
2900
2901 #[test]
2902 fn format_review_error_preserves_non_rate_limit_variants() {
2903 let err = runner::agents::AgentError::Auth {
2904 message: "invalid bearer token".to_string(),
2905 };
2906 assert_eq!(
2907 format_review_error("kimi", &err),
2908 "kimi auth error: invalid bearer token"
2909 );
2910
2911 let err = runner::agents::AgentError::Timeout {
2912 elapsed: std::time::Duration::from_secs(1801),
2913 };
2914 assert_eq!(
2915 format_review_error("kimi", &err),
2916 "kimi timed out after 1801s"
2917 );
2918 }
2919
2920 fn git(dir: &std::path::Path, args: &[&str]) {
2921 let mut cmd = std::process::Command::new("git");
2922 cmd.current_dir(dir);
2923 if args.first() == Some(&"commit") {
2925 cmd.args(["-c", "commit.gpgsign=false"]);
2926 }
2927 let status = cmd.args(args).status().unwrap();
2928 assert!(status.success(), "git {:?} failed", args);
2929 }
2930
2931 #[test]
2932 fn test_pull_request_review_requests_changes() {
2933 let review = PullRequestReview {
2934 review: GitHubReview {
2935 id: 1,
2936 user: GitHubUser {
2937 login: "reviewer".to_string(),
2938 },
2939 body: Some("Please fix".to_string()),
2940 state: "CHANGES_REQUESTED".to_string(),
2941 html_url: None,
2942 submitted_at: "2024-01-01T00:00:00Z".to_string(),
2943 commit_id: None,
2944 },
2945 comments: vec![],
2946 };
2947 assert!(review.requests_changes());
2948 }
2949
2950 #[test]
2951 fn test_pull_request_review_does_not_request_changes() {
2952 let review = PullRequestReview {
2953 review: GitHubReview {
2954 id: 1,
2955 user: GitHubUser {
2956 login: "reviewer".to_string(),
2957 },
2958 body: Some("LGTM".to_string()),
2959 state: "APPROVED".to_string(),
2960 html_url: None,
2961 submitted_at: "2024-01-01T00:00:00Z".to_string(),
2962 commit_id: None,
2963 },
2964 comments: vec![],
2965 };
2966 assert!(!review.requests_changes());
2967 }
2968
2969 #[test]
2970 fn test_pull_request_review_actionable_comments_filters_empty_and_replies() {
2971 let review = PullRequestReview {
2972 review: GitHubReview {
2973 id: 1,
2974 user: GitHubUser {
2975 login: "reviewer".to_string(),
2976 },
2977 body: None,
2978 state: "CHANGES_REQUESTED".to_string(),
2979 html_url: None,
2980 submitted_at: "2024-01-01T00:00:00Z".to_string(),
2981 commit_id: None,
2982 },
2983 comments: vec![
2984 GitHubReviewComment {
2985 id: 1,
2986 user: GitHubUser {
2987 login: "reviewer".to_string(),
2988 },
2989 body: "Fix this issue".to_string(),
2990 path: "src/main.rs".to_string(),
2991 line: Some(10),
2992 original_line: Some(10),
2993 commit_id: "abc".to_string(),
2994 original_commit_id: "abc".to_string(),
2995 html_url: "url".to_string(),
2996 created_at: "2024-01-01T00:00:00Z".to_string(),
2997 updated_at: "2024-01-01T00:00:00Z".to_string(),
2998 in_reply_to_id: None,
2999 diff_hunk: Some(
3000 "@@ -8,5 +8,5 @@ fn main() {\n- let x = 1;\n+ let x = 2;".to_string(),
3001 ),
3002 },
3003 GitHubReviewComment {
3004 id: 2,
3005 user: GitHubUser {
3006 login: "reviewer".to_string(),
3007 },
3008 body: "".to_string(),
3009 path: "src/lib.rs".to_string(),
3010 line: Some(20),
3011 original_line: Some(20),
3012 commit_id: "abc".to_string(),
3013 original_commit_id: "abc".to_string(),
3014 html_url: "url".to_string(),
3015 created_at: "2024-01-01T00:00:00Z".to_string(),
3016 updated_at: "2024-01-01T00:00:00Z".to_string(),
3017 in_reply_to_id: None,
3018 diff_hunk: None,
3019 },
3020 GitHubReviewComment {
3021 id: 3,
3022 user: GitHubUser {
3023 login: "reviewer".to_string(),
3024 },
3025 body: "Reply to this".to_string(),
3026 path: "src/lib.rs".to_string(),
3027 line: Some(30),
3028 original_line: Some(30),
3029 commit_id: "abc".to_string(),
3030 original_commit_id: "abc".to_string(),
3031 html_url: "url".to_string(),
3032 created_at: "2024-01-01T00:00:00Z".to_string(),
3033 updated_at: "2024-01-01T00:00:00Z".to_string(),
3034 in_reply_to_id: Some(1),
3035 diff_hunk: None,
3036 },
3037 ],
3038 };
3039 let actionable = review.actionable_comments();
3040 assert_eq!(actionable.len(), 1);
3041 assert_eq!(actionable[0].id, 1);
3042 assert_eq!(actionable[0].body, "Fix this issue");
3043 assert_eq!(actionable[0].path, "src/main.rs");
3044 assert_eq!(
3045 actionable[0].diff_hunk.as_ref().unwrap(),
3046 "@@ -8,5 +8,5 @@ fn main() {\n- let x = 1;\n+ let x = 2;"
3047 );
3048 }
3049
3050 #[test]
3051 fn test_router_config_model_for_complexity_returns_none_without_config() {
3052 let cfg = RouterConfig::default();
3055 assert!(cfg.model_for_complexity("claude", "simple", "").is_none());
3056 assert!(cfg.model_for_complexity("opencode", "review", "").is_none());
3057 assert!(cfg
3058 .model_for_complexity("unknown_agent_xyz", "simple", "")
3059 .is_none());
3060 }
3061
3062 #[test]
3063 fn review_started_comment_includes_agent_and_model() {
3064 let expected_footer = attribution_footer("Review started", "kimi", Some("opus"));
3065 let expected = format!("🔍 Automated review started{}", expected_footer);
3066 assert_eq!(review_started_comment("kimi", "opus"), expected);
3067 }
3068
3069 fn no_code_review_task(id: &str) -> ExternalTask {
3072 ExternalTask {
3073 id: crate::backends::ExternalId(id.to_string()),
3074 title: "opencode discovery timeout".to_string(),
3075 body: "".to_string(),
3076 state: "open".to_string(),
3077 labels: vec![],
3078 author: "tester".to_string(),
3079 created_at: "2026-01-01T00:00:00Z".to_string(),
3080 updated_at: "2026-01-01T00:00:00Z".to_string(),
3081 url: "https://example.com/issues/1".to_string(),
3082 }
3083 }
3084
3085 #[tokio::test]
3086 async fn build_review_context_reroutes_external_task_with_no_code() {
3087 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3088 let repo = "owner/repo";
3089 let ext = no_code_review_task("3617");
3090 store.ensure_external_task(repo, &ext).await.unwrap();
3091
3092 let backend = Arc::new(crate::backends::test_helpers::NoopBackend);
3093 let task_manager = Arc::new(TaskManager::with_store(
3094 backend,
3095 Arc::clone(&store),
3096 repo.to_string(),
3097 ));
3098 let router = Arc::new(RwLock::new(Router::new(RouterConfig::default())));
3099
3100 let phase = build_review_context(&ext, repo, &router, &task_manager, &store)
3101 .await
3102 .unwrap();
3103
3104 assert!(
3105 matches!(phase, ReviewPhase::EarlyReturn(ReviewDecision::Rerouted)),
3106 "external task with no worktree/branch/pr must reroute, not skip-to-done"
3107 );
3108
3109 let stored = store
3110 .get_by_external_id(repo, &ext.id.0)
3111 .await
3112 .unwrap()
3113 .expect("task must still exist");
3114 assert_eq!(stored.status, TaskStatus::New);
3115 assert!(stored.agent.is_none());
3116 assert!(stored.model.is_none());
3117 }
3118
3119 #[tokio::test]
3120 async fn build_review_context_blocks_external_task_after_max_reroutes() {
3121 let temp_home = TempDir::new().unwrap();
3124 let orch_dir = temp_home.path().join(".orch");
3125 std::fs::create_dir_all(&orch_dir).unwrap();
3126 let old_orch_home = std::env::var_os("ORCH_HOME");
3127 std::env::set_var("ORCH_HOME", &orch_dir);
3128
3129 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3130 let repo = "owner/repo";
3131 let ext = no_code_review_task("3618");
3132 store.ensure_external_task(repo, &ext).await.unwrap();
3133
3134 let backend = Arc::new(crate::backends::test_helpers::NoopBackend);
3135 let task_manager = Arc::new(TaskManager::with_store(
3136 backend,
3137 Arc::clone(&store),
3138 repo.to_string(),
3139 ));
3140 let router = Arc::new(RwLock::new(Router::new(RouterConfig::default())));
3141
3142 for _ in 0..3 {
3144 let phase = build_review_context(&ext, repo, &router, &task_manager, &store)
3145 .await
3146 .unwrap();
3147 if let ReviewPhase::EarlyReturn(ReviewDecision::Blocked(_)) = phase {
3148 break;
3149 }
3150 }
3151
3152 let stored = store
3153 .get_by_external_id(repo, &ext.id.0)
3154 .await
3155 .unwrap()
3156 .expect("task must still exist");
3157 assert_eq!(stored.status, TaskStatus::Blocked);
3158
3159 if let Some(old) = old_orch_home {
3160 std::env::set_var("ORCH_HOME", old);
3161 } else {
3162 std::env::remove_var("ORCH_HOME");
3163 }
3164 }
3165
3166 #[tokio::test]
3167 async fn build_review_context_marks_internal_task_done_with_no_code() {
3168 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3169 let repo = "owner/repo";
3170 let internal_id = store
3171 .create_internal(repo, "Daily retrospective", "", "job", "job-1", None)
3172 .await
3173 .unwrap();
3174 let ext_id = format!("internal:{internal_id}");
3175 let ext = no_code_review_task(&ext_id);
3176
3177 let backend = Arc::new(crate::backends::test_helpers::NoopBackend);
3178 let task_manager = Arc::new(TaskManager::with_store(
3179 backend,
3180 Arc::clone(&store),
3181 repo.to_string(),
3182 ));
3183 let router = Arc::new(RwLock::new(Router::new(RouterConfig::default())));
3184
3185 let phase = build_review_context(&ext, repo, &router, &task_manager, &store)
3186 .await
3187 .unwrap();
3188
3189 assert!(matches!(
3190 phase,
3191 ReviewPhase::EarlyReturn(ReviewDecision::Skipped)
3192 ));
3193 let stored = store.get(internal_id).await.unwrap();
3194 assert_eq!(stored.status, TaskStatus::Done);
3195 }
3196
3197 #[test]
3198 fn no_code_reroute_skip_check_rejects_agent_stream_disconnects() {
3199 let last_error =
3200 "codex failed: Reconnecting... (stream disconnected before completion: Broken pipe)";
3201 assert!(!should_skip_no_code_reroute_increment(last_error));
3202 }
3203
3204 #[test]
3213 fn review_fallback_scan_ignores_rate_limit_word_buried_in_long_report() {
3214 let padding = "x".repeat(10_000);
3215 let text_for_review = format!(
3216 "{padding}\n+| codex | gpt-5.4 | failed | 1 |\n+| codex | gpt-5.5 | failed / aborted / (empty) | 1 each |\n+| kimi | opus | rate_limit | 1 |\n{padding}"
3217 );
3218
3219 let scan_head = runner::agents::patterns::safe_head(
3220 &text_for_review,
3221 runner::agents::patterns::RATE_LIMIT_SCAN_HEAD_BYTES,
3222 );
3223 let scan_tail = runner::agents::patterns::safe_tail(
3224 &text_for_review,
3225 runner::agents::patterns::RATE_LIMIT_SCAN_TAIL_BYTES,
3226 );
3227
3228 assert!(runner::agents::patterns::detect_rate_limit(scan_head).is_none());
3229 assert!(runner::agents::patterns::detect_rate_limit(scan_tail).is_none());
3230 }
3231
3232 #[test]
3233 fn review_fallback_scan_still_detects_real_rate_limit_at_tail() {
3234 let padding = "x".repeat(10_000);
3235 let text_for_review =
3236 format!("{padding}\nError: rate limit exceeded, please try again later");
3237
3238 let scan_tail = runner::agents::patterns::safe_tail(
3239 &text_for_review,
3240 runner::agents::patterns::RATE_LIMIT_SCAN_TAIL_BYTES,
3241 );
3242
3243 assert!(runner::agents::patterns::detect_rate_limit(scan_tail).is_some());
3244 }
3245
3246 #[tokio::test]
3247 async fn recover_missing_review_worktree_recreates_worktree() {
3248 let temp_home = TempDir::new().unwrap();
3249 let orch_dir = temp_home.path().join(".orch");
3250 std::fs::create_dir_all(&orch_dir).unwrap();
3251 let old_orch_home = std::env::var_os("ORCH_HOME");
3252 std::env::set_var("ORCH_HOME", &orch_dir);
3253
3254 let project_dir = temp_home.path().join("project");
3255 std::fs::create_dir_all(&project_dir).unwrap();
3256 git(&project_dir, &["init", "-b", "main"]);
3257 git(&project_dir, &["config", "user.name", "Test User"]);
3258 git(&project_dir, &["config", "user.email", "test@example.com"]);
3259 std::fs::write(project_dir.join("README.md"), "hello").unwrap();
3260 git(&project_dir, &["add", "README.md"]);
3261 git(&project_dir, &["commit", "-m", "init"]);
3262
3263 std::fs::write(
3264 orch_dir.join("config.yml"),
3265 format!("projects:\n - {}\n", project_dir.display()),
3266 )
3267 .unwrap();
3268 std::fs::write(project_dir.join(".orch.yml"), "gh:\n repo: owner/repo\n").unwrap();
3269
3270 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3271 let repo = "owner/repo";
3272 store
3273 .create_internal(repo, "Fix review recovery", "", "review", "review-1", None)
3274 .await
3275 .unwrap();
3276 let ext = ExternalTask {
3277 id: crate::backends::ExternalId("1".to_string()),
3278 title: "Fix review recovery".to_string(),
3279 body: "".to_string(),
3280 state: "open".to_string(),
3281 labels: vec![],
3282 author: "tester".to_string(),
3283 created_at: "2026-01-01T00:00:00Z".to_string(),
3284 updated_at: "2026-01-01T00:00:00Z".to_string(),
3285 url: "https://example.com/issues/1".to_string(),
3286 };
3287 store.ensure_external_task(repo, &ext).await.unwrap();
3288
3289 let recovered = recover_missing_review_worktree(&ext, repo, &store)
3290 .await
3291 .unwrap();
3292
3293 assert!(recovered.work_dir.exists());
3294 assert!(recovered.branch.starts_with("gh-issue-1-"));
3295
3296 if let Some(old) = old_orch_home {
3297 std::env::set_var("ORCH_HOME", old);
3298 } else {
3299 std::env::remove_var("ORCH_HOME");
3300 }
3301 }
3302
3303 #[tokio::test]
3306 async fn count_ahead_commits_returns_zero_when_no_ahead_commits() {
3307 let temp = TempDir::new().unwrap();
3308 let dir = temp.path();
3309
3310 let bare = temp.path().join("bare.git");
3312 std::fs::create_dir_all(&bare).unwrap();
3313 std::process::Command::new("git")
3314 .args(["init", "--bare"])
3315 .current_dir(&bare)
3316 .output()
3317 .unwrap();
3318
3319 git(dir, &["init", "-b", "main"]);
3320 git(dir, &["config", "user.name", "Test"]);
3321 git(dir, &["config", "user.email", "test@test.com"]);
3322 std::fs::write(dir.join("f"), "a").unwrap();
3323 git(dir, &["add", "f"]);
3324 git(dir, &["commit", "-m", "init"]);
3325 std::process::Command::new("git")
3326 .args(["remote", "add", "origin", bare.to_str().unwrap()])
3327 .current_dir(dir)
3328 .output()
3329 .unwrap();
3330 std::process::Command::new("git")
3331 .args(["push", "origin", "main"])
3332 .current_dir(dir)
3333 .output()
3334 .unwrap();
3335
3336 let result = count_ahead_commits(dir, "main").await;
3338 assert!(
3339 result.is_ok(),
3340 "expected Ok when origin/main exists, got {result:?}"
3341 );
3342 assert_eq!(
3343 result.unwrap(),
3344 0,
3345 "expected 0 commits ahead when branch matches origin/main"
3346 );
3347 }
3348
3349 #[tokio::test]
3350 async fn count_ahead_commits_returns_error_when_remote_ref_missing() {
3351 let temp = TempDir::new().unwrap();
3352 let dir = temp.path();
3353 git(dir, &["init", "-b", "main"]);
3354 git(dir, &["config", "user.name", "Test"]);
3355 git(dir, &["config", "user.email", "test@test.com"]);
3356 std::fs::write(dir.join("f"), "a").unwrap();
3357 git(dir, &["add", "f"]);
3358 git(dir, &["commit", "-m", "init"]);
3359 let result = count_ahead_commits(dir, "main").await;
3361 assert!(
3362 result.is_err(),
3363 "expected Err when origin/main does not exist, got {result:?}"
3364 );
3365 let err = result.unwrap_err();
3366 assert!(
3367 err.contains("git rev-list exited"),
3368 "error should mention exit failure, got: {err}"
3369 );
3370 }
3371
3372 #[tokio::test]
3373 async fn count_ahead_commits_returns_error_when_git_cannot_start() {
3374 let temp = TempDir::new().unwrap();
3376 let dir = temp.path();
3377 let result = count_ahead_commits(dir, "main").await;
3379 assert!(
3380 result.is_err(),
3381 "expected Err when directory is not a git repo, got {result:?}"
3382 );
3383 }
3384
3385 #[serial_test::serial(cooldown_state)]
3395 #[tokio::test]
3396 async fn review_invalid_response_applies_persistent_model_cooldown() {
3397 crate::engine::cooldown::reset_global_state().await;
3398 let agent = "opencode-3314-review-invalid";
3399 let model = "opencode/north-mini-code-free";
3400 let key = format!("{agent}:{model}");
3401
3402 assert!(
3403 !crate::engine::cooldown::is_model_in_cooldown(agent, model),
3404 "model should not be in cooldown before record_review_agent_failure"
3405 );
3406
3407 let err = runner::agents::AgentError::InvalidResponse {
3408 raw: "empty review output".to_string(),
3409 };
3410 record_review_agent_failure("test-3314-a", agent, Some(model), &err).await;
3411
3412 let now = chrono::Utc::now().timestamp();
3413 let until = crate::engine::cooldown::cooldown_until(&key)
3414 .expect("review InvalidResponse should set model cooldown");
3415 let remaining = until.saturating_sub(now);
3416
3417 assert!(
3420 remaining >= crate::engine::cooldown::PERSISTENT_MODEL_BACKOFF_BASE_SECS - 30,
3421 "review InvalidResponse must apply ~4h persistent cooldown (issue #3314), \
3422 got {remaining}s — may have used 5-min standard backoff instead"
3423 );
3424 }
3425
3426 #[serial_test::serial(cooldown_state)]
3434 #[tokio::test]
3435 async fn review_persistent_cooldown_survives_after_first_failure() {
3436 crate::engine::cooldown::reset_global_state().await;
3437 let agent = "opencode-3314-review-parse";
3438 let model = "opencode/north-mini-code-free";
3439 let key = format!("{agent}:{model}");
3440
3441 let err = runner::agents::AgentError::InvalidResponse {
3443 raw: "failed to parse review response".to_string(),
3444 };
3445 record_review_agent_failure("test-3314-b", agent, Some(model), &err).await;
3446
3447 let now = chrono::Utc::now().timestamp();
3448 let until1 = crate::engine::cooldown::cooldown_until(&key)
3449 .expect("first failure should set cooldown");
3450 let remaining1 = until1.saturating_sub(now);
3451
3452 crate::engine::cooldown::record_agent_success(agent, model).await;
3455
3456 assert!(
3462 remaining1 >= crate::engine::cooldown::PERSISTENT_MODEL_BACKOFF_BASE_SECS - 30,
3463 "after success reset, first-failure cooldown should still be ~4h, got {remaining1}s"
3464 );
3465 }
3466
3467 #[serial_test::serial(cooldown_state)]
3473 #[tokio::test]
3474 async fn review_inline_scan_billing_cycle_message_gets_model_scoped_cooldown() {
3475 crate::engine::cooldown::reset_global_state().await;
3476 let agent = "kimi-3551-review-inline";
3477 let model = "opus";
3478 let key = format!("{agent}:{model}");
3479
3480 let text_for_review = "kimi rate limit: Failed to authenticate. API Error: 403 \
3482 You've reached your usage limit for this billing cycle. Your quota will be \
3483 refreshed next cycle.";
3484
3485 let scan_tail = runner::agents::patterns::safe_tail(
3486 text_for_review,
3487 runner::agents::patterns::RATE_LIMIT_SCAN_TAIL_BYTES,
3488 );
3489 let err = runner::agents::patterns::detect_rate_limit(scan_tail)
3490 .expect("billing-cycle message must still be caught by the rate-limit scan");
3491
3492 assert_eq!(
3493 outcome_for_agent_error(&err),
3494 "billing_cycle_exhausted",
3495 "inline-scanned billing-cycle message must not be stored as plain rate_limit"
3496 );
3497
3498 record_review_agent_failure("test-3551", agent, Some(model), &err).await;
3499
3500 let now = chrono::Utc::now().timestamp();
3501 let until = crate::engine::cooldown::cooldown_until(&key)
3502 .expect("billing-cycle exhaustion must set a model-scoped cooldown");
3503 let remaining = until.saturating_sub(now);
3504 assert!(
3509 remaining >= crate::engine::cooldown::PERSISTENT_MODEL_BACKOFF_BASE_SECS - 30,
3510 "billing-cycle exhaustion for a known model must apply the ~4h persistent \
3511 model cooldown, got {remaining}s — may have used the generic short agent \
3512 cooldown instead"
3513 );
3514 }
3515
3516 fn test_review_task(id: &str) -> ExternalTask {
3519 ExternalTask {
3520 id: crate::backends::ExternalId(id.to_string()),
3521 title: "Test review task".to_string(),
3522 body: "".to_string(),
3523 state: "open".to_string(),
3524 labels: vec![],
3525 author: "tester".to_string(),
3526 created_at: "2026-01-01T00:00:00Z".to_string(),
3527 updated_at: "2026-01-01T00:00:00Z".to_string(),
3528 url: "https://example.com/issues/1".to_string(),
3529 }
3530 }
3531
3532 fn test_review_context(agent: &str, model: &str, work_dir: &std::path::Path) -> ReviewContext {
3533 ReviewContext {
3534 store_id: None,
3535 had_prev_push_failure: false,
3536 review_attempt: 1,
3537 worktree_path: work_dir.to_path_buf(),
3538 branch_name: "gh-issue-1-test".to_string(),
3539 pr_number: 1,
3540 default_branch: "main".to_string(),
3541 review_agent: agent.to_string(),
3542 review_model: Some(model.to_string()),
3543 review_task_id: "review-1".to_string(),
3544 review_attempt_dir: work_dir.to_path_buf(),
3545 output_file: work_dir.join("output.txt"),
3546 invocation: runner::agent::AgentInvocation {
3547 agent: agent.to_string(),
3548 model: Some(model.to_string()),
3549 work_dir: work_dir.to_path_buf(),
3550 system_prompt: String::new(),
3551 agent_message: String::new(),
3552 task_id: "review-1".to_string(),
3553 disallowed_tools: vec![],
3554 git_author_name: "Test".to_string(),
3555 git_author_email: "test@example.com".to_string(),
3556 output_file: work_dir.join("output.txt"),
3557 timeout_seconds: 0,
3558 repo: "owner/repo".to_string(),
3559 attempt: 1,
3560 },
3561 same_agent_fallback: false,
3562 }
3563 }
3564
3565 fn test_parsed_review(decision: ReviewDecision) -> ParsedReview {
3566 ParsedReview {
3567 run_id: None,
3568 exit_code: 0,
3569 stderr: String::new(),
3570 raw_output: String::new(),
3571 text_for_review: String::new(),
3572 token_usage: RunTokenUsage::default(),
3573 review_notes_for_comment: String::new(),
3574 decision,
3575 }
3576 }
3577
3578 fn write_review_output(dir: &std::path::Path, stdout: &str, exit_code: i32) {
3579 std::fs::write(dir.join("output.txt"), stdout).unwrap();
3580 std::fs::write(dir.join("exit.txt"), exit_code.to_string()).unwrap();
3581 std::fs::write(dir.join("stderr.txt"), "").unwrap();
3582 }
3583
3584 #[serial_test::serial(cooldown_state)]
3588 #[tokio::test]
3589 async fn parse_review_output_mid_thought_is_incomplete_without_persistent_cooldown() {
3590 crate::engine::cooldown::reset_global_state().await;
3591 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3592 let work_dir = TempDir::new().unwrap();
3593 let agent = "kimi";
3594 let model = "opus-3734-mid-thought";
3595 let ctx = test_review_context(agent, model, work_dir.path());
3596 let redraw = "^D\u{0008}".repeat(20);
3597 let ndjson = format!(
3598 "{}\n{}\n{}\n{}\n{}",
3599 r#"{"type":"system","subtype":"thinking_tokens","tokens":1945}"#,
3600 r#"{"type":"assistant","message":{"content":[{"type":"text","text":"CI passes. Now verifying callers and the lease logic edge cases:"}],"stop_reason":null,"usage":{"input_tokens":10,"output_tokens":0}}}"#,
3601 r#"{"type":"assistant","message":{"content":[{"type":"thinking","thinking":"checking lease edge cases"}],"stop_reason":null,"usage":{"input_tokens":10,"output_tokens":0}}}"#,
3602 "\x1b[?25h",
3603 redraw,
3604 );
3605 write_review_output(work_dir.path(), &ndjson, 0);
3606
3607 let task = test_review_task("3734");
3608 let phase = parse_review_output(
3609 &task,
3610 "owner/repo",
3611 &ctx,
3612 ReviewRun { run_id: None },
3613 &store,
3614 )
3615 .await;
3616
3617 match phase {
3618 ReviewPhase::EarlyReturn(ReviewDecision::Failed(reason)) => {
3619 assert!(
3620 reason.contains("incomplete:"),
3621 "mid-thought review must be stored as incomplete, got {reason}"
3622 );
3623 }
3624 ReviewPhase::EarlyReturn(_) => panic!("expected Failed(incomplete)"),
3625 ReviewPhase::Ready(_) => panic!("expected Failed(incomplete), got Ready"),
3626 }
3627 assert!(
3628 !crate::engine::cooldown::is_model_in_cooldown(agent, model),
3629 "mid-thought incomplete must not apply persistent model cooldown"
3630 );
3631 }
3632
3633 #[serial_test::serial(cooldown_state)]
3636 #[tokio::test]
3637 async fn parse_review_output_tool_use_without_envelope_is_incomplete() {
3638 crate::engine::cooldown::reset_global_state().await;
3639 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3640 let work_dir = TempDir::new().unwrap();
3641 let agent = "kimi";
3642 let model = "opus-3734-tool-use";
3643 let ctx = test_review_context(agent, model, work_dir.path());
3644 let ndjson = r#"{"type":"assistant","message":{"content":[{"type":"tool_use","name":"Read","input":{"file_path":"src/lib.rs"}}]}}"#;
3645 write_review_output(work_dir.path(), ndjson, 0);
3646
3647 let task = test_review_task("3734-tool");
3648 let phase = parse_review_output(
3649 &task,
3650 "owner/repo",
3651 &ctx,
3652 ReviewRun { run_id: None },
3653 &store,
3654 )
3655 .await;
3656
3657 match phase {
3658 ReviewPhase::EarlyReturn(ReviewDecision::Failed(reason)) => {
3659 assert!(
3660 reason.contains("incomplete:"),
3661 "tool_use without envelope must be incomplete, got {reason}"
3662 );
3663 }
3664 ReviewPhase::EarlyReturn(_) => panic!("expected Failed(incomplete)"),
3665 ReviewPhase::Ready(_) => panic!("expected Failed(incomplete), got Ready"),
3666 }
3667 assert!(!crate::engine::cooldown::is_model_in_cooldown(agent, model));
3668 }
3669
3670 #[serial_test::serial(cooldown_state)]
3673 #[tokio::test]
3674 async fn parse_review_output_unparseable_envelope_applies_persistent_cooldown() {
3675 crate::engine::cooldown::reset_global_state().await;
3676 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3677 let work_dir = TempDir::new().unwrap();
3678 let agent = "kimi";
3679 let model = "opus-3734-format";
3680 let ctx = test_review_context(agent, model, work_dir.path());
3681 let ndjson = concat!(
3682 r#"{"type":"assistant","message":{"content":[{"type":"text","text":"I have finished looking at the code."}]}}"#,
3683 "\n",
3684 r#"{"type":"result","subtype":"success","is_error":false,"result":"I have finished looking at the code."}"#,
3685 );
3686 write_review_output(work_dir.path(), ndjson, 0);
3687
3688 let task = test_review_task("3734-format");
3689 let phase = parse_review_output(
3690 &task,
3691 "owner/repo",
3692 &ctx,
3693 ReviewRun { run_id: None },
3694 &store,
3695 )
3696 .await;
3697
3698 match phase {
3699 ReviewPhase::EarlyReturn(ReviewDecision::Failed(reason)) => {
3700 assert!(
3701 reason.contains("parse error"),
3702 "finished unparseable envelope must stay a parse error, got {reason}"
3703 );
3704 }
3705 ReviewPhase::EarlyReturn(_) => panic!("expected Failed(parse error)"),
3706 ReviewPhase::Ready(_) => panic!("expected Failed(parse error), got Ready"),
3707 }
3708 assert!(
3709 crate::engine::cooldown::is_model_in_cooldown(agent, model),
3710 "finished format failure must apply persistent model cooldown"
3711 );
3712 }
3713
3714 #[serial_test::serial(cooldown_state)]
3717 #[tokio::test]
3718 async fn parse_review_output_missing_envelope_uses_assistant_decision() {
3719 crate::engine::cooldown::reset_global_state().await;
3720 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3721 let work_dir = TempDir::new().unwrap();
3722 let agent = "kimi";
3723 let model = "opus-3734-assistant";
3724 let ctx = test_review_context(agent, model, work_dir.path());
3725 let ndjson = r#"{"type":"assistant","message":{"content":[{"type":"text","text":"{\"decision\":\"approve\",\"notes\":\"LGTM\",\"issues\":[]}"}]}}"#;
3726 write_review_output(work_dir.path(), ndjson, 0);
3727
3728 let task = test_review_task("3734-asst");
3729 let phase = parse_review_output(
3730 &task,
3731 "owner/repo",
3732 &ctx,
3733 ReviewRun { run_id: None },
3734 &store,
3735 )
3736 .await;
3737
3738 match phase {
3739 ReviewPhase::Ready(parsed) => {
3740 assert!(matches!(parsed.decision, ReviewDecision::Approve));
3741 }
3742 ReviewPhase::EarlyReturn(_) => {
3743 panic!("expected Ready(Approve) from assistant text, got EarlyReturn")
3744 }
3745 }
3746 assert!(!crate::engine::cooldown::is_model_in_cooldown(agent, model));
3747 }
3748
3749 #[serial_test::serial(cooldown_state)]
3752 #[tokio::test]
3753 async fn parse_review_output_does_not_treat_task_done_synthesis_as_approval() {
3754 crate::engine::cooldown::reset_global_state().await;
3755 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3756 let work_dir = TempDir::new().unwrap();
3757 let agent = "kimi";
3758 let model = "opus-3734-done";
3759 let ctx = test_review_context(agent, model, work_dir.path());
3760 let ndjson = r#"{"type":"assistant","message":{"content":[{"type":"text","text":"analyzing the diff"}]}}"#;
3761 write_review_output(work_dir.path(), ndjson, 0);
3762
3763 let task = test_review_task("3734-done");
3764 let phase = parse_review_output(
3765 &task,
3766 "owner/repo",
3767 &ctx,
3768 ReviewRun { run_id: None },
3769 &store,
3770 )
3771 .await;
3772
3773 match phase {
3774 ReviewPhase::EarlyReturn(ReviewDecision::Failed(reason)) => {
3775 assert!(
3776 reason.contains("parse error"),
3777 "unfinished prose without a decision must not approve, got {reason}"
3778 );
3779 }
3780 ReviewPhase::Ready(_) => {
3781 panic!("task-path status=done synthesis must not become a review approval")
3782 }
3783 ReviewPhase::EarlyReturn(_) => panic!("expected Failed(parse error)"),
3784 }
3785 }
3786
3787 #[serial_test::serial(cooldown_state)]
3793 #[tokio::test]
3794 async fn finalize_review_run_success_resets_generic_model_failure_count() {
3795 crate::engine::cooldown::reset_global_state().await;
3796 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3797 crate::engine::cooldown::init_cooldown_store(store.clone()).await;
3798
3799 let agent = "test-3478-agent";
3800 let model = "test-3478-model";
3801 let failure_key = format!("failure_count:{agent}:{model}");
3802
3803 let err = runner::agents::AgentError::AgentFailed {
3807 message: "agent reported a generic failure".to_string(),
3808 };
3809 record_review_agent_failure("task-3478-a", agent, Some(model), &err).await;
3810 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
3811 assert_eq!(
3812 store.kv_get(&failure_key).await.unwrap().as_deref(),
3813 Some("1"),
3814 "sanity check: failure should have incremented the persisted count"
3815 );
3816
3817 let work_dir = TempDir::new().unwrap();
3818 let ctx = test_review_context(agent, model, work_dir.path());
3819 let parsed = test_parsed_review(ReviewDecision::Approve);
3820 let task = test_review_task("1");
3821
3822 finalize_review_run(&task, &ctx, &parsed, &store).await;
3823 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
3824
3825 assert_eq!(
3826 store.kv_get(&failure_key).await.unwrap().as_deref(),
3827 Some("0"),
3828 "successful review must reset failure_count:{{agent}}:{{model}} (issue #3478)"
3829 );
3830 }
3831
3832 #[serial_test::serial(cooldown_state)]
3838 #[tokio::test]
3839 async fn finalize_review_run_interleaved_success_keeps_generic_cooldown_at_base() {
3840 crate::engine::cooldown::reset_global_state().await;
3841 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3842 crate::engine::cooldown::init_cooldown_store(store.clone()).await;
3843
3844 let agent = "test-3478-interleave-agent";
3845 let model = "test-3478-interleave-model";
3846 let cooldown_key = format!("{agent}:{model}");
3847
3848 let work_dir = TempDir::new().unwrap();
3849 let ctx = test_review_context(agent, model, work_dir.path());
3850 let task = test_review_task("2");
3851
3852 for i in 0..4 {
3853 let err = runner::agents::AgentError::AgentFailed {
3854 message: format!("failure {i}"),
3855 };
3856 record_review_agent_failure("task-3478-b", agent, Some(model), &err).await;
3857
3858 let now = chrono::Utc::now().timestamp();
3859 let until = crate::engine::cooldown::cooldown_until(&cooldown_key)
3860 .expect("failure should set cooldown");
3861 let remaining = until.saturating_sub(now);
3862
3863 assert!(
3864 remaining <= crate::engine::cooldown::BACKOFF_BASE_SECS + 60,
3865 "failure #{} must stay at base backoff since the prior success reset the \
3866 counter (issue #3478); got {remaining}s, base is {}s — an escalating value \
3867 here means review successes are not resetting failure_count",
3868 i + 1,
3869 crate::engine::cooldown::BACKOFF_BASE_SECS
3870 );
3871
3872 let parsed = test_parsed_review(ReviewDecision::Approve);
3876 finalize_review_run(&task, &ctx, &parsed, &store).await;
3877 tokio::time::sleep(std::time::Duration::from_millis(20)).await;
3878 }
3879 }
3880
3881 #[serial_test::serial(cooldown_state)]
3887 #[tokio::test]
3888 async fn finalize_review_run_success_does_not_reset_persistent_model_failure_count() {
3889 crate::engine::cooldown::reset_global_state().await;
3890 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3891 crate::engine::cooldown::init_cooldown_store(store.clone()).await;
3892
3893 let agent = "test-3571-agent";
3894 let model = "test-3571-model";
3895 let cooldown_key = format!("{agent}:{model}");
3896
3897 let work_dir = TempDir::new().unwrap();
3898 let ctx = test_review_context(agent, model, work_dir.path());
3899 let task = test_review_task("3");
3900
3901 let err = runner::agents::AgentError::InvalidResponse {
3903 raw: "empty review output".to_string(),
3904 };
3905 record_review_agent_failure("task-3571-a", agent, Some(model), &err).await;
3906 tokio::time::sleep(std::time::Duration::from_millis(20)).await;
3907
3908 let parsed = test_parsed_review(ReviewDecision::Approve);
3911 finalize_review_run(&task, &ctx, &parsed, &store).await;
3912 tokio::time::sleep(std::time::Duration::from_millis(20)).await;
3913
3914 let err2 = runner::agents::AgentError::InvalidResponse {
3917 raw: "empty review output again".to_string(),
3918 };
3919 record_review_agent_failure("task-3571-b", agent, Some(model), &err2).await;
3920 tokio::time::sleep(std::time::Duration::from_millis(20)).await;
3921
3922 let now = chrono::Utc::now().timestamp();
3923 let until = crate::engine::cooldown::cooldown_until(&cooldown_key)
3924 .expect("second failure should set cooldown");
3925 let remaining = until.saturating_sub(now);
3926
3927 assert!(
3928 remaining > crate::engine::cooldown::PERSISTENT_MODEL_BACKOFF_BASE_SECS,
3929 "second persistent model failure should escalate past the 4h base despite an \
3930 intervening success (issue #3571); got {remaining}s, base is {}s",
3931 crate::engine::cooldown::PERSISTENT_MODEL_BACKOFF_BASE_SECS
3932 );
3933 }
3934}