1pub(crate) const MAX_REVIEW_AGENT_FAILURES: u64 = 3;
13
14const MAX_PR_CREATE_FAILURES: u64 = 3;
16
17fn outcome_for_agent_error(err: &runner::agents::AgentError) -> &'static str {
22 match err {
23 runner::agents::AgentError::Timeout { .. } => "timeout",
24 runner::agents::AgentError::RateLimit { message } => {
29 crate::engine::cooldown::detect_credit_exhaustion_outcome(message)
30 .unwrap_or("rate_limit")
31 }
32 runner::agents::AgentError::Auth { message } => {
33 crate::engine::cooldown::detect_credit_exhaustion_outcome(message)
34 .unwrap_or("auth_error")
35 }
36 runner::agents::AgentError::InvalidResponse { .. } => "parse_error",
37 _ => "failed",
38 }
39}
40
41fn format_review_error(agent: &str, err: &runner::agents::AgentError) -> String {
50 match err {
51 runner::agents::AgentError::RateLimit { message } => format!(
52 "{agent} rate limit: {}",
53 runner::fallback::summarize_rate_limit_error(message)
54 ),
55 runner::agents::AgentError::Auth { message } => format!("{agent} auth error: {message}"),
56 runner::agents::AgentError::Timeout { elapsed } => {
57 format!("{agent} timed out after {}s", elapsed.as_secs())
58 }
59 other => format!("{agent} agent error: {other}"),
60 }
61}
62
63pub(crate) fn parse_pr_number_from_url(url: &str) -> Result<u64, String> {
75 let url = url.trim();
76 if url.is_empty() {
77 return Err("URL is empty".to_string());
78 }
79 if !url.starts_with("https://github.com/") {
80 return Err(format!(
81 "URL does not start with 'https://github.com/': {url}"
82 ));
83 }
84 if !url.contains("/pull/") {
85 return Err(format!("URL missing '/pull/' segment: {url}"));
86 }
87 let last_segment = url
88 .trim_end_matches('/')
89 .rsplit('/')
90 .next()
91 .ok_or_else(|| format!("No segment after '/pull/' in URL: {url}"))?;
92 let number_part = last_segment
93 .split(['?', '#'])
94 .next()
95 .filter(|s| !s.is_empty())
96 .ok_or_else(|| format!("PR number segment is empty in URL: {url}"))?;
97 let pr_num: u64 = number_part.parse().map_err(|_| {
98 format!("PR number is not a valid positive integer: '{number_part}' in URL: {url}")
99 })?;
100 if pr_num == 0 {
101 return Err(format!("PR number cannot be zero in URL: {url}"));
102 }
103 Ok(pr_num)
104}
105
106fn review_started_comment(review_agent: &str, review_model: &str) -> String {
107 let footer = attribution_footer("Review started", review_agent, Some(review_model));
108 format!("🔍 Automated review started{}", footer)
109}
110
111fn should_skip_no_code_reroute_increment(last_error: &str) -> bool {
112 crate::engine::runner::git_ops::is_transient_github_api_error(last_error)
113}
114
115use crate::backends::{ExternalBackend, ExternalTask, Status};
116use crate::cmd::CommandErrorContext;
117use crate::config;
118use crate::engine::attribution_footer;
119use crate::engine::auto_merge::{auto_merge_pr, handle_review_changes};
120use crate::engine::runner;
121use crate::engine::runner::worktree;
122use crate::engine::tasks::TaskManager;
123use crate::github::http::GhHttp;
124use crate::store::store_log_activity;
125use crate::store::TaskStore;
126use crate::store::{opt_store_get_task, set_review_session_expected, store_increment};
127use crate::store::{store_set, store_set_result};
128use crate::store::{CompleteRun, RunTokenUsage, StartRun};
129use crate::tmux::TmuxManager;
130use anyhow::Context;
131use std::collections::HashSet;
132use std::sync::Arc;
133use tokio::process::Command;
134use tokio::sync::RwLock;
135
136use super::router::Router;
137
138#[derive(Debug, Clone)]
140pub(crate) enum ReviewDecision {
141 Approve,
143 RequestChanges {
145 notes: String,
146 issues: Vec<crate::engine::runner::response::ReviewIssue>,
147 },
148 Failed(String),
150 Blocked(String),
152 Skipped,
154 Rerouted,
156}
157
158enum EnsurePrResult {
160 Ready(u64),
162 EarlyReturn(ReviewDecision),
164}
165
166enum ReviewPhase<T> {
167 Ready(T),
168 EarlyReturn(ReviewDecision),
169}
170
171struct ReviewContext {
172 store_id: Option<i64>,
173 had_prev_push_failure: bool,
174 review_attempt: u32,
175 worktree_path: std::path::PathBuf,
176 branch_name: String,
177 pr_number: u64,
178 head_sha: Option<String>,
180 default_branch: String,
181 review_agent: String,
182 review_model: Option<String>,
183 review_task_id: String,
184 review_attempt_dir: std::path::PathBuf,
185 output_file: std::path::PathBuf,
186 invocation: runner::agent::AgentInvocation,
187 same_agent_fallback: bool,
188}
189
190impl ReviewContext {
191 fn review_model_str(&self) -> &str {
192 self.review_model.as_deref().unwrap_or("")
193 }
194}
195
196struct ReviewRun {
197 run_id: Option<i64>,
198}
199
200struct ParsedReview {
201 run_id: Option<i64>,
202 exit_code: i32,
203 stderr: String,
204 raw_output: String,
205 text_for_review: String,
206 token_usage: RunTokenUsage,
207 review_notes_for_comment: String,
208 decision: ReviewDecision,
209}
210
211fn build_review_disallowed_tools(worktree_path: &std::path::Path) -> Vec<String> {
212 let mut tools = vec![
213 "Bash(rm *)".to_string(),
214 "Bash(rm -*)".to_string(),
215 "Bash(git push*)".to_string(),
216 ];
217 let orch_yml = worktree_path.join(".orch.yml");
218 let orch_yml_str = orch_yml.to_string_lossy();
219 tools.extend([
220 format!("Write({orch_yml_str})"),
221 format!("Edit({orch_yml_str})"),
222 ]);
223 if let Ok(orch_home) = crate::home::orch_home() {
224 for path in [
225 orch_home.join("config.yml"),
226 orch_home.join("config.example.yml"),
227 ] {
228 let path_str = path.to_string_lossy();
229 tools.extend([
230 format!("Read({path_str})"),
231 format!("Write({path_str})"),
232 format!("Edit({path_str})"),
233 ]);
234 }
235 }
236 tools
237}
238
239async fn select_review_agent(
240 task_id: &str,
241 task_agent: &str,
242 store_id: Option<i64>,
243 router: &Arc<RwLock<Router>>,
244 store: &Arc<TaskStore>,
245) -> anyhow::Result<(String, Option<String>, bool)> {
246 let mut exclude_set: HashSet<String> = HashSet::new();
247 if !task_agent.is_empty() {
248 exclude_set.insert(task_agent.to_string());
249 }
250 if let Some(store_id) = store_id {
251 if let Ok(failed) = store.previous_review_agents(store_id).await {
252 for agent in failed {
253 exclude_set.insert(agent);
254 }
255 }
256 }
257
258 let mut r = router.write().await;
259 let mut tried_agents: HashSet<String> = HashSet::new();
260 let mut chosen_agent: Option<String> = None;
261 let mut chosen_model: Option<String> = None;
262 let available_count = r.available_agents.len().max(1);
263
264 loop {
265 let tmp_exclude_refs: Vec<&str> = exclude_set.iter().map(|s| s.as_str()).collect();
266 let agent = match r.next_round_robin_agent(&tmp_exclude_refs, "review") {
267 Some(a) => a,
268 None => break,
269 };
270
271 if tried_agents.contains(&agent) {
272 break;
273 }
274 tried_agents.insert(agent.clone());
275
276 let model = r.config.model_for_complexity(&agent, "review", task_id);
277 if model.is_some() {
278 chosen_agent = Some(agent);
279 chosen_model = model;
280 break;
281 }
282
283 exclude_set.insert(agent);
284 if exclude_set.len() >= available_count {
285 break;
286 }
287 }
288
289 if let Some(agent) = chosen_agent {
290 let same = !task_agent.is_empty() && agent == task_agent;
291 Ok((agent, chosen_model, same))
292 } else {
293 let final_exclude_refs: Vec<&str> = exclude_set.iter().map(|s| s.as_str()).collect();
294 let fallback_agent = r
295 .next_round_robin_agent(&final_exclude_refs, "review")
296 .unwrap_or_else(|| "claude".to_string());
297 let fallback_model = r
298 .config
299 .model_for_complexity(&fallback_agent, "review", task_id);
300 if fallback_model.is_none() {
301 anyhow::bail!(
302 "no review agent with a configured review model is available for task {}",
303 task_id
304 );
305 }
306 let same = !task_agent.is_empty() && fallback_agent == task_agent;
307 Ok((fallback_agent, fallback_model, same))
308 }
309}
310
311#[allow(clippy::too_many_arguments)]
312async fn build_review_context(
313 task: &ExternalTask,
314 repo: &str,
315 router: &Arc<RwLock<Router>>,
316 task_manager: &Arc<TaskManager>,
317 store: &Arc<TaskStore>,
318) -> anyhow::Result<ReviewPhase<ReviewContext>> {
319 let stored_task = store
320 .get_by_external_id(repo, &task.id.0)
321 .await
322 .with_context(|| format!("store lookup failed for task {}", task.id.0))?;
323
324 let store_id: Option<i64> = stored_task.as_ref().map(|t| t.id);
325 let had_prev_push_failure = stored_task
326 .as_ref()
327 .is_some_and(|t| t.last_error.contains("push failed"));
328 let review_attempt: u32 = match store_increment(
329 &Some(Arc::clone(store)),
330 repo,
331 &task.id.0,
332 "review_invocations",
333 )
334 .await
335 {
336 Ok(v) => v as u32,
337 Err(e) => {
338 tracing::warn!(task_id = task.id.0, err = %e, "failed to increment review_invocations — using fallback attempt=1");
339 1
340 }
341 };
342
343 let no_code_review = stored_task.as_ref().is_some_and(|t| {
344 t.worktree.trim().is_empty() && t.branch.trim().is_empty() && t.pr_number.is_none()
345 });
346 if no_code_review {
347 if crate::engine::tasks::is_internal_id(&task.id.0) {
348 tracing::info!(
350 task_id = task.id.0,
351 "no worktree/branch/pr for internal needs_review task — skipping review and marking done"
352 );
353 if let Err(e) = task_manager
354 .update_task_status(&task.id, Status::Done)
355 .await
356 {
357 tracing::error!(
358 task_id = task.id.0,
359 err = %e,
360 "update_task_status(Done) failed for no-code review skip"
361 );
362 }
363 return Ok(ReviewPhase::EarlyReturn(ReviewDecision::Skipped));
364 }
365
366 let max_reroutes: u32 = config::get("workflow.max_reroute_attempts")
369 .ok()
370 .and_then(|s| s.parse().ok())
371 .or_else(|| {
372 config::get("workflow.max_attempts")
373 .ok()
374 .and_then(|s| s.parse().ok())
375 })
376 .unwrap_or(3);
377
378 let reroutes = match store_increment(
379 &Some(Arc::clone(store)),
380 repo,
381 &task.id.0,
382 "no_code_reroutes",
383 )
384 .await
385 {
386 Ok(v) => v,
387 Err(e) => {
388 tracing::warn!(task_id = task.id.0, err = %e, "failed to increment no_code_reroutes — skipping reroute/block decision this tick");
389 return Ok(ReviewPhase::EarlyReturn(ReviewDecision::Failed(format!(
390 "transient store error: {e}"
391 ))));
392 }
393 };
394
395 if reroutes as u32 >= max_reroutes {
396 tracing::error!(
397 task_id = task.id.0,
398 reroutes,
399 max_reroutes,
400 "reached max reroute attempts for no-worktree/branch/pr result — blocking for human review"
401 );
402 let msg = format!(
403 "no worktree/branch/pr after {}/{} reroute attempts",
404 reroutes, max_reroutes
405 );
406 let fields = [
407 (
408 "block_reason",
409 serde_json::json!(format!(
410 "max reroute attempts ({}) reached — no worktree/branch/pr produced",
411 max_reroutes
412 )),
413 ),
414 ("last_error", serde_json::json!(msg)),
415 ("agent", serde_json::json!(null)),
416 ("model", serde_json::json!(null)),
417 ];
418 if let Err(e) = task_manager
419 .update_task_status_and_result(&task.id, Status::Blocked, &fields)
420 .await
421 {
422 tracing::error!(task_id = task.id.0, err = %e, "update_task_status_and_result(Blocked) failed — skipping block to avoid silent auto-unblock loop");
423 }
424 return Ok(ReviewPhase::EarlyReturn(ReviewDecision::Blocked(msg)));
425 }
426
427 tracing::warn!(
428 task_id = task.id.0,
429 reroutes,
430 max_reroutes,
431 "no worktree/branch/pr for external needs_review task — re-routing to a different agent"
432 );
433 if let Err(e) = store_set_result(
434 &Some(Arc::clone(store)),
435 repo,
436 &task.id.0,
437 &[
438 ("agent", serde_json::json!(null)),
439 ("model", serde_json::json!(null)),
440 (
441 "last_error",
442 serde_json::json!("no worktree/branch/pr produced"),
443 ),
444 ],
445 )
446 .await
447 {
448 tracing::error!(task_id = task.id.0, err = %e, "failed to clear agent/model for reroute — skipping status transition");
449 } else if let Err(e) = task_manager.update_task_status(&task.id, Status::New).await {
450 tracing::error!(task_id = task.id.0, err = %e, "update_task_status(New) failed — task may be stuck in InReview");
451 }
452 return Ok(ReviewPhase::EarlyReturn(ReviewDecision::Rerouted));
453 }
454
455 let mut worktree_path = match stored_task.as_ref().map(|t| t.worktree.as_str()) {
456 Some(w) if !w.is_empty() => std::path::PathBuf::from(w),
457 _ => {
458 tracing::warn!(task_id = task.id.0, "no worktree found for review");
459 return Ok(ReviewPhase::EarlyReturn(ReviewDecision::Failed(
460 "no worktree found".to_string(),
461 )));
462 }
463 };
464
465 let mut branch_name = match stored_task.as_ref().map(|t| t.branch.as_str()) {
466 Some(b) if !b.is_empty() => b.to_string(),
467 _ => {
468 tracing::warn!(task_id = task.id.0, "no branch found for review");
469 return Ok(ReviewPhase::EarlyReturn(ReviewDecision::Failed(
470 "no branch found".to_string(),
471 )));
472 }
473 };
474
475 let mut missing_worktree_recovered = false;
476 if !worktree_path.exists() {
477 match recover_missing_review_worktree(task, repo, store).await {
478 Ok(recovered) => {
479 branch_name = recovered.branch;
480 worktree_path = recovered.work_dir;
481 missing_worktree_recovered = true;
482 }
483 Err(e) => {
484 let reason = format!("missing review worktree recovery failed: {e}");
485 tracing::error!(task_id = task.id.0, error = %reason, "cannot recover review worktree");
486 store_set(
487 &Some(Arc::clone(store)),
488 repo,
489 &task.id.0,
490 &[("last_error", serde_json::json!(reason.clone()))],
491 )
492 .await;
493 return Ok(ReviewPhase::EarlyReturn(ReviewDecision::Blocked(reason)));
494 }
495 }
496 }
497
498 if missing_worktree_recovered {
499 store_set(
500 &Some(Arc::clone(store)),
501 repo,
502 &task.id.0,
503 &[("last_error", serde_json::json!(""))],
504 )
505 .await;
506 }
507
508 let agent_summary = stored_task
509 .as_ref()
510 .map(|t| t.summary.clone())
511 .unwrap_or_default();
512 let stored_pr_number = stored_task
513 .as_ref()
514 .and_then(|t| t.pr_number)
515 .map(|n| n as u64)
516 .filter(|&n| n > 0);
517
518 let pr_number = match ensure_pr_exists(
519 task,
520 &branch_name,
521 &worktree_path,
522 repo,
523 &agent_summary,
524 store,
525 task_manager,
526 stored_pr_number,
527 )
528 .await?
529 {
530 EnsurePrResult::Ready(n) => n,
531 EnsurePrResult::EarlyReturn(decision) => return Ok(ReviewPhase::EarlyReturn(decision)),
532 };
533
534 {
538 let fetch_timeout = std::time::Duration::from_secs(60);
539 let fetch_future = async {
540 let mut cmd = Command::new("git");
541 cmd.args(["fetch", "origin", "--prune"])
542 .current_dir(&worktree_path);
543 cmd.kill_on_drop(true);
545 cmd.output_with_context().await
546 };
547
548 match tokio::time::timeout(fetch_timeout, fetch_future).await {
549 Ok(Ok(o)) if o.status.success() => {
550 }
552 Ok(Ok(o)) => {
553 tracing::warn!(
554 task_id = task.id.0,
555 exit_code = ?o.status.code(),
556 stdout = %String::from_utf8_lossy(&o.stdout),
557 stderr = %String::from_utf8_lossy(&o.stderr),
558 "review: git fetch failed (non-zero exit) — diff/log may use stale remote refs"
559 );
560 }
561 Ok(Err(e)) => {
562 tracing::warn!(
563 task_id = task.id.0,
564 error = %e,
565 "review: git fetch failed — diff/log may use stale remote refs"
566 );
567 }
568 Err(_) => {
569 tracing::warn!(
570 task_id = task.id.0,
571 "review: git fetch timed out after 60s — diff/log may use stale remote refs"
572 );
573 }
574 }
575 }
576
577 let default_branch = runner::worktree::detect_default_branch(&worktree_path).await;
578 let git_diff = runner::context::build_git_diff(&worktree_path, &default_branch).await;
579 let git_log = runner::context::build_git_log(&worktree_path, &default_branch).await;
580 let review_prompt = runner::agent::build_review_prompt(
581 task,
582 &agent_summary,
583 &git_diff,
584 &git_log,
585 &default_branch,
586 pr_number,
587 runner::agent::Shipcheck::detect(&worktree_path).await,
588 );
589
590 let task_agent = stored_task
591 .as_ref()
592 .and_then(|t| t.agent.clone())
593 .unwrap_or_default();
594 let (review_agent, review_model, same_agent_fallback) =
595 select_review_agent(&task.id.0, &task_agent, store_id, router, store).await?;
596
597 anyhow::ensure!(
598 review_model.is_some(),
599 "review model missing for task {} after agent selection — refusing to dispatch",
600 task.id.0
601 );
602
603 if same_agent_fallback {
604 tracing::warn!(
605 task_id = task.id.0,
606 agent = %review_agent,
607 "no other agent available for review — falling back to the agent that wrote the code"
608 );
609 store_log_activity(
610 &Some(Arc::clone(store)),
611 repo,
612 &task.id.0,
613 "review_agent_fallback",
614 None,
615 None,
616 Some(&review_agent),
617 review_model.as_deref(),
618 Some(&serde_json::json!({
619 "reason": "all available agents excluded (author + previous reviewers)",
620 })),
621 )
622 .await;
623 }
624
625 tracing::info!(
626 task_id = task.id.0,
627 agent = %review_agent,
628 model = ?review_model,
629 "spawning review agent"
630 );
631
632 let review_task_id = format!("{}-review", task.id.0);
633 let review_attempt_dir =
634 crate::home::task_attempt_dir_async(repo, &review_task_id, review_attempt).await?;
635 let output_file = review_attempt_dir.join("output.json");
636
637 let git_name =
638 crate::config::get("git.name").unwrap_or_else(|_| format!("{review_agent}[bot]"));
639 let git_email = crate::config::get("git.email")
640 .unwrap_or_else(|_| format!("{review_agent}[bot]@users.noreply.github.com"));
641 let review_timeout_secs: u64 = crate::config::get("workflow.review_timeout_seconds")
642 .ok()
643 .and_then(|s| s.parse().ok())
644 .unwrap_or_else(|| {
645 crate::config::get("workflow.timeout_seconds")
646 .ok()
647 .and_then(|s| s.parse().ok())
648 .unwrap_or(1800)
649 })
650 .max(1800);
651
652 let invocation = runner::agent::AgentInvocation {
653 agent: review_agent.clone(),
654 model: review_model.clone(),
655 work_dir: worktree_path.clone(),
656 system_prompt: runner::agent::review_system_prompt(),
657 agent_message: review_prompt,
658 task_id: review_task_id.clone(),
659 disallowed_tools: build_review_disallowed_tools(&worktree_path),
660 git_author_name: git_name,
661 git_author_email: git_email,
662 output_file: output_file.clone(),
663 timeout_seconds: review_timeout_secs,
664 repo: repo.to_string(),
665 attempt: review_attempt,
666 };
667
668 let head_sha = match GhHttp::new() {
669 Ok(gh) => gh.get_pr(repo, pr_number).await.ok().map(|pr| pr.head.sha),
670 Err(_) => None,
671 };
672
673 Ok(ReviewPhase::Ready(ReviewContext {
674 store_id,
675 had_prev_push_failure,
676 review_attempt,
677 worktree_path,
678 branch_name,
679 pr_number,
680 head_sha,
681 default_branch,
682 review_agent,
683 review_model,
684 review_task_id,
685 review_attempt_dir,
686 output_file,
687 invocation,
688 same_agent_fallback,
689 }))
690}
691
692#[allow(clippy::too_many_arguments)]
693async fn complete_review_run(
694 store: &Arc<TaskStore>,
695 run_id: Option<i64>,
696 exit_code: Option<i32>,
697 stdout: &str,
698 stderr: &str,
699 parsed: &str,
700 outcome: &str,
701 error: &str,
702 tokens: RunTokenUsage,
703) {
704 if let Some(run_id) = run_id {
705 if let Err(e) = store
706 .complete_run(&CompleteRun {
707 run_id,
708 exit_code,
709 stdout,
710 stderr,
711 parsed,
712 outcome,
713 error,
714 tokens,
715 })
716 .await
717 {
718 tracing::warn!(
719 run_id,
720 error = %e,
721 "failed to record review run completion in audit trail"
722 );
723 }
724 }
725}
726
727async fn invoke_review_agent(
728 task: &ExternalTask,
729 tmux: &Arc<TmuxManager>,
730 ctx: &ReviewContext,
731 repo: &str,
732 store: &Arc<TaskStore>,
733) -> ReviewPhase<ReviewRun> {
734 let run_id = if let Some(sid) = ctx.store_id {
735 match store
736 .start_run(&StartRun {
737 task_id: sid,
738 attempt: ctx.review_attempt as i32,
739 run_type: "review",
740 agent: &ctx.review_agent,
741 model: ctx.review_model.as_deref().unwrap_or(""),
742 command: "",
743 prompt: "",
744 })
745 .await
746 {
747 Ok(run_id) => Some(run_id),
748 Err(e) => {
749 tracing::warn!(
750 task_id = task.id.0,
751 error = %e,
752 "failed to record review run start in audit trail"
753 );
754 None
755 }
756 }
757 } else {
758 None
759 };
760
761 let session = match runner::agent::spawn_in_tmux(tmux, &ctx.invocation).await {
762 Ok(session) => session,
763 Err(e) => {
764 tracing::error!(task_id = task.id.0, error = %e, "failed to spawn review agent");
765 complete_review_run(
766 store,
767 run_id,
768 None,
769 "",
770 &e.to_string(),
771 "",
772 "failed",
773 &format!("spawn failed: {e}"),
774 RunTokenUsage::default(),
775 )
776 .await;
777 return ReviewPhase::EarlyReturn(ReviewDecision::Failed(format!("spawn failed: {e}")));
778 }
779 };
780
781 let start_comment = review_started_comment(&ctx.review_agent, ctx.review_model_str());
782 match GhHttp::new() {
783 Ok(gh) => {
784 post_review_status(
785 &gh,
786 repo,
787 ctx.head_sha.as_deref(),
788 "pending",
789 "Automated review running",
790 )
791 .await;
792 if let Err(e) = gh
793 .add_comment(repo, &ctx.pr_number.to_string(), &start_comment)
794 .await
795 {
796 tracing::debug!(
797 task_id = task.id.0,
798 pr_number = ctx.pr_number,
799 error = %e,
800 "failed to post review-started comment"
801 );
802 }
803 }
804 Err(e) => {
805 tracing::debug!(
806 task_id = task.id.0,
807 pr_number = ctx.pr_number,
808 error = %e,
809 "failed to create GitHub client for review-started comment"
810 );
811 }
812 }
813
814 let poll_interval = std::time::Duration::from_secs(5);
815 let timeout_duration = std::time::Duration::from_secs(ctx.invocation.timeout_seconds);
816 let wait_result = tokio::time::timeout(
817 timeout_duration,
818 tmux.wait_for_completion(&session, poll_interval),
819 )
820 .await;
821
822 match wait_result {
823 Ok(Ok(_)) => {
824 tracing::info!(task_id = task.id.0, "review agent completed");
825 let _ = tmux.kill_session(&session).await;
826 ReviewPhase::Ready(ReviewRun { run_id })
827 }
828 Ok(Err(e)) => {
829 tracing::warn!(task_id = task.id.0, error = %e, "review agent process exited with error");
830 let _ = tmux.kill_session(&session).await;
831
832 let exit_code: i32 = tokio::task::spawn_blocking({
837 let review_attempt_dir = ctx.review_attempt_dir.clone();
838 move || {
839 std::fs::read_to_string(review_attempt_dir.join("exit.txt"))
840 .ok()
841 .and_then(|s| s.trim().parse().ok())
842 .unwrap_or(-1)
843 }
844 })
845 .await
846 .unwrap_or(-1);
847
848 if exit_code != 0 {
849 let raw_output =
854 runner::response::read_output_file(&ctx.review_task_id, &ctx.output_file, repo)
855 .await;
856 let agent_result =
857 runner::agents::find_agent_result(&ctx.review_agent, &raw_output);
858 if agent_result.as_ref().map(|r| !r.is_error).unwrap_or(false) {
860 tracing::info!(
861 task_id = task.id.0,
862 agent = %ctx.review_agent,
863 exit_code,
864 output_found = !raw_output.is_empty(),
865 "review agent succeeded despite non-zero exit — proceeding to parse output"
866 );
867 return ReviewPhase::Ready(ReviewRun { run_id });
868 }
869 }
870
871 tracing::error!(task_id = task.id.0, error = %e, "review agent failed");
872 complete_review_run(
873 store,
874 run_id,
875 Some(exit_code),
876 "",
877 &e.to_string(),
878 "",
879 "failed",
880 &format!("agent error: {e}"),
881 RunTokenUsage::default(),
882 )
883 .await;
884 ReviewPhase::EarlyReturn(ReviewDecision::Failed(format!("agent error: {e}")))
885 }
886 Err(_) => {
887 tracing::error!(task_id = task.id.0, "review agent timed out");
888 let _ = tmux.kill_session(&session).await;
889 if let Some(model) = ctx.review_model.as_deref() {
890 crate::engine::cooldown::record_persistent_model_failure(&ctx.review_agent, model)
895 .await;
896 }
897 complete_review_run(
898 store,
899 run_id,
900 Some(-1),
901 "",
902 "timeout",
903 "",
904 "timeout",
905 "timeout",
906 RunTokenUsage::default(),
907 )
908 .await;
909 ReviewPhase::EarlyReturn(ReviewDecision::Failed("timeout".to_string()))
910 }
911 }
912}
913
914async fn record_review_agent_failure(
915 task_id: &str,
916 review_agent: &str,
917 review_model: Option<&str>,
918 err: &runner::agents::AgentError,
919) {
920 match err {
921 runner::agents::AgentError::RateLimit { message }
922 | runner::agents::AgentError::Auth { message } => {
923 tracing::warn!(
924 task_id,
925 agent = %review_agent,
926 "review agent hit error — adding to cooldown"
927 );
928 if let Some(reason) = crate::engine::cooldown::detect_credit_exhaustion(message) {
929 if reason == crate::engine::cooldown::CreditExhaustionReason::BillingCycleExhausted
930 {
931 if let Some(model) = review_model {
932 crate::engine::cooldown::record_persistent_model_failure(
934 review_agent,
935 model,
936 )
937 .await;
938 } else {
939 crate::engine::cooldown::record_credit_exhaustion(review_agent, reason)
940 .await;
941 }
942 } else {
943 crate::engine::cooldown::record_credit_exhaustion(review_agent, reason).await;
944 }
945 } else {
946 runner::response::record_agent_failure_with_message(review_agent, &err.to_string())
947 .await;
948 }
949 }
950 runner::agents::AgentError::InvalidResponse { .. } => {
951 if let Some(model) = review_model {
956 crate::engine::cooldown::record_persistent_model_failure(review_agent, model).await;
957 }
958 }
959 runner::agents::AgentError::ModelUnavailable { .. } => {
960 if let Some(model) = review_model {
965 crate::engine::cooldown::record_persistent_model_failure(review_agent, model).await;
966 }
967 }
968 _ => {
969 if let Some(model) = review_model {
972 crate::engine::cooldown::record_model_failure(review_agent, model).await;
973 }
974 }
975 }
976}
977
978fn review_decision_from_response(
979 review_response: runner::response::ReviewResponse,
980) -> ReviewDecision {
981 match review_response.decision.as_str() {
982 "approve" => ReviewDecision::Approve,
983 "request_changes" => ReviewDecision::RequestChanges {
984 notes: review_response.notes,
985 issues: review_response.issues,
986 },
987 _ => ReviewDecision::Failed(format!("unknown decision: {}", review_response.decision)),
988 }
989}
990
991fn review_decision_name(decision: &ReviewDecision) -> &'static str {
992 match decision {
993 ReviewDecision::Approve => "approve",
994 ReviewDecision::RequestChanges { .. } => "request_changes",
995 ReviewDecision::Failed(_) => "failed",
996 ReviewDecision::Blocked(_) => "blocked",
997 ReviewDecision::Skipped => "skipped",
998 ReviewDecision::Rerouted => "rerouted",
999 }
1000}
1001
1002async fn parse_review_output(
1003 task: &ExternalTask,
1004 repo: &str,
1005 ctx: &ReviewContext,
1006 run: ReviewRun,
1007 store: &Arc<TaskStore>,
1008) -> ReviewPhase<ParsedReview> {
1009 let (file_exists, file_size, exit_code, stderr) = {
1010 let output_file_clone = ctx.output_file.clone();
1011 let review_attempt_dir_clone = ctx.review_attempt_dir.clone();
1012 match tokio::task::spawn_blocking(move || {
1013 let file_exists = output_file_clone.exists();
1014 let file_size = std::fs::metadata(&output_file_clone)
1015 .map(|m| m.len())
1016 .unwrap_or(0);
1017 let exit_code = std::fs::read_to_string(review_attempt_dir_clone.join("exit.txt"))
1018 .ok()
1019 .and_then(|s| s.trim().parse::<i32>().ok())
1020 .unwrap_or(-1);
1021 let stderr = std::fs::read_to_string(review_attempt_dir_clone.join("stderr.txt"))
1022 .unwrap_or_default();
1023 (file_exists, file_size, exit_code, stderr)
1024 })
1025 .await
1026 {
1027 Ok(tuple) => tuple,
1028 Err(e) => {
1029 tracing::error!(
1030 task_id = task.id.0,
1031 error = %e,
1032 "spawn_blocking panicked reading review output metadata"
1033 );
1034 (false, 0, -1, format!("spawn_blocking failed: {e}"))
1035 }
1036 }
1037 };
1038
1039 tracing::info!(
1040 task_id = task.id.0,
1041 path = %ctx.output_file.display(),
1042 file_exists,
1043 file_size,
1044 "reading review agent output"
1045 );
1046
1047 let raw_output =
1048 runner::response::read_output_file(&ctx.review_task_id, &ctx.output_file, repo).await;
1049 let raw_output = runner::agents::claude::strip_tty_control_lines(&raw_output);
1050 let has_result_envelope =
1051 runner::agents::claude::find_ndjson_result_line(&raw_output).is_some();
1052 let agent_result = runner::agents::find_agent_result(&ctx.review_agent, &raw_output);
1053 let token_usage = RunTokenUsage {
1054 input_tokens: agent_result
1055 .as_ref()
1056 .and_then(|r| r.input_tokens)
1057 .unwrap_or(0),
1058 output_tokens: agent_result
1059 .as_ref()
1060 .and_then(|r| r.output_tokens)
1061 .unwrap_or(0),
1062 total_cost_usd: agent_result
1063 .as_ref()
1064 .and_then(|r| r.cost_usd)
1065 .unwrap_or(0.0),
1066 duration_secs: 0.0,
1067 };
1068
1069 let agent_result_is_error = agent_result.as_ref().map(|r| r.is_error).unwrap_or(false);
1070 let ndjson_completed_no_error = raw_output.contains("\"terminal_reason\":\"completed\"")
1071 && !raw_output.contains("\"is_error\":true");
1072 let is_hard_failure =
1074 !ndjson_completed_no_error && (raw_output.is_empty() || agent_result_is_error);
1075
1076 if is_hard_failure {
1077 tracing::warn!(
1078 task_id = task.id.0,
1079 exit_code,
1080 raw_output_len = raw_output.len(),
1081 stderr_len = stderr.len(),
1082 "review agent: entering error path"
1083 );
1084 let agent_runner = runner::agents::get_runner(&ctx.review_agent);
1085 let err = agent_runner.classify_error(exit_code, &raw_output, &stderr);
1086 record_review_agent_failure(
1087 &task.id.0,
1088 &ctx.review_agent,
1089 ctx.review_model.as_deref(),
1090 &err,
1091 )
1092 .await;
1093 tracing::error!(task_id = task.id.0, error = %err, "review agent failed");
1094 let stored_error = format_review_error(&ctx.review_agent, &err);
1095 complete_review_run(
1096 store,
1097 run.run_id,
1098 Some(exit_code),
1099 &raw_output,
1100 &stderr,
1101 "",
1102 outcome_for_agent_error(&err),
1103 &stored_error,
1104 token_usage,
1105 )
1106 .await;
1107 return ReviewPhase::EarlyReturn(ReviewDecision::Failed(stored_error));
1108 }
1109
1110 let text_for_review = agent_result
1111 .as_ref()
1112 .map(|r| r.result_text.clone())
1113 .filter(|text| !text.is_empty())
1114 .unwrap_or_else(|| {
1115 tracing::debug!(
1116 task_id = task.id.0,
1117 agent = %ctx.review_agent,
1118 "review agent: per-agent extractor returned no text, using raw output"
1119 );
1120 raw_output.clone()
1121 });
1122
1123 let review_response = match runner::response::parse_review_response(&text_for_review) {
1124 Ok(response) => response,
1125 Err(e) => {
1126 if let Some(response) = runner::response::infer_review_response(&text_for_review) {
1127 tracing::warn!(
1128 task_id = task.id.0,
1129 agent = %ctx.review_agent,
1130 "review response parsed via keyword fallback"
1131 );
1132 response
1133 } else {
1134 let ndjson_completed = ndjson_completed_no_error;
1140
1141 if ndjson_completed || !has_result_envelope {
1144 let assistant_text = runner::agents::collect_assistant_messages_text(
1145 &ctx.review_agent,
1146 &raw_output,
1147 );
1148 if !assistant_text.is_empty() {
1149 if let Ok(response) =
1150 runner::response::parse_review_response(&assistant_text)
1151 {
1152 tracing::debug!(
1153 task_id = task.id.0,
1154 agent = %ctx.review_agent,
1155 "found review response JSON in assistant message (terminal_reason:completed)"
1156 );
1157 let review_notes_for_comment = response.notes.clone();
1158 let decision = review_decision_from_response(response);
1159 return ReviewPhase::Ready(ParsedReview {
1160 run_id: run.run_id,
1161 exit_code,
1162 stderr,
1163 raw_output,
1164 text_for_review: assistant_text,
1165 token_usage,
1166 review_notes_for_comment,
1167 decision,
1168 });
1169 }
1170 if let Some(response) =
1171 runner::response::infer_review_response(&assistant_text)
1172 {
1173 tracing::warn!(
1174 task_id = task.id.0,
1175 agent = %ctx.review_agent,
1176 "inferred review response from assistant message (terminal_reason:completed)"
1177 );
1178 let review_notes_for_comment = response.notes.clone();
1179 let decision = review_decision_from_response(response);
1180 return ReviewPhase::Ready(ParsedReview {
1181 run_id: run.run_id,
1182 exit_code,
1183 stderr,
1184 raw_output,
1185 text_for_review: assistant_text,
1186 token_usage,
1187 review_notes_for_comment,
1188 decision,
1189 });
1190 }
1191 }
1192 }
1193
1194 let scan_head = runner::agents::patterns::safe_head(
1201 &text_for_review,
1202 runner::agents::patterns::RATE_LIMIT_SCAN_HEAD_BYTES,
1203 );
1204 let scan_tail = runner::agents::patterns::safe_tail(
1205 &text_for_review,
1206 runner::agents::patterns::RATE_LIMIT_SCAN_TAIL_BYTES,
1207 );
1208
1209 if let Some(err) = (!ndjson_completed)
1210 .then(|| {
1211 runner::agents::patterns::detect_rate_limit(scan_head)
1212 .or_else(|| runner::agents::patterns::detect_rate_limit(scan_tail))
1213 })
1214 .flatten()
1215 {
1216 let runner::agents::AgentError::RateLimit { message } = &err else {
1217 unreachable!("detect_rate_limit only returns AgentError::RateLimit")
1218 };
1219 tracing::warn!(
1220 task_id = task.id.0,
1221 agent = %ctx.review_agent,
1222 "review agent hit rate limit — adding to cooldown"
1223 );
1224 let summary = runner::fallback::summarize_rate_limit_error(message);
1231 let stored_error = format!("{} rate limit: {summary}", ctx.review_agent);
1232 record_review_agent_failure(
1233 &task.id.0,
1234 &ctx.review_agent,
1235 ctx.review_model.as_deref(),
1236 &err,
1237 )
1238 .await;
1239 complete_review_run(
1240 store,
1241 run.run_id,
1242 Some(exit_code),
1243 &raw_output,
1244 &stderr,
1245 &text_for_review,
1246 outcome_for_agent_error(&err),
1247 &stored_error,
1248 token_usage,
1249 )
1250 .await;
1251 return ReviewPhase::EarlyReturn(ReviewDecision::Failed(stored_error));
1252 }
1253
1254 if let Some(err) = runner::agents::patterns::detect_auth_error(scan_tail) {
1255 let runner::agents::AgentError::Auth { message } = &err else {
1256 unreachable!("detect_auth_error only returns AgentError::Auth")
1257 };
1258 tracing::warn!(
1259 task_id = task.id.0,
1260 agent = %ctx.review_agent,
1261 "review agent hit auth error — adding to cooldown"
1262 );
1263 let stored_error = format!("auth error: {message}");
1264 record_review_agent_failure(
1265 &task.id.0,
1266 &ctx.review_agent,
1267 ctx.review_model.as_deref(),
1268 &err,
1269 )
1270 .await;
1271 complete_review_run(
1272 store,
1273 run.run_id,
1274 Some(exit_code),
1275 &raw_output,
1276 &stderr,
1277 &text_for_review,
1278 outcome_for_agent_error(&err),
1279 &stored_error,
1280 token_usage,
1281 )
1282 .await;
1283 return ReviewPhase::EarlyReturn(ReviewDecision::Failed(stored_error));
1284 }
1285
1286 if agent_result
1293 .as_ref()
1294 .map(|r| r.truncated_by_length)
1295 .unwrap_or(false)
1296 {
1297 tracing::warn!(
1298 task_id = task.id.0,
1299 agent = %ctx.review_agent,
1300 model = ?ctx.review_model,
1301 "review agent output truncated: step_finish reason=length \
1302 (token budget exceeded before completion)"
1303 );
1304 if let Some(model) = ctx.review_model.as_deref() {
1305 crate::engine::cooldown::record_persistent_model_failure(
1306 &ctx.review_agent,
1307 model,
1308 )
1309 .await;
1310 }
1311 let stored_error = "truncated: review output was cut off before completion \
1312 (output/reasoning token budget exceeded)"
1313 .to_string();
1314 complete_review_run(
1315 store,
1316 run.run_id,
1317 Some(exit_code),
1318 &raw_output,
1319 &stderr,
1320 &text_for_review,
1321 "truncated",
1322 &stored_error,
1323 token_usage,
1324 )
1325 .await;
1326 return ReviewPhase::EarlyReturn(ReviewDecision::Failed(stored_error));
1327 }
1328
1329 if runner::agents::claude::review_session_exited_mid_thought(&raw_output) {
1332 tracing::warn!(
1333 task_id = task.id.0,
1334 agent = %ctx.review_agent,
1335 model = ?ctx.review_model,
1336 "review session exited mid-thought with no result envelope \
1337 — storing as incomplete and retrying without persistent cooldown"
1338 );
1339 let stored_error =
1340 "incomplete: review session exited before producing a decision".to_string();
1341 complete_review_run(
1342 store,
1343 run.run_id,
1344 Some(exit_code),
1345 &raw_output,
1346 &stderr,
1347 &text_for_review,
1348 "incomplete",
1349 &stored_error,
1350 token_usage,
1351 )
1352 .await;
1353 return ReviewPhase::EarlyReturn(ReviewDecision::Failed(stored_error));
1354 }
1355
1356 tracing::error!(
1357 task_id = task.id.0,
1358 error = %e,
1359 agent = %ctx.review_agent,
1360 output = %text_for_review.chars().take(300).collect::<String>(),
1361 raw_output = %raw_output.chars().take(1000).collect::<String>(),
1362 "failed to parse review response"
1363 );
1364 if let Some(model) = ctx.review_model.as_deref() {
1365 crate::engine::cooldown::record_persistent_model_failure(
1368 &ctx.review_agent,
1369 model,
1370 )
1371 .await;
1372 }
1373 complete_review_run(
1374 store,
1375 run.run_id,
1376 Some(exit_code),
1377 &raw_output,
1378 &stderr,
1379 &text_for_review,
1380 "parse_error",
1381 &format!("parse error: {e}"),
1382 token_usage,
1383 )
1384 .await;
1385 return ReviewPhase::EarlyReturn(ReviewDecision::Failed(format!(
1386 "parse error: {e}"
1387 )));
1388 }
1389 }
1390 };
1391
1392 let review_notes_for_comment = review_response.notes.clone();
1393 let decision = review_decision_from_response(review_response);
1394
1395 tracing::info!(
1396 task_id = task.id.0,
1397 pr_number = ctx.pr_number,
1398 decision = ?decision,
1399 "review agent decision received"
1400 );
1401 store_log_activity(
1402 &Some(Arc::clone(store)),
1403 repo,
1404 &task.id.0,
1405 "review_decision",
1406 None,
1407 None,
1408 Some(&ctx.review_agent),
1409 ctx.review_model.as_deref(),
1410 Some(&serde_json::json!({
1411 "decision": review_decision_name(&decision),
1412 "pr_number": ctx.pr_number,
1413 })),
1414 )
1415 .await;
1416
1417 ReviewPhase::Ready(ParsedReview {
1418 run_id: run.run_id,
1419 exit_code,
1420 stderr,
1421 raw_output,
1422 text_for_review,
1423 token_usage,
1424 review_notes_for_comment,
1425 decision,
1426 })
1427}
1428
1429async fn finalize_review_run(
1430 task: &ExternalTask,
1431 ctx: &ReviewContext,
1432 parsed: &ParsedReview,
1433 store: &Arc<TaskStore>,
1434) {
1435 let outcome = match &parsed.decision {
1436 ReviewDecision::Approve
1437 | ReviewDecision::RequestChanges { .. }
1438 | ReviewDecision::Skipped
1439 | ReviewDecision::Rerouted => "success",
1440 ReviewDecision::Failed(_) | ReviewDecision::Blocked(_) => "failed",
1441 };
1442 let error = match &parsed.decision {
1443 ReviewDecision::Failed(error) | ReviewDecision::Blocked(error) => error.as_str(),
1444 _ => "",
1445 };
1446
1447 if outcome == "success" {
1448 crate::engine::cooldown::record_agent_success(&ctx.review_agent, ctx.review_model_str())
1451 .await;
1452 }
1453
1454 complete_review_run(
1455 store,
1456 parsed.run_id,
1457 Some(parsed.exit_code),
1458 &parsed.raw_output,
1459 &parsed.stderr,
1460 &parsed.text_for_review,
1461 outcome,
1462 error,
1463 parsed.token_usage,
1464 )
1465 .await;
1466
1467 if let Some(store_id) = ctx.store_id {
1468 if parsed.token_usage.input_tokens > 0 || parsed.token_usage.output_tokens > 0 {
1469 let model = if ctx.review_model_str().is_empty() {
1470 "unknown"
1471 } else {
1472 ctx.review_model_str()
1473 };
1474 if let Err(e) = store
1475 .store_tokens(
1476 store_id,
1477 parsed.token_usage.input_tokens,
1478 parsed.token_usage.output_tokens,
1479 model,
1480 )
1481 .await
1482 {
1483 tracing::warn!(
1484 task_id = task.id.0,
1485 ?e,
1486 "failed to store review agent token usage"
1487 );
1488 }
1489 }
1490 }
1491}
1492
1493async fn restore_review_config_if_needed(
1494 task_id: &str,
1495 worktree_path: &std::path::Path,
1496 default_branch: &str,
1497) {
1498 let _ = Command::new("git")
1499 .args(["checkout", "HEAD", "--", ".orch.yml"])
1500 .current_dir(worktree_path)
1501 .status()
1502 .await;
1503
1504 let merge_base = Command::new("git")
1505 .args(["merge-base", "HEAD", default_branch])
1506 .current_dir(worktree_path)
1507 .output()
1508 .await
1509 .ok()
1510 .filter(|output| output.status.success())
1511 .and_then(|output| String::from_utf8(output.stdout).ok())
1512 .map(|s| s.trim().to_string())
1513 .unwrap_or_default();
1514
1515 if merge_base.is_empty() {
1516 return;
1517 }
1518
1519 let orch_yml_changed = Command::new("git")
1520 .args([
1521 "diff",
1522 "--name-only",
1523 &merge_base,
1524 "HEAD",
1525 "--",
1526 ".orch.yml",
1527 ])
1528 .current_dir(worktree_path)
1529 .output()
1530 .await
1531 .ok()
1532 .filter(|output| output.status.success())
1533 .map(|output| !output.stdout.is_empty())
1534 .unwrap_or(false);
1535
1536 if !orch_yml_changed {
1537 return;
1538 }
1539
1540 tracing::warn!(
1541 task_id,
1542 "review agent modified .orch.yml (forbidden); restoring to merge-base version"
1543 );
1544 let restore_ok = Command::new("git")
1545 .args(["checkout", &merge_base, "--", ".orch.yml"])
1546 .current_dir(worktree_path)
1547 .status()
1548 .await
1549 .map(|status| status.success())
1550 .unwrap_or(false);
1551
1552 if !restore_ok {
1553 return;
1554 }
1555
1556 let staged = Command::new("git")
1557 .args(["diff", "--cached", "--name-only", "--", ".orch.yml"])
1558 .current_dir(worktree_path)
1559 .output()
1560 .await
1561 .ok()
1562 .filter(|output| output.status.success())
1563 .map(|output| !output.stdout.is_empty())
1564 .unwrap_or(false);
1565
1566 if staged {
1567 let _ = Command::new("git")
1568 .args([
1569 "-c",
1570 "commit.gpgsign=false",
1571 "commit",
1572 "-m",
1573 "revert: restore .orch.yml (review agent must not modify project config)",
1574 ])
1575 .current_dir(worktree_path)
1576 .status()
1577 .await;
1578 }
1579}
1580
1581fn is_missing_path_error_message(err: &str) -> bool {
1582 err.to_ascii_lowercase()
1583 .contains("no such file or directory")
1584}
1585
1586async fn push_review_branch(
1587 task: &ExternalTask,
1588 repo: &str,
1589 ctx: &ReviewContext,
1590 store: &Arc<TaskStore>,
1591) -> Result<(), String> {
1592 let mut worktree_path = ctx.worktree_path.clone();
1593 let mut branch_name = ctx.branch_name.clone();
1594 let mut retried_missing_worktree = false;
1595
1596 let push_result = loop {
1597 match runner::git_ops::push_branch(&worktree_path, &branch_name, &ctx.default_branch).await
1598 {
1599 Ok(ok) => break Ok(ok),
1600 Err(e)
1601 if !retried_missing_worktree
1602 && !worktree_path.exists()
1603 && is_missing_path_error_message(&e.to_string()) =>
1604 {
1605 retried_missing_worktree = true;
1606 tracing::warn!(
1607 task_id = task.id.0,
1608 worktree = %worktree_path.display(),
1609 branch = %branch_name,
1610 "review push failed because worktree disappeared; attempting one-time recovery"
1611 );
1612 match recover_missing_review_worktree(task, repo, store).await {
1613 Ok(recovered) => {
1614 worktree_path = recovered.work_dir;
1615 branch_name = recovered.branch;
1616 continue;
1617 }
1618 Err(recover_err) => {
1619 break Err(anyhow::anyhow!(
1620 "missing review worktree recovery failed after push error: {recover_err}"
1621 ));
1622 }
1623 }
1624 }
1625 Err(e) => break Err(e),
1626 }
1627 };
1628
1629 match push_result {
1630 Ok(_) => {
1631 store_log_activity(
1632 &Some(Arc::clone(store)),
1633 repo,
1634 &task.id.0,
1635 "push",
1636 None,
1637 None,
1638 Some(&ctx.review_agent),
1639 ctx.review_model.as_deref(),
1640 Some(&serde_json::json!({
1641 "status": "ok",
1642 "branch": branch_name.clone(),
1643 "default_branch": ctx.default_branch.clone(),
1644 "phase": "review",
1645 })),
1646 )
1647 .await;
1648 if ctx.had_prev_push_failure {
1649 if let Err(e) = store_set_result(
1650 &Some(Arc::clone(store)),
1651 repo,
1652 &task.id.0,
1653 &[
1654 ("last_error", serde_json::json!("")),
1655 ("push_failures", serde_json::json!(0)),
1656 ],
1657 )
1658 .await
1659 {
1660 tracing::warn!(task_id = task.id.0, err = %e, "failed to reset push_failures — counter may drift");
1661 }
1662 }
1663 Ok(())
1664 }
1665 Err(e) => {
1666 store_log_activity(
1667 &Some(Arc::clone(store)),
1668 repo,
1669 &task.id.0,
1670 "push",
1671 None,
1672 None,
1673 Some(&ctx.review_agent),
1674 ctx.review_model.as_deref(),
1675 Some(&serde_json::json!({
1676 "status": "error",
1677 "branch": branch_name.clone(),
1678 "default_branch": ctx.default_branch.clone(),
1679 "phase": "review",
1680 "error": e.to_string(),
1681 })),
1682 )
1683 .await;
1684 tracing::error!(
1685 task_id = task.id.0,
1686 branch = %ctx.branch_name,
1687 error = %e,
1688 "review push failed"
1689 );
1690 store_set(
1691 &Some(Arc::clone(store)),
1692 repo,
1693 &task.id.0,
1694 &[("last_error", serde_json::json!(format!("push failed: {e}")))],
1695 )
1696 .await;
1697 Err(e.to_string())
1698 }
1699 }
1700}
1701
1702fn build_pr_review_comment(decision: &ReviewDecision, review_notes_for_comment: &str) -> String {
1703 match decision {
1704 ReviewDecision::Approve => {
1705 format!(
1706 "## Automated Review \u{2014} Approve\n\n{}",
1707 review_notes_for_comment
1708 )
1709 }
1710 ReviewDecision::RequestChanges { notes, issues } => {
1711 let mut body = format!(
1712 "## Automated Review \u{2014} Changes Requested\n\n{}\n",
1713 notes
1714 );
1715 if !issues.is_empty() {
1716 body.push_str("\n**Issues Found:**\n");
1717 for issue in issues {
1718 body.push_str(&format!(
1719 "- `{}` line {}: {} [{}]\n",
1720 issue.file,
1721 issue
1722 .line
1723 .map(|line| line.to_string())
1724 .unwrap_or_else(|| "?".to_string()),
1725 issue.description,
1726 issue.severity
1727 ));
1728 }
1729 }
1730 body
1731 }
1732 _ => String::new(),
1733 }
1734}
1735
1736const REVIEW_STATUS_CONTEXT: &str = "orch/review";
1737
1738fn review_status(decision: &ReviewDecision) -> Option<(&'static str, &'static str)> {
1740 match decision {
1741 ReviewDecision::Approve => Some(("success", "Approved by automated review")),
1742 ReviewDecision::RequestChanges { .. } => {
1743 Some(("failure", "Automated review requested changes"))
1744 }
1745 _ => None,
1746 }
1747}
1748
1749async fn post_review_status(
1752 gh: &GhHttp,
1753 repo: &str,
1754 sha: Option<&str>,
1755 state: &str,
1756 description: &str,
1757) {
1758 let Some(sha) = sha else {
1759 return;
1760 };
1761 if let Err(e) = gh
1762 .post_commit_status(repo, sha, state, REVIEW_STATUS_CONTEXT, description)
1763 .await
1764 {
1765 tracing::warn!(sha, error = %e, "failed to post orch/review commit status");
1766 }
1767}
1768
1769async fn post_review_comment(
1770 task: &ExternalTask,
1771 repo: &str,
1772 ctx: &ReviewContext,
1773 decision: &ReviewDecision,
1774 review_notes_for_comment: &str,
1775) -> anyhow::Result<()> {
1776 let gh = match GhHttp::new() {
1777 Ok(gh) => gh,
1778 Err(e) => {
1779 tracing::warn!(
1780 task_id = task.id.0,
1781 error = %e,
1782 "failed to create GH client for review comment"
1783 );
1784 return Ok(());
1785 }
1786 };
1787 let mut pr_comment = build_pr_review_comment(decision, review_notes_for_comment);
1788 if ctx.same_agent_fallback && !pr_comment.is_empty() {
1789 pr_comment.push_str(
1790 "\n> Note: no other agent was available, so this PR was reviewed by the same agent that wrote the code.\n",
1791 );
1792 }
1793
1794 if let Some((state, description)) = review_status(decision) {
1795 post_review_status(&gh, repo, ctx.head_sha.as_deref(), state, description).await;
1796 }
1797
1798 if !pr_comment.is_empty() {
1799 let footer =
1800 attribution_footer("Reviewed", &ctx.review_agent, Some(ctx.review_model_str()));
1801 let pr_comment_with_footer = format!("{}{}", pr_comment, footer);
1802 if let Err(e) = gh
1803 .add_comment(repo, &ctx.pr_number.to_string(), &pr_comment_with_footer)
1804 .await
1805 {
1806 tracing::warn!(
1807 task_id = task.id.0,
1808 pr_number = ctx.pr_number,
1809 error = %e,
1810 "failed to post automated review comment on PR"
1811 );
1812 }
1813 }
1814
1815 Ok(())
1816}
1817
1818#[allow(clippy::too_many_arguments)]
1819async fn apply_review_decision(
1820 task: &ExternalTask,
1821 backend: &Arc<dyn ExternalBackend>,
1822 repo: &str,
1823 task_manager: &Arc<TaskManager>,
1824 store: &Arc<TaskStore>,
1825 ctx: &ReviewContext,
1826 parsed: &ParsedReview,
1827 router_config: &crate::engine::router::RouterConfig,
1828) -> anyhow::Result<ReviewDecision> {
1829 restore_review_config_if_needed(&task.id.0, &ctx.worktree_path, &ctx.default_branch).await;
1830
1831 if let Err(e) = push_review_branch(task, repo, ctx, store).await {
1832 return Ok(ReviewDecision::Failed(format!("push failed: {e}")));
1833 }
1834
1835 post_review_comment(
1836 task,
1837 repo,
1838 ctx,
1839 &parsed.decision,
1840 &parsed.review_notes_for_comment,
1841 )
1842 .await?;
1843
1844 match &parsed.decision {
1845 ReviewDecision::Approve => {
1846 if let Err(e) = auto_merge_pr(
1851 task,
1852 &ctx.branch_name,
1853 backend,
1854 repo,
1855 &ctx.review_agent,
1856 ctx.review_model_str(),
1857 task_manager,
1858 store,
1859 )
1860 .await
1861 {
1862 let error_msg = e.to_string();
1863 if error_msg.contains("not yet computed") {
1864 tracing::warn!(
1865 task_id = task.id.0,
1866 pr_number = ctx.pr_number,
1867 branch = %ctx.branch_name,
1868 error = %e,
1869 "auto-merge deferred — PR mergeability not yet computed"
1870 );
1871 } else {
1872 tracing::error!(
1873 task_id = task.id.0,
1874 pr_number = ctx.pr_number,
1875 branch = %ctx.branch_name,
1876 error = %e,
1877 "auto-merge failed"
1878 );
1879 }
1880 return Ok(ReviewDecision::Failed(format!("merge failed: {e}")));
1881 }
1882 Ok(ReviewDecision::Approve)
1883 }
1884 ReviewDecision::RequestChanges { notes, issues } => {
1885 handle_review_changes(
1886 task,
1887 notes,
1888 issues,
1889 backend,
1890 repo,
1891 ctx.pr_number,
1892 &ctx.review_agent,
1893 ctx.review_model_str(),
1894 task_manager,
1895 store,
1896 router_config,
1897 )
1898 .await?;
1899 Ok(parsed.decision.clone())
1900 }
1901 _ => Ok(parsed.decision.clone()),
1902 }
1903}
1904
1905async fn recover_missing_review_worktree(
1906 task: &ExternalTask,
1907 repo: &str,
1908 store: &Arc<TaskStore>,
1909) -> anyhow::Result<worktree::WorktreeSetup> {
1910 let project_dir = crate::config::get_projects_with_paths()?
1911 .into_iter()
1912 .find(|(candidate_repo, _)| candidate_repo == repo)
1913 .map(|(_, project_dir)| project_dir)
1914 .ok_or_else(|| anyhow::anyhow!("no project directory configured for repo {repo}"))?;
1915
1916 tracing::warn!(
1917 task_id = task.id.0,
1918 repo,
1919 project_dir = %project_dir.display(),
1920 "review worktree missing, recreating via normal setup"
1921 );
1922
1923 let store_ref = Some(Arc::clone(store));
1924 let wt = worktree::setup_worktree(&task.id.0, &task.title, &project_dir, &store_ref, repo)
1925 .await
1926 .context("recreating missing review worktree")?;
1927
1928 if !wt.work_dir.exists() {
1929 anyhow::bail!(
1930 "recreated review worktree at {} for task {} is still missing",
1931 wt.work_dir.display(),
1932 task.id.0
1933 );
1934 }
1935
1936 Ok(wt)
1937}
1938
1939async fn count_ahead_commits(
1945 worktree_path: &std::path::Path,
1946 default_branch: &str,
1947) -> Result<u64, String> {
1948 let worktree_str = worktree_path.to_str().ok_or_else(|| {
1949 format!(
1950 "worktree path contains non-UTF-8 characters: {:?}",
1951 worktree_path
1952 )
1953 })?;
1954
1955 let output = tokio::process::Command::new("git")
1956 .args([
1957 "-C",
1958 worktree_str,
1959 "rev-list",
1960 "--count",
1961 &format!("origin/{default_branch}..HEAD"),
1962 ])
1963 .output()
1964 .await
1965 .map_err(|e| format!("git rev-list command failed to start: {e}"))?;
1966
1967 if !output.status.success() {
1968 let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string();
1969 return Err(format!(
1970 "git rev-list exited with {:?}: {}",
1971 output.status.code(),
1972 if stderr.is_empty() {
1973 String::from_utf8_lossy(&output.stdout).trim().to_string()
1974 } else {
1975 stderr
1976 }
1977 ));
1978 }
1979
1980 String::from_utf8_lossy(&output.stdout)
1981 .trim()
1982 .parse::<u64>()
1983 .map_err(|e| format!("could not parse git rev-list output as u64: {e}"))
1984}
1985
1986#[allow(clippy::too_many_arguments)]
1993async fn ensure_pr_exists(
1994 task: &ExternalTask,
1995 branch_name: &str,
1996 worktree_path: &std::path::Path,
1997 repo: &str,
1998 agent_summary: &str,
1999 store: &Arc<TaskStore>,
2000 task_manager: &Arc<TaskManager>,
2001 stored_pr_number: Option<u64>,
2002) -> anyhow::Result<EnsurePrResult> {
2003 let gh_check = GhHttp::new()?;
2004
2005 if let Some(n) = stored_pr_number {
2006 tracing::info!(
2007 task_id = task.id.0,
2008 pr_number = n,
2009 branch = %branch_name,
2010 "open PR found in store, proceeding with review"
2011 );
2012 return Ok(EnsurePrResult::Ready(n));
2013 }
2014
2015 match gh_check.get_pr_number(repo, branch_name).await {
2016 Ok(Some(n)) => {
2017 tracing::info!(
2018 task_id = task.id.0,
2019 pr_number = n,
2020 branch = %branch_name,
2021 "open PR found, proceeding with review"
2022 );
2023 Ok(EnsurePrResult::Ready(n))
2024 }
2025 Ok(None) => {
2026 let default_branch = worktree::detect_default_branch(worktree_path).await;
2028 let has_commits = match count_ahead_commits(worktree_path, &default_branch).await {
2029 Ok(count) => count > 0,
2030 Err(e) => {
2031 tracing::error!(
2032 task_id = task.id.0,
2033 branch = %branch_name,
2034 default_branch = %default_branch,
2035 err = %e,
2036 "could not determine if branch has commits ahead of default"
2037 );
2038 return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2039 format!("git ancestry check failed: {e}"),
2040 )));
2041 }
2042 };
2043
2044 if has_commits {
2045 tracing::warn!(
2047 task_id = task.id.0,
2048 branch = %branch_name,
2049 "no open PR but branch has commits — attempting to create PR"
2050 );
2051 let worktree_str = worktree_path.to_str().ok_or_else(|| {
2052 anyhow::anyhow!(
2053 "worktree path contains non-UTF-8 characters: {:?}",
2054 worktree_path
2055 )
2056 })?;
2057 let auth_env = runner::git_ops::build_git_auth_env();
2059
2060 let push_timeout = std::time::Duration::from_secs(120);
2065 match tokio::time::timeout(
2066 push_timeout,
2067 tokio::process::Command::new("git")
2068 .args(["-C", worktree_str, "push", "-u", "origin", branch_name])
2069 .envs(auth_env.iter().map(|(k, v)| (k.as_str(), v.as_str())))
2070 .output(),
2071 )
2072 .await
2073 {
2074 Err(_elapsed) => {
2075 tracing::warn!(
2076 task_id = task.id.0,
2077 branch = %branch_name,
2078 timeout_secs = push_timeout.as_secs(),
2079 "review gate fallback push timed out"
2080 );
2081 }
2082 Ok(Ok(o)) => {
2083 if !o.status.success() {
2084 tracing::warn!(
2085 task_id = task.id.0,
2086 branch = %branch_name,
2087 exit_code = ?o.status.code(),
2088 stdout = %String::from_utf8_lossy(&o.stdout),
2089 stderr = %String::from_utf8_lossy(&o.stderr),
2090 "review gate fallback push failed"
2091 );
2092 } else {
2093 tracing::info!(task_id = task.id.0, branch = %branch_name, "review gate fallback push succeeded");
2094 }
2095 }
2096 Ok(Err(e)) => {
2097 tracing::error!(
2098 task_id = task.id.0,
2099 branch = %branch_name,
2100 error = %e,
2101 "review gate fallback push command failed to run"
2102 );
2103 }
2104 }
2105
2106 let task_ref = runner::git_ops::format_task_ref(&task.id.0);
2107 let pr_body = format!(
2108 "Resolves {task_ref}\n\nAuto-created by orch review gate (agent forgot to open PR)"
2109 );
2110 let gh = GhHttp::new()?;
2111 match gh
2112 .create_pr(repo, &task.title, &pr_body, branch_name, &default_branch)
2113 .await
2114 {
2115 Ok(url) => match parse_pr_number_from_url(&url) {
2116 Ok(pr_num) => {
2117 store_set_result(
2118 &Some(Arc::clone(store)),
2119 repo,
2120 &task.id.0,
2121 &[("pr_number", serde_json::json!(pr_num as i64))],
2122 )
2123 .await?;
2124 tracing::info!(
2125 task_id = task.id.0,
2126 branch = %branch_name,
2127 pr_url = %url,
2128 "created missing PR via GhHttp — continuing review"
2129 );
2130 Ok(EnsurePrResult::Ready(pr_num))
2131 }
2132 Err(e) => {
2133 tracing::warn!(
2134 task_id = task.id.0,
2135 branch = %branch_name,
2136 pr_url = %url,
2137 error = %e,
2138 "created missing PR via GhHttp, but failed to parse PR number"
2139 );
2140 Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2141 format!("created missing PR, but could not parse PR number: {e}"),
2142 )))
2143 }
2144 },
2145 Err(e) => {
2146 let e_str = format!("{e}");
2147 if e_str.contains("already exists") {
2148 let backoff_delays = [2u64, 4u64];
2152 let mut found_pr: Option<u64> = None;
2153 for delay_secs in backoff_delays {
2154 tokio::time::sleep(tokio::time::Duration::from_secs(delay_secs))
2155 .await;
2156 if let Ok(Some(n)) = gh_check.get_pr_number(repo, branch_name).await
2157 {
2158 found_pr = Some(n);
2159 break;
2160 }
2161 }
2162 if let Some(n) = found_pr {
2163 tracing::info!(
2164 task_id = task.id.0,
2165 pr_number = n,
2166 branch = %branch_name,
2167 "found existing PR after create_pr 422 — retrying review"
2168 );
2169 store_set_result(
2170 &Some(Arc::clone(store)),
2171 repo,
2172 &task.id.0,
2173 &[("pr_number", serde_json::json!(n as i64))],
2174 )
2175 .await?;
2176 return Ok(EnsurePrResult::Ready(n));
2177 }
2178 }
2179 tracing::warn!(
2180 task_id = task.id.0,
2181 branch = %branch_name,
2182 error = %e,
2183 "create_pr failed via GhHttp, falling back to CLI"
2184 );
2185 let pr_timeout = std::time::Duration::from_secs(120);
2188 let mut gh_cmd = tokio::process::Command::new("gh");
2189 gh_cmd.args([
2190 "pr",
2191 "create",
2192 "--repo",
2193 repo,
2194 "--head",
2195 branch_name,
2196 "--title",
2197 &task.title,
2198 "--body",
2199 &pr_body,
2200 ]);
2201 gh_cmd.current_dir(worktree_path);
2202 gh_cmd.kill_on_drop(true);
2203
2204 let pr_result = tokio::time::timeout(pr_timeout, gh_cmd.output()).await;
2205 match pr_result {
2206 Ok(Ok(o)) if o.status.success() => {
2207 let stdout = String::from_utf8_lossy(&o.stdout);
2208 match parse_pr_number_from_url(stdout.trim()) {
2209 Ok(pr_num) => {
2210 store_set_result(
2211 &Some(Arc::clone(store)),
2212 repo,
2213 &task.id.0,
2214 &[("pr_number", serde_json::json!(pr_num as i64))],
2215 )
2216 .await?;
2217 tracing::info!(
2218 task_id = task.id.0,
2219 branch = %branch_name,
2220 "created missing PR via CLI — continuing review"
2221 );
2222 Ok(EnsurePrResult::Ready(pr_num))
2223 }
2224 Err(e) => {
2225 tracing::warn!(
2226 task_id = task.id.0,
2227 branch = %branch_name,
2228 stdout = %stdout,
2229 error = %e,
2230 "created missing PR via CLI, but failed to parse PR number"
2231 );
2232 Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2233 format!("created missing PR via CLI, but could not parse PR number: {e}")
2234 )))
2235 }
2236 }
2237 }
2238 Ok(Ok(o)) => {
2239 let stderr = String::from_utf8_lossy(&o.stderr);
2240 if stderr.contains("already exists") {
2241 if let Some(pr_url) =
2242 stderr.lines().find(|l| l.trim().starts_with("https://"))
2243 {
2244 match parse_pr_number_from_url(pr_url.trim()) {
2245 Ok(pr_num) => {
2246 if let Err(e) = store_set_result(
2247 &Some(Arc::clone(store)),
2248 repo,
2249 &task.id.0,
2250 &[(
2251 "pr_number",
2252 serde_json::json!(pr_num as i64),
2253 )],
2254 )
2255 .await
2256 {
2257 tracing::warn!(
2258 task_id = task.id.0,
2259 pr_num,
2260 err = %e,
2261 "failed to persist pr_number from already-exists recovery — downstream may retry PR creation"
2262 );
2263 }
2264 tracing::info!(
2265 task_id = task.id.0,
2266 branch = %branch_name,
2267 pr_url = %pr_url,
2268 "PR already exists (from CLI stderr) — retrying review"
2269 );
2270 return Ok(EnsurePrResult::Ready(pr_num));
2271 }
2272 Err(e) => {
2273 tracing::warn!(
2274 task_id = task.id.0,
2275 branch = %branch_name,
2276 pr_url = %pr_url,
2277 error = %e,
2278 "found existing PR URL in CLI stderr, but failed to parse PR number"
2279 );
2280 }
2282 }
2283 }
2284 }
2285 tracing::error!(
2286 task_id = task.id.0,
2287 branch = %branch_name,
2288 stderr = %stderr,
2289 "failed to create missing PR — work may be stuck"
2290 );
2291
2292 if crate::engine::runner::git_ops::is_transient_github_api_error(
2299 &stderr,
2300 ) {
2301 tracing::warn!(
2302 task_id = task.id.0,
2303 branch = %branch_name,
2304 "transient GitHub error creating PR; will retry later without incrementing persistent failure counter"
2305 );
2306 return Ok(EnsurePrResult::EarlyReturn(
2307 ReviewDecision::Failed(format!(
2308 "transient github error creating PR: {stderr}"
2309 )),
2310 ));
2311 }
2312
2313 match store_increment(
2314 &Some(Arc::clone(store)),
2315 repo,
2316 &task.id.0,
2317 "pr_create_failures",
2318 )
2319 .await
2320 {
2321 Ok(failures) if failures >= MAX_PR_CREATE_FAILURES => {
2322 return Ok(EnsurePrResult::EarlyReturn(
2323 ReviewDecision::Blocked(format!(
2324 "no PR, create failed {failures} times: {stderr}"
2325 )),
2326 ));
2327 }
2328 Ok(_) => {}
2329 Err(e) => {
2330 tracing::warn!(task_id = task.id.0, err = %e, "failed to increment pr_create_failures — skipping blocking decision this tick");
2331 }
2332 }
2333 Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2334 format!("no PR, create failed: {stderr}"),
2335 )))
2336 }
2337 Ok(Err(e)) => {
2338 tracing::error!(
2339 task_id = task.id.0,
2340 error = %e,
2341 "failed to run gh pr create"
2342 );
2343 let e_str = format!("{e}");
2344 if crate::engine::runner::git_ops::is_transient_github_api_error(
2345 &e_str,
2346 ) {
2347 tracing::warn!(
2348 task_id = task.id.0,
2349 branch = %branch_name,
2350 "transient GitHub error from gh CLI fallback; will retry later without incrementing persistent failure counter"
2351 );
2352 return Ok(EnsurePrResult::EarlyReturn(
2353 ReviewDecision::Failed(format!("transient gh error: {e}")),
2354 ));
2355 }
2356
2357 match store_increment(
2358 &Some(Arc::clone(store)),
2359 repo,
2360 &task.id.0,
2361 "pr_create_failures",
2362 )
2363 .await
2364 {
2365 Ok(failures) if failures >= MAX_PR_CREATE_FAILURES => {
2366 return Ok(EnsurePrResult::EarlyReturn(
2367 ReviewDecision::Blocked(format!(
2368 "no PR, gh error {failures} times: {e}"
2369 )),
2370 ));
2371 }
2372 Ok(_) => {}
2373 Err(e) => {
2374 tracing::warn!(task_id = task.id.0, err = %e, "failed to increment pr_create_failures — skipping blocking decision this tick");
2375 }
2376 }
2377 Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2378 format!("no PR, gh error: {e}"),
2379 )))
2380 }
2381 Err(_elapsed) => {
2382 tracing::error!(
2384 task_id = task.id.0,
2385 timeout_secs = pr_timeout.as_secs(),
2386 "gh pr create CLI fallback timed out"
2387 );
2388 let e_str = format!(
2389 "gh pr create timed out after {}s",
2390 pr_timeout.as_secs()
2391 );
2392 if crate::engine::runner::git_ops::is_transient_github_api_error(
2393 &e_str,
2394 ) {
2395 tracing::warn!(
2396 task_id = task.id.0,
2397 branch = %branch_name,
2398 "transient GitHub timeout from gh CLI fallback; will retry later without incrementing persistent failure counter"
2399 );
2400 return Ok(EnsurePrResult::EarlyReturn(
2401 ReviewDecision::Failed(format!(
2402 "transient gh error: {e_str}"
2403 )),
2404 ));
2405 }
2406
2407 match store_increment(
2408 &Some(Arc::clone(store)),
2409 repo,
2410 &task.id.0,
2411 "pr_create_failures",
2412 )
2413 .await
2414 {
2415 Ok(failures) if failures >= MAX_PR_CREATE_FAILURES => {
2416 return Ok(EnsurePrResult::EarlyReturn(
2417 ReviewDecision::Blocked(format!(
2418 "no PR, gh timeout {failures} times: {e_str}"
2419 )),
2420 ));
2421 }
2422 Ok(_) => {}
2423 Err(e) => {
2424 tracing::warn!(task_id = task.id.0, err = %e, "failed to increment pr_create_failures — skipping blocking decision this tick");
2425 }
2426 }
2427 Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2428 format!("no PR, gh timeout: {e_str}"),
2429 )))
2430 }
2431 }
2432 }
2433 }
2434 } else {
2435 let merged = match gh_check.is_pr_merged(repo, branch_name).await {
2437 Ok(v) => v,
2438 Err(e) => {
2439 tracing::warn!(task_id = task.id.0, branch = %branch_name, err = %e, "merge check failed, skipping task this tick");
2440 return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2441 format!("merge check failed: {e}"),
2442 )));
2443 }
2444 };
2445 if merged {
2446 tracing::info!(
2447 task_id = task.id.0,
2448 branch = %branch_name,
2449 "PR already merged, marking done"
2450 );
2451 if let Err(e) = task_manager
2452 .update_task_status(&task.id, Status::Done)
2453 .await
2454 {
2455 tracing::error!(task_id = task.id.0, err = %e, "update_task_status(Done) failed — task may be stuck in InReview");
2456 }
2457 return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Skipped));
2458 }
2459
2460 let last_error = opt_store_get_task(&Some(Arc::clone(store)), repo, &task.id.0)
2461 .await
2462 .map(|t| t.last_error)
2463 .unwrap_or_default();
2464 let reason = if !agent_summary.is_empty() {
2465 agent_summary.to_string()
2466 } else {
2467 last_error.clone()
2468 };
2469
2470 if last_error.contains("exceeded max attempts") {
2471 tracing::warn!(
2472 task_id = task.id.0,
2473 branch = %branch_name,
2474 "no PR and no commits after max attempts — marking blocked to stop loop"
2475 );
2476 let fields = [
2480 (
2481 "block_reason",
2482 serde_json::json!("max attempts exceeded — no PR or commits produced"),
2483 ),
2484 ("last_error", serde_json::json!(last_error)),
2485 ];
2486 if let Err(e) = task_manager
2487 .update_task_status_and_result(&task.id, Status::Blocked, &fields)
2488 .await
2489 {
2490 tracing::error!(task_id = task.id.0, err = %e, "update_task_status_and_result(Blocked) failed — skipping block to avoid silent auto-unblock loop");
2491 return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2492 format!("failed to write block_reason: {e}"),
2493 )));
2494 }
2495 return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Skipped));
2496 }
2497
2498 if last_error.contains("422") && last_error.contains("head") {
2499 tracing::info!(
2500 task_id = task.id.0,
2501 branch = %branch_name,
2502 "no PR — 422/head-invalid means work already merged, marking done"
2503 );
2504 if let Err(e) = task_manager
2505 .update_task_status(&task.id, Status::Done)
2506 .await
2507 {
2508 tracing::error!(task_id = task.id.0, err = %e, "update_task_status(Done) failed");
2509 }
2510 return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Skipped));
2511 }
2512
2513 if last_error.contains("no fallback agents")
2514 || last_error.contains("all agents exhausted")
2515 {
2516 tracing::warn!(
2517 task_id = task.id.0,
2518 branch = %branch_name,
2519 last_error = %last_error,
2520 "no PR and no commits after failover exhaustion — marking blocked to stop loop"
2521 );
2522 let fields = [
2526 (
2527 "block_reason",
2528 serde_json::json!("all agents exhausted — no PR or commits produced"),
2529 ),
2530 ("last_error", serde_json::json!(last_error)),
2531 ];
2532 if let Err(e) = task_manager
2533 .update_task_status_and_result(&task.id, Status::Blocked, &fields)
2534 .await
2535 {
2536 tracing::error!(task_id = task.id.0, err = %e, "update_task_status_and_result(Blocked) failed — skipping block to avoid silent auto-unblock loop");
2537 return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2538 format!("failed to write block_reason: {e}"),
2539 )));
2540 }
2541 return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Skipped));
2542 }
2543
2544 let max_reroutes: u32 = config::get("workflow.max_reroute_attempts")
2551 .ok()
2552 .and_then(|s| s.parse().ok())
2553 .or_else(|| {
2554 config::get("workflow.max_attempts")
2555 .ok()
2556 .and_then(|s| s.parse().ok())
2557 })
2558 .unwrap_or(3);
2559
2560 tracing::warn!(
2561 task_id = task.id.0,
2562 branch = %branch_name,
2563 reason = %reason,
2564 "no PR and no commits — re-routing for retry"
2565 );
2566
2567 if should_skip_no_code_reroute_increment(&last_error) {
2571 tracing::warn!(
2572 task_id = task.id.0,
2573 last_error = %last_error,
2574 "transient GitHub error recorded; skipping persistent no_pr_reroutes increment"
2575 );
2576 return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2577 format!("transient github error: {last_error}"),
2578 )));
2579 }
2580
2581 let reroutes = match store_increment(
2583 &Some(Arc::clone(store)),
2584 repo,
2585 &task.id.0,
2586 "no_code_reroutes",
2587 )
2588 .await
2589 {
2590 Ok(v) => v,
2591 Err(e) => {
2592 tracing::warn!(task_id = task.id.0, err = %e, "failed to increment no_code_reroutes — skipping reroute/block decision this tick");
2593 return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2595 format!("transient store error: {e}"),
2596 )));
2597 }
2598 };
2599
2600 if reroutes as u32 >= max_reroutes {
2601 tracing::error!(
2602 task_id = task.id.0,
2603 reroutes,
2604 max_reroutes,
2605 "reached max reroute attempts for no-pr-result — blocking for human review"
2606 );
2607 let msg = format!(
2608 "no PR or code changes after {}/{} reroute attempts",
2609 reroutes, max_reroutes
2610 );
2611 let fields = [
2615 (
2616 "block_reason",
2617 serde_json::json!(format!(
2618 "max reroute attempts ({}) reached — no PR or code changes produced",
2619 max_reroutes
2620 )),
2621 ),
2622 ("last_error", serde_json::json!(msg)),
2623 ("agent", serde_json::json!(null)),
2624 ("model", serde_json::json!(null)),
2625 ];
2626 if let Err(e) = task_manager
2627 .update_task_status_and_result(&task.id, Status::Blocked, &fields)
2628 .await
2629 {
2630 tracing::error!(task_id = task.id.0, err = %e, "update_task_status_and_result(Blocked) failed — skipping block to avoid silent auto-unblock loop");
2631 }
2632 Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Blocked(msg)))
2633 } else if task.id.0.starts_with("internal:") {
2634 tracing::info!(
2641 task_id = task.id.0,
2642 "no PR and no commits for internal task — marking done"
2643 );
2644 if let Err(e) = task_manager
2645 .update_task_status(&task.id, Status::Done)
2646 .await
2647 {
2648 tracing::error!(task_id = task.id.0, err = %e, "update_task_status(Done) failed — task may be stuck in InReview");
2649 }
2650 Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Skipped))
2651 } else {
2652 if let Err(e) = store_set_result(
2657 &Some(Arc::clone(store)),
2658 repo,
2659 &task.id.0,
2660 &[
2661 ("agent", serde_json::json!(null)),
2662 ("model", serde_json::json!(null)),
2663 (
2664 "last_error",
2665 serde_json::json!("no PR or code changes produced"),
2666 ),
2667 ],
2668 )
2669 .await
2670 {
2671 tracing::error!(task_id = task.id.0, err = %e, "failed to clear agent/model for reroute — skipping status transition");
2672 } else if let Err(e) =
2674 task_manager.update_task_status(&task.id, Status::New).await
2675 {
2676 tracing::error!(task_id = task.id.0, err = %e, "update_task_status(New) failed — task may be stuck in InReview");
2677 }
2678 Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Rerouted))
2679 }
2680 }
2681 }
2682 Err(e) => {
2683 tracing::warn!(
2684 task_id = task.id.0,
2685 branch = %branch_name,
2686 error = %e,
2687 "failed to check PR status"
2688 );
2689 Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2690 format!("PR check failed: {e}"),
2691 )))
2692 }
2693 }
2694}
2695
2696pub(crate) async fn review_and_merge(
2702 task: &ExternalTask,
2703 backend: &Arc<dyn ExternalBackend>,
2704 tmux: &Arc<TmuxManager>,
2705 repo: &str,
2706 router: &Arc<RwLock<Router>>,
2707 task_manager: &Arc<TaskManager>,
2708 store: &Arc<TaskStore>,
2709) -> anyhow::Result<ReviewDecision> {
2710 set_review_session_expected(store, repo, &task.id.0, true).await;
2711
2712 let ctx = match build_review_context(task, repo, router, task_manager, store).await? {
2713 ReviewPhase::Ready(ctx) => ctx,
2714 ReviewPhase::EarlyReturn(decision) => return Ok(decision),
2715 };
2716
2717 let run = match invoke_review_agent(task, tmux, &ctx, repo, store).await {
2718 ReviewPhase::Ready(run) => run,
2719 ReviewPhase::EarlyReturn(decision) => return Ok(decision),
2720 };
2721
2722 let parsed = match parse_review_output(task, repo, &ctx, run, store).await {
2723 ReviewPhase::Ready(parsed) => parsed,
2724 ReviewPhase::EarlyReturn(decision) => return Ok(decision),
2725 };
2726
2727 finalize_review_run(task, &ctx, &parsed, store).await;
2728
2729 let router_config = router.read().await.config.clone();
2730 apply_review_decision(
2731 task,
2732 backend,
2733 repo,
2734 task_manager,
2735 store,
2736 &ctx,
2737 &parsed,
2738 &router_config,
2739 )
2740 .await
2741}
2742
2743#[cfg(test)]
2744mod tests {
2745 use super::*;
2746
2747 use crate::backends::ExternalTask;
2748 use crate::engine::router::RouterConfig;
2749 use crate::github::types::{GitHubReview, GitHubReviewComment, GitHubUser, PullRequestReview};
2750 use crate::store::{TaskStatus, TaskStore};
2751 use tempfile::TempDir;
2752
2753 #[test]
2756 fn outcome_for_agent_error_kimi_billing_cycle_is_not_rate_limit() {
2757 let err = runner::agents::AgentError::RateLimit {
2761 message: "Failed to authenticate. API Error: 403 You've reached your usage limit \
2762 for this billing cycle. Your quota will be refreshed soon."
2763 .to_string(),
2764 };
2765 assert_eq!(outcome_for_agent_error(&err), "billing_cycle_exhausted");
2766 }
2767
2768 #[test]
2769 fn outcome_for_agent_error_generic_rate_limit_unaffected() {
2770 let err = runner::agents::AgentError::RateLimit {
2771 message: "429 Too Many Requests".to_string(),
2772 };
2773 assert_eq!(outcome_for_agent_error(&err), "rate_limit");
2774 }
2775
2776 #[test]
2777 fn outcome_for_agent_error_generic_auth_unaffected() {
2778 let err = runner::agents::AgentError::Auth {
2779 message: "401 Unauthorized".to_string(),
2780 };
2781 assert_eq!(outcome_for_agent_error(&err), "auth_error");
2782 }
2783
2784 #[test]
2787 fn parse_pr_number_standard_url() {
2788 assert_eq!(
2789 parse_pr_number_from_url("https://github.com/owner/repo/pull/42"),
2790 Ok(42)
2791 );
2792 }
2793
2794 #[test]
2795 fn parse_pr_number_trailing_slash() {
2796 assert_eq!(
2797 parse_pr_number_from_url("https://github.com/owner/repo/pull/42/"),
2798 Ok(42)
2799 );
2800 }
2801
2802 #[test]
2803 fn parse_pr_number_with_query_string() {
2804 assert_eq!(
2805 parse_pr_number_from_url("https://github.com/owner/repo/pull/42?tab=files"),
2806 Ok(42)
2807 );
2808 }
2809
2810 #[test]
2811 fn parse_pr_number_with_hash_fragment() {
2812 assert_eq!(
2813 parse_pr_number_from_url("https://github.com/owner/repo/pull/42#issuecomment-123"),
2814 Ok(42)
2815 );
2816 }
2817
2818 #[test]
2819 fn parse_pr_number_with_query_and_hash() {
2820 assert_eq!(
2821 parse_pr_number_from_url("https://github.com/owner/repo/pull/42?quick_pull=1#top"),
2822 Ok(42)
2823 );
2824 }
2825
2826 #[test]
2827 fn parse_pr_number_with_leading_whitespace() {
2828 assert_eq!(
2829 parse_pr_number_from_url(" https://github.com/owner/repo/pull/42\n"),
2830 Ok(42)
2831 );
2832 }
2833
2834 #[test]
2835 fn parse_pr_number_wrong_domain() {
2836 let url = "https://gitlab.com/owner/repo/merge_requests/42";
2837 let result = parse_pr_number_from_url(url);
2838 assert!(result.is_err());
2839 assert!(result
2840 .unwrap_err()
2841 .contains("does not start with 'https://github.com/'"));
2842 }
2843
2844 #[test]
2845 fn parse_pr_number_missing_pull_segment() {
2846 let url = "https://github.com/owner/repo/issues/42";
2848 let result = parse_pr_number_from_url(url);
2849 assert!(result.is_err());
2850 assert!(result.unwrap_err().contains("missing '/pull/' segment"));
2851 }
2852
2853 #[test]
2854 fn parse_pr_number_zero_is_rejected() {
2855 let url = "https://github.com/owner/repo/pull/0";
2856 let result = parse_pr_number_from_url(url);
2857 assert!(result.is_err());
2858 assert!(result.unwrap_err().contains("cannot be zero"));
2859 }
2860
2861 #[test]
2862 fn parse_pr_number_non_numeric_segment() {
2863 let url = "https://github.com/owner/repo/pull/abc";
2864 let result = parse_pr_number_from_url(url);
2865 assert!(result.is_err());
2866 assert!(result.unwrap_err().contains("not a valid positive integer"));
2867 }
2868
2869 #[test]
2870 fn parse_pr_number_empty_string() {
2871 let result = parse_pr_number_from_url("");
2872 assert!(result.is_err());
2873 assert!(result.unwrap_err().contains("empty"));
2874 }
2875
2876 #[test]
2877 fn parse_pr_number_http_not_https() {
2878 let url = "http://github.com/owner/repo/pull/42";
2880 let result = parse_pr_number_from_url(url);
2881 assert!(result.is_err());
2882 assert!(result
2883 .unwrap_err()
2884 .contains("does not start with 'https://github.com/'"));
2885 }
2886
2887 #[test]
2895 fn format_review_error_summarises_rate_limit_fragment_with_api_retry() {
2896 let raw_fragment = "{\"attempt\":6,\"max_retries\":10,\"retry_delay_ms\":17266.789,\"error_status\":429,\"error\":\"rate_limit\",\"session_id\":\"5c7148ee\"}\n\
2900 {\"type\":\"system\",\"subtype\":\"api_retry\",\"attempt\":7,\"max_retries\":10,\"retry_delay_ms\":35162.66,\"error_status\":429,\"error\":\"rate_limit\",\"session_id\":\"5c7148ee\"}";
2901 let err = runner::agents::AgentError::RateLimit {
2902 message: raw_fragment.to_string(),
2903 };
2904
2905 let stored = format_review_error("minimax", &err);
2906
2907 assert!(
2910 !stored.contains("{\"type\""),
2911 "stored error must not contain raw NDJSON envelope, got: {stored}"
2912 );
2913 assert!(
2914 !stored.contains("subtype"),
2915 "stored error must not contain raw api_retry fragment, got: {stored}"
2916 );
2917 assert!(
2918 !stored.contains("session_id"),
2919 "stored error must not contain session_id from provider payload, got: {stored}"
2920 );
2921 assert_eq!(
2922 stored,
2923 "minimax rate limit: status=429 after 7 attempts (last delay 35s)"
2924 );
2925 }
2926
2927 #[test]
2928 fn format_review_error_summarises_truncated_fragment_without_subtype_marker() {
2929 let raw_fragment = ",\"attempt\":5,\"max_retries\":10,\"retry_delay_ms\":9034.81,\"error_status\":429,\"error\":\"rate_limit\",\"session_id\":\"992e17d3\"";
2934 let err = runner::agents::AgentError::RateLimit {
2935 message: raw_fragment.to_string(),
2936 };
2937
2938 let stored = format_review_error("glm", &err);
2939
2940 assert!(stored.starts_with("glm rate limit:"));
2941 assert!(
2942 !stored.contains("session_id"),
2943 "stored error must not leak session_id from truncated fragment, got: {stored}"
2944 );
2945 assert!(
2948 stored.len() <= 200,
2949 "stored error should be compact (<=200 chars), got {} chars: {stored}",
2950 stored.len()
2951 );
2952 }
2953
2954 #[test]
2955 fn format_review_error_preserves_non_rate_limit_variants() {
2956 let err = runner::agents::AgentError::Auth {
2957 message: "invalid bearer token".to_string(),
2958 };
2959 assert_eq!(
2960 format_review_error("kimi", &err),
2961 "kimi auth error: invalid bearer token"
2962 );
2963
2964 let err = runner::agents::AgentError::Timeout {
2965 elapsed: std::time::Duration::from_secs(1801),
2966 };
2967 assert_eq!(
2968 format_review_error("kimi", &err),
2969 "kimi timed out after 1801s"
2970 );
2971 }
2972
2973 fn git(dir: &std::path::Path, args: &[&str]) {
2974 let mut cmd = std::process::Command::new("git");
2975 cmd.current_dir(dir);
2976 if args.first() == Some(&"commit") {
2978 cmd.args(["-c", "commit.gpgsign=false"]);
2979 }
2980 let status = cmd.args(args).status().unwrap();
2981 assert!(status.success(), "git {:?} failed", args);
2982 }
2983
2984 #[test]
2985 fn test_pull_request_review_requests_changes() {
2986 let review = PullRequestReview {
2987 review: GitHubReview {
2988 id: 1,
2989 user: GitHubUser {
2990 login: "reviewer".to_string(),
2991 },
2992 body: Some("Please fix".to_string()),
2993 state: "CHANGES_REQUESTED".to_string(),
2994 html_url: None,
2995 submitted_at: "2024-01-01T00:00:00Z".to_string(),
2996 commit_id: None,
2997 },
2998 comments: vec![],
2999 };
3000 assert!(review.requests_changes());
3001 }
3002
3003 #[test]
3004 fn test_pull_request_review_does_not_request_changes() {
3005 let review = PullRequestReview {
3006 review: GitHubReview {
3007 id: 1,
3008 user: GitHubUser {
3009 login: "reviewer".to_string(),
3010 },
3011 body: Some("LGTM".to_string()),
3012 state: "APPROVED".to_string(),
3013 html_url: None,
3014 submitted_at: "2024-01-01T00:00:00Z".to_string(),
3015 commit_id: None,
3016 },
3017 comments: vec![],
3018 };
3019 assert!(!review.requests_changes());
3020 }
3021
3022 #[test]
3023 fn test_pull_request_review_actionable_comments_filters_empty_and_replies() {
3024 let review = PullRequestReview {
3025 review: GitHubReview {
3026 id: 1,
3027 user: GitHubUser {
3028 login: "reviewer".to_string(),
3029 },
3030 body: None,
3031 state: "CHANGES_REQUESTED".to_string(),
3032 html_url: None,
3033 submitted_at: "2024-01-01T00:00:00Z".to_string(),
3034 commit_id: None,
3035 },
3036 comments: vec![
3037 GitHubReviewComment {
3038 id: 1,
3039 user: GitHubUser {
3040 login: "reviewer".to_string(),
3041 },
3042 body: "Fix this issue".to_string(),
3043 path: "src/main.rs".to_string(),
3044 line: Some(10),
3045 original_line: Some(10),
3046 commit_id: "abc".to_string(),
3047 original_commit_id: "abc".to_string(),
3048 html_url: "url".to_string(),
3049 created_at: "2024-01-01T00:00:00Z".to_string(),
3050 updated_at: "2024-01-01T00:00:00Z".to_string(),
3051 in_reply_to_id: None,
3052 diff_hunk: Some(
3053 "@@ -8,5 +8,5 @@ fn main() {\n- let x = 1;\n+ let x = 2;".to_string(),
3054 ),
3055 },
3056 GitHubReviewComment {
3057 id: 2,
3058 user: GitHubUser {
3059 login: "reviewer".to_string(),
3060 },
3061 body: "".to_string(),
3062 path: "src/lib.rs".to_string(),
3063 line: Some(20),
3064 original_line: Some(20),
3065 commit_id: "abc".to_string(),
3066 original_commit_id: "abc".to_string(),
3067 html_url: "url".to_string(),
3068 created_at: "2024-01-01T00:00:00Z".to_string(),
3069 updated_at: "2024-01-01T00:00:00Z".to_string(),
3070 in_reply_to_id: None,
3071 diff_hunk: None,
3072 },
3073 GitHubReviewComment {
3074 id: 3,
3075 user: GitHubUser {
3076 login: "reviewer".to_string(),
3077 },
3078 body: "Reply to this".to_string(),
3079 path: "src/lib.rs".to_string(),
3080 line: Some(30),
3081 original_line: Some(30),
3082 commit_id: "abc".to_string(),
3083 original_commit_id: "abc".to_string(),
3084 html_url: "url".to_string(),
3085 created_at: "2024-01-01T00:00:00Z".to_string(),
3086 updated_at: "2024-01-01T00:00:00Z".to_string(),
3087 in_reply_to_id: Some(1),
3088 diff_hunk: None,
3089 },
3090 ],
3091 };
3092 let actionable = review.actionable_comments();
3093 assert_eq!(actionable.len(), 1);
3094 assert_eq!(actionable[0].id, 1);
3095 assert_eq!(actionable[0].body, "Fix this issue");
3096 assert_eq!(actionable[0].path, "src/main.rs");
3097 assert_eq!(
3098 actionable[0].diff_hunk.as_ref().unwrap(),
3099 "@@ -8,5 +8,5 @@ fn main() {\n- let x = 1;\n+ let x = 2;"
3100 );
3101 }
3102
3103 #[test]
3104 fn test_router_config_model_for_complexity_returns_none_without_config() {
3105 let cfg = RouterConfig::default();
3108 assert!(cfg.model_for_complexity("claude", "simple", "").is_none());
3109 assert!(cfg.model_for_complexity("opencode", "review", "").is_none());
3110 assert!(cfg
3111 .model_for_complexity("unknown_agent_xyz", "simple", "")
3112 .is_none());
3113 }
3114
3115 #[test]
3116 fn review_status_maps_decisions() {
3117 assert_eq!(
3118 review_status(&ReviewDecision::Approve).unwrap().0,
3119 "success"
3120 );
3121 let changes = ReviewDecision::RequestChanges {
3122 notes: String::new(),
3123 issues: vec![],
3124 };
3125 assert_eq!(review_status(&changes).unwrap().0, "failure");
3126 assert!(review_status(&ReviewDecision::Failed("x".into())).is_none());
3127 }
3128
3129 #[test]
3130 fn review_started_comment_includes_agent_and_model() {
3131 let expected_footer = attribution_footer("Review started", "kimi", Some("opus"));
3132 let expected = format!("🔍 Automated review started{}", expected_footer);
3133 assert_eq!(review_started_comment("kimi", "opus"), expected);
3134 }
3135
3136 fn no_code_review_task(id: &str) -> ExternalTask {
3139 ExternalTask {
3140 id: crate::backends::ExternalId(id.to_string()),
3141 title: "opencode discovery timeout".to_string(),
3142 body: "".to_string(),
3143 state: "open".to_string(),
3144 labels: vec![],
3145 author: "tester".to_string(),
3146 created_at: "2026-01-01T00:00:00Z".to_string(),
3147 updated_at: "2026-01-01T00:00:00Z".to_string(),
3148 url: "https://example.com/issues/1".to_string(),
3149 }
3150 }
3151
3152 #[tokio::test]
3153 async fn build_review_context_reroutes_external_task_with_no_code() {
3154 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3155 let repo = "owner/repo";
3156 let ext = no_code_review_task("3617");
3157 store.ensure_external_task(repo, &ext).await.unwrap();
3158
3159 let backend = Arc::new(crate::backends::test_helpers::NoopBackend);
3160 let task_manager = Arc::new(TaskManager::with_store(
3161 backend,
3162 Arc::clone(&store),
3163 repo.to_string(),
3164 ));
3165 let router = Arc::new(RwLock::new(Router::new(RouterConfig::default())));
3166
3167 let phase = build_review_context(&ext, repo, &router, &task_manager, &store)
3168 .await
3169 .unwrap();
3170
3171 assert!(
3172 matches!(phase, ReviewPhase::EarlyReturn(ReviewDecision::Rerouted)),
3173 "external task with no worktree/branch/pr must reroute, not skip-to-done"
3174 );
3175
3176 let stored = store
3177 .get_by_external_id(repo, &ext.id.0)
3178 .await
3179 .unwrap()
3180 .expect("task must still exist");
3181 assert_eq!(stored.status, TaskStatus::New);
3182 assert!(stored.agent.is_none());
3183 assert!(stored.model.is_none());
3184 }
3185
3186 #[tokio::test]
3187 async fn build_review_context_blocks_external_task_after_max_reroutes() {
3188 let temp_home = TempDir::new().unwrap();
3191 let orch_dir = temp_home.path().join(".orch");
3192 std::fs::create_dir_all(&orch_dir).unwrap();
3193 let old_orch_home = std::env::var_os("ORCH_HOME");
3194 std::env::set_var("ORCH_HOME", &orch_dir);
3195
3196 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3197 let repo = "owner/repo";
3198 let ext = no_code_review_task("3618");
3199 store.ensure_external_task(repo, &ext).await.unwrap();
3200
3201 let backend = Arc::new(crate::backends::test_helpers::NoopBackend);
3202 let task_manager = Arc::new(TaskManager::with_store(
3203 backend,
3204 Arc::clone(&store),
3205 repo.to_string(),
3206 ));
3207 let router = Arc::new(RwLock::new(Router::new(RouterConfig::default())));
3208
3209 for _ in 0..3 {
3211 let phase = build_review_context(&ext, repo, &router, &task_manager, &store)
3212 .await
3213 .unwrap();
3214 if let ReviewPhase::EarlyReturn(ReviewDecision::Blocked(_)) = phase {
3215 break;
3216 }
3217 }
3218
3219 let stored = store
3220 .get_by_external_id(repo, &ext.id.0)
3221 .await
3222 .unwrap()
3223 .expect("task must still exist");
3224 assert_eq!(stored.status, TaskStatus::Blocked);
3225
3226 if let Some(old) = old_orch_home {
3227 std::env::set_var("ORCH_HOME", old);
3228 } else {
3229 std::env::remove_var("ORCH_HOME");
3230 }
3231 }
3232
3233 #[tokio::test]
3234 async fn build_review_context_marks_internal_task_done_with_no_code() {
3235 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3236 let repo = "owner/repo";
3237 let internal_id = store
3238 .create_internal(repo, "Daily retrospective", "", "job", "job-1", None)
3239 .await
3240 .unwrap();
3241 let ext_id = format!("internal:{internal_id}");
3242 let ext = no_code_review_task(&ext_id);
3243
3244 let backend = Arc::new(crate::backends::test_helpers::NoopBackend);
3245 let task_manager = Arc::new(TaskManager::with_store(
3246 backend,
3247 Arc::clone(&store),
3248 repo.to_string(),
3249 ));
3250 let router = Arc::new(RwLock::new(Router::new(RouterConfig::default())));
3251
3252 let phase = build_review_context(&ext, repo, &router, &task_manager, &store)
3253 .await
3254 .unwrap();
3255
3256 assert!(matches!(
3257 phase,
3258 ReviewPhase::EarlyReturn(ReviewDecision::Skipped)
3259 ));
3260 let stored = store.get(internal_id).await.unwrap();
3261 assert_eq!(stored.status, TaskStatus::Done);
3262 }
3263
3264 #[test]
3265 fn no_code_reroute_skip_check_rejects_agent_stream_disconnects() {
3266 let last_error =
3267 "codex failed: Reconnecting... (stream disconnected before completion: Broken pipe)";
3268 assert!(!should_skip_no_code_reroute_increment(last_error));
3269 }
3270
3271 #[test]
3280 fn review_fallback_scan_ignores_rate_limit_word_buried_in_long_report() {
3281 let padding = "x".repeat(10_000);
3282 let text_for_review = format!(
3283 "{padding}\n+| codex | gpt-5.4 | failed | 1 |\n+| codex | gpt-5.5 | failed / aborted / (empty) | 1 each |\n+| kimi | opus | rate_limit | 1 |\n{padding}"
3284 );
3285
3286 let scan_head = runner::agents::patterns::safe_head(
3287 &text_for_review,
3288 runner::agents::patterns::RATE_LIMIT_SCAN_HEAD_BYTES,
3289 );
3290 let scan_tail = runner::agents::patterns::safe_tail(
3291 &text_for_review,
3292 runner::agents::patterns::RATE_LIMIT_SCAN_TAIL_BYTES,
3293 );
3294
3295 assert!(runner::agents::patterns::detect_rate_limit(scan_head).is_none());
3296 assert!(runner::agents::patterns::detect_rate_limit(scan_tail).is_none());
3297 }
3298
3299 #[test]
3300 fn review_fallback_scan_still_detects_real_rate_limit_at_tail() {
3301 let padding = "x".repeat(10_000);
3302 let text_for_review =
3303 format!("{padding}\nError: rate limit exceeded, please try again later");
3304
3305 let scan_tail = runner::agents::patterns::safe_tail(
3306 &text_for_review,
3307 runner::agents::patterns::RATE_LIMIT_SCAN_TAIL_BYTES,
3308 );
3309
3310 assert!(runner::agents::patterns::detect_rate_limit(scan_tail).is_some());
3311 }
3312
3313 #[tokio::test]
3314 async fn recover_missing_review_worktree_recreates_worktree() {
3315 let temp_home = TempDir::new().unwrap();
3316 let orch_dir = temp_home.path().join(".orch");
3317 std::fs::create_dir_all(&orch_dir).unwrap();
3318 let old_orch_home = std::env::var_os("ORCH_HOME");
3319 std::env::set_var("ORCH_HOME", &orch_dir);
3320
3321 let project_dir = temp_home.path().join("project");
3322 std::fs::create_dir_all(&project_dir).unwrap();
3323 git(&project_dir, &["init", "-b", "main"]);
3324 git(&project_dir, &["config", "user.name", "Test User"]);
3325 git(&project_dir, &["config", "user.email", "test@example.com"]);
3326 std::fs::write(project_dir.join("README.md"), "hello").unwrap();
3327 git(&project_dir, &["add", "README.md"]);
3328 git(&project_dir, &["commit", "-m", "init"]);
3329
3330 std::fs::write(
3331 orch_dir.join("config.yml"),
3332 format!("projects:\n - {}\n", project_dir.display()),
3333 )
3334 .unwrap();
3335 std::fs::write(project_dir.join(".orch.yml"), "gh:\n repo: owner/repo\n").unwrap();
3336
3337 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3338 let repo = "owner/repo";
3339 store
3340 .create_internal(repo, "Fix review recovery", "", "review", "review-1", None)
3341 .await
3342 .unwrap();
3343 let ext = ExternalTask {
3344 id: crate::backends::ExternalId("1".to_string()),
3345 title: "Fix review recovery".to_string(),
3346 body: "".to_string(),
3347 state: "open".to_string(),
3348 labels: vec![],
3349 author: "tester".to_string(),
3350 created_at: "2026-01-01T00:00:00Z".to_string(),
3351 updated_at: "2026-01-01T00:00:00Z".to_string(),
3352 url: "https://example.com/issues/1".to_string(),
3353 };
3354 store.ensure_external_task(repo, &ext).await.unwrap();
3355
3356 let recovered = recover_missing_review_worktree(&ext, repo, &store)
3357 .await
3358 .unwrap();
3359
3360 assert!(recovered.work_dir.exists());
3361 assert!(recovered.branch.starts_with("gh-issue-1-"));
3362
3363 if let Some(old) = old_orch_home {
3364 std::env::set_var("ORCH_HOME", old);
3365 } else {
3366 std::env::remove_var("ORCH_HOME");
3367 }
3368 }
3369
3370 #[tokio::test]
3373 async fn count_ahead_commits_returns_zero_when_no_ahead_commits() {
3374 let temp = TempDir::new().unwrap();
3375 let dir = temp.path();
3376
3377 let bare = temp.path().join("bare.git");
3379 std::fs::create_dir_all(&bare).unwrap();
3380 std::process::Command::new("git")
3381 .args(["init", "--bare"])
3382 .current_dir(&bare)
3383 .output()
3384 .unwrap();
3385
3386 git(dir, &["init", "-b", "main"]);
3387 git(dir, &["config", "user.name", "Test"]);
3388 git(dir, &["config", "user.email", "test@test.com"]);
3389 std::fs::write(dir.join("f"), "a").unwrap();
3390 git(dir, &["add", "f"]);
3391 git(dir, &["commit", "-m", "init"]);
3392 std::process::Command::new("git")
3393 .args(["remote", "add", "origin", bare.to_str().unwrap()])
3394 .current_dir(dir)
3395 .output()
3396 .unwrap();
3397 std::process::Command::new("git")
3398 .args(["push", "origin", "main"])
3399 .current_dir(dir)
3400 .output()
3401 .unwrap();
3402
3403 let result = count_ahead_commits(dir, "main").await;
3405 assert!(
3406 result.is_ok(),
3407 "expected Ok when origin/main exists, got {result:?}"
3408 );
3409 assert_eq!(
3410 result.unwrap(),
3411 0,
3412 "expected 0 commits ahead when branch matches origin/main"
3413 );
3414 }
3415
3416 #[tokio::test]
3417 async fn count_ahead_commits_returns_error_when_remote_ref_missing() {
3418 let temp = TempDir::new().unwrap();
3419 let dir = temp.path();
3420 git(dir, &["init", "-b", "main"]);
3421 git(dir, &["config", "user.name", "Test"]);
3422 git(dir, &["config", "user.email", "test@test.com"]);
3423 std::fs::write(dir.join("f"), "a").unwrap();
3424 git(dir, &["add", "f"]);
3425 git(dir, &["commit", "-m", "init"]);
3426 let result = count_ahead_commits(dir, "main").await;
3428 assert!(
3429 result.is_err(),
3430 "expected Err when origin/main does not exist, got {result:?}"
3431 );
3432 let err = result.unwrap_err();
3433 assert!(
3434 err.contains("git rev-list exited"),
3435 "error should mention exit failure, got: {err}"
3436 );
3437 }
3438
3439 #[tokio::test]
3440 async fn count_ahead_commits_returns_error_when_git_cannot_start() {
3441 let temp = TempDir::new().unwrap();
3443 let dir = temp.path();
3444 let result = count_ahead_commits(dir, "main").await;
3446 assert!(
3447 result.is_err(),
3448 "expected Err when directory is not a git repo, got {result:?}"
3449 );
3450 }
3451
3452 #[serial_test::serial(cooldown_state)]
3462 #[tokio::test]
3463 async fn review_invalid_response_applies_persistent_model_cooldown() {
3464 crate::engine::cooldown::reset_global_state().await;
3465 let agent = "opencode-3314-review-invalid";
3466 let model = "opencode/north-mini-code-free";
3467 let key = format!("{agent}:{model}");
3468
3469 assert!(
3470 !crate::engine::cooldown::is_model_in_cooldown(agent, model),
3471 "model should not be in cooldown before record_review_agent_failure"
3472 );
3473
3474 let err = runner::agents::AgentError::InvalidResponse {
3475 raw: "empty review output".to_string(),
3476 };
3477 record_review_agent_failure("test-3314-a", agent, Some(model), &err).await;
3478
3479 let now = chrono::Utc::now().timestamp();
3480 let until = crate::engine::cooldown::cooldown_until(&key)
3481 .expect("review InvalidResponse should set model cooldown");
3482 let remaining = until.saturating_sub(now);
3483
3484 assert!(
3487 remaining >= crate::engine::cooldown::PERSISTENT_MODEL_BACKOFF_BASE_SECS - 30,
3488 "review InvalidResponse must apply ~4h persistent cooldown (issue #3314), \
3489 got {remaining}s — may have used 5-min standard backoff instead"
3490 );
3491 }
3492
3493 #[serial_test::serial(cooldown_state)]
3501 #[tokio::test]
3502 async fn review_persistent_cooldown_survives_after_first_failure() {
3503 crate::engine::cooldown::reset_global_state().await;
3504 let agent = "opencode-3314-review-parse";
3505 let model = "opencode/north-mini-code-free";
3506 let key = format!("{agent}:{model}");
3507
3508 let err = runner::agents::AgentError::InvalidResponse {
3510 raw: "failed to parse review response".to_string(),
3511 };
3512 record_review_agent_failure("test-3314-b", agent, Some(model), &err).await;
3513
3514 let now = chrono::Utc::now().timestamp();
3515 let until1 = crate::engine::cooldown::cooldown_until(&key)
3516 .expect("first failure should set cooldown");
3517 let remaining1 = until1.saturating_sub(now);
3518
3519 crate::engine::cooldown::record_agent_success(agent, model).await;
3522
3523 assert!(
3529 remaining1 >= crate::engine::cooldown::PERSISTENT_MODEL_BACKOFF_BASE_SECS - 30,
3530 "after success reset, first-failure cooldown should still be ~4h, got {remaining1}s"
3531 );
3532 }
3533
3534 #[serial_test::serial(cooldown_state)]
3540 #[tokio::test]
3541 async fn review_inline_scan_billing_cycle_message_gets_model_scoped_cooldown() {
3542 crate::engine::cooldown::reset_global_state().await;
3543 let agent = "kimi-3551-review-inline";
3544 let model = "opus";
3545 let key = format!("{agent}:{model}");
3546
3547 let text_for_review = "kimi rate limit: Failed to authenticate. API Error: 403 \
3549 You've reached your usage limit for this billing cycle. Your quota will be \
3550 refreshed next cycle.";
3551
3552 let scan_tail = runner::agents::patterns::safe_tail(
3553 text_for_review,
3554 runner::agents::patterns::RATE_LIMIT_SCAN_TAIL_BYTES,
3555 );
3556 let err = runner::agents::patterns::detect_rate_limit(scan_tail)
3557 .expect("billing-cycle message must still be caught by the rate-limit scan");
3558
3559 assert_eq!(
3560 outcome_for_agent_error(&err),
3561 "billing_cycle_exhausted",
3562 "inline-scanned billing-cycle message must not be stored as plain rate_limit"
3563 );
3564
3565 record_review_agent_failure("test-3551", agent, Some(model), &err).await;
3566
3567 let now = chrono::Utc::now().timestamp();
3568 let until = crate::engine::cooldown::cooldown_until(&key)
3569 .expect("billing-cycle exhaustion must set a model-scoped cooldown");
3570 let remaining = until.saturating_sub(now);
3571 assert!(
3576 remaining >= crate::engine::cooldown::PERSISTENT_MODEL_BACKOFF_BASE_SECS - 30,
3577 "billing-cycle exhaustion for a known model must apply the ~4h persistent \
3578 model cooldown, got {remaining}s — may have used the generic short agent \
3579 cooldown instead"
3580 );
3581 }
3582
3583 fn test_review_task(id: &str) -> ExternalTask {
3586 ExternalTask {
3587 id: crate::backends::ExternalId(id.to_string()),
3588 title: "Test review task".to_string(),
3589 body: "".to_string(),
3590 state: "open".to_string(),
3591 labels: vec![],
3592 author: "tester".to_string(),
3593 created_at: "2026-01-01T00:00:00Z".to_string(),
3594 updated_at: "2026-01-01T00:00:00Z".to_string(),
3595 url: "https://example.com/issues/1".to_string(),
3596 }
3597 }
3598
3599 fn test_review_context(agent: &str, model: &str, work_dir: &std::path::Path) -> ReviewContext {
3600 ReviewContext {
3601 store_id: None,
3602 had_prev_push_failure: false,
3603 review_attempt: 1,
3604 worktree_path: work_dir.to_path_buf(),
3605 branch_name: "gh-issue-1-test".to_string(),
3606 pr_number: 1,
3607 head_sha: None,
3608 default_branch: "main".to_string(),
3609 review_agent: agent.to_string(),
3610 review_model: Some(model.to_string()),
3611 review_task_id: "review-1".to_string(),
3612 review_attempt_dir: work_dir.to_path_buf(),
3613 output_file: work_dir.join("output.txt"),
3614 invocation: runner::agent::AgentInvocation {
3615 agent: agent.to_string(),
3616 model: Some(model.to_string()),
3617 work_dir: work_dir.to_path_buf(),
3618 system_prompt: String::new(),
3619 agent_message: String::new(),
3620 task_id: "review-1".to_string(),
3621 disallowed_tools: vec![],
3622 git_author_name: "Test".to_string(),
3623 git_author_email: "test@example.com".to_string(),
3624 output_file: work_dir.join("output.txt"),
3625 timeout_seconds: 0,
3626 repo: "owner/repo".to_string(),
3627 attempt: 1,
3628 },
3629 same_agent_fallback: false,
3630 }
3631 }
3632
3633 fn test_parsed_review(decision: ReviewDecision) -> ParsedReview {
3634 ParsedReview {
3635 run_id: None,
3636 exit_code: 0,
3637 stderr: String::new(),
3638 raw_output: String::new(),
3639 text_for_review: String::new(),
3640 token_usage: RunTokenUsage::default(),
3641 review_notes_for_comment: String::new(),
3642 decision,
3643 }
3644 }
3645
3646 fn write_review_output(dir: &std::path::Path, stdout: &str, exit_code: i32) {
3647 std::fs::write(dir.join("output.txt"), stdout).unwrap();
3648 std::fs::write(dir.join("exit.txt"), exit_code.to_string()).unwrap();
3649 std::fs::write(dir.join("stderr.txt"), "").unwrap();
3650 }
3651
3652 #[serial_test::serial(cooldown_state)]
3656 #[tokio::test]
3657 async fn parse_review_output_mid_thought_is_incomplete_without_persistent_cooldown() {
3658 crate::engine::cooldown::reset_global_state().await;
3659 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3660 let work_dir = TempDir::new().unwrap();
3661 let agent = "kimi";
3662 let model = "opus-3734-mid-thought";
3663 let ctx = test_review_context(agent, model, work_dir.path());
3664 let redraw = "^D\u{0008}".repeat(20);
3665 let ndjson = format!(
3666 "{}\n{}\n{}\n{}\n{}",
3667 r#"{"type":"system","subtype":"thinking_tokens","tokens":1945}"#,
3668 r#"{"type":"assistant","message":{"content":[{"type":"text","text":"CI passes. Now verifying callers and the lease logic edge cases:"}],"stop_reason":null,"usage":{"input_tokens":10,"output_tokens":0}}}"#,
3669 r#"{"type":"assistant","message":{"content":[{"type":"thinking","thinking":"checking lease edge cases"}],"stop_reason":null,"usage":{"input_tokens":10,"output_tokens":0}}}"#,
3670 "\x1b[?25h",
3671 redraw,
3672 );
3673 write_review_output(work_dir.path(), &ndjson, 0);
3674
3675 let task = test_review_task("3734");
3676 let phase = parse_review_output(
3677 &task,
3678 "owner/repo",
3679 &ctx,
3680 ReviewRun { run_id: None },
3681 &store,
3682 )
3683 .await;
3684
3685 match phase {
3686 ReviewPhase::EarlyReturn(ReviewDecision::Failed(reason)) => {
3687 assert!(
3688 reason.contains("incomplete:"),
3689 "mid-thought review must be stored as incomplete, got {reason}"
3690 );
3691 }
3692 ReviewPhase::EarlyReturn(_) => panic!("expected Failed(incomplete)"),
3693 ReviewPhase::Ready(_) => panic!("expected Failed(incomplete), got Ready"),
3694 }
3695 assert!(
3696 !crate::engine::cooldown::is_model_in_cooldown(agent, model),
3697 "mid-thought incomplete must not apply persistent model cooldown"
3698 );
3699 }
3700
3701 #[serial_test::serial(cooldown_state)]
3704 #[tokio::test]
3705 async fn parse_review_output_tool_use_without_envelope_is_incomplete() {
3706 crate::engine::cooldown::reset_global_state().await;
3707 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3708 let work_dir = TempDir::new().unwrap();
3709 let agent = "kimi";
3710 let model = "opus-3734-tool-use";
3711 let ctx = test_review_context(agent, model, work_dir.path());
3712 let ndjson = r#"{"type":"assistant","message":{"content":[{"type":"tool_use","name":"Read","input":{"file_path":"src/lib.rs"}}]}}"#;
3713 write_review_output(work_dir.path(), ndjson, 0);
3714
3715 let task = test_review_task("3734-tool");
3716 let phase = parse_review_output(
3717 &task,
3718 "owner/repo",
3719 &ctx,
3720 ReviewRun { run_id: None },
3721 &store,
3722 )
3723 .await;
3724
3725 match phase {
3726 ReviewPhase::EarlyReturn(ReviewDecision::Failed(reason)) => {
3727 assert!(
3728 reason.contains("incomplete:"),
3729 "tool_use without envelope must be incomplete, got {reason}"
3730 );
3731 }
3732 ReviewPhase::EarlyReturn(_) => panic!("expected Failed(incomplete)"),
3733 ReviewPhase::Ready(_) => panic!("expected Failed(incomplete), got Ready"),
3734 }
3735 assert!(!crate::engine::cooldown::is_model_in_cooldown(agent, model));
3736 }
3737
3738 #[serial_test::serial(cooldown_state)]
3741 #[tokio::test]
3742 async fn parse_review_output_unparseable_envelope_applies_persistent_cooldown() {
3743 crate::engine::cooldown::reset_global_state().await;
3744 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3745 let work_dir = TempDir::new().unwrap();
3746 let agent = "kimi";
3747 let model = "opus-3734-format";
3748 let ctx = test_review_context(agent, model, work_dir.path());
3749 let ndjson = concat!(
3750 r#"{"type":"assistant","message":{"content":[{"type":"text","text":"I have finished looking at the code."}]}}"#,
3751 "\n",
3752 r#"{"type":"result","subtype":"success","is_error":false,"result":"I have finished looking at the code."}"#,
3753 );
3754 write_review_output(work_dir.path(), ndjson, 0);
3755
3756 let task = test_review_task("3734-format");
3757 let phase = parse_review_output(
3758 &task,
3759 "owner/repo",
3760 &ctx,
3761 ReviewRun { run_id: None },
3762 &store,
3763 )
3764 .await;
3765
3766 match phase {
3767 ReviewPhase::EarlyReturn(ReviewDecision::Failed(reason)) => {
3768 assert!(
3769 reason.contains("parse error"),
3770 "finished unparseable envelope must stay a parse error, got {reason}"
3771 );
3772 }
3773 ReviewPhase::EarlyReturn(_) => panic!("expected Failed(parse error)"),
3774 ReviewPhase::Ready(_) => panic!("expected Failed(parse error), got Ready"),
3775 }
3776 assert!(
3777 crate::engine::cooldown::is_model_in_cooldown(agent, model),
3778 "finished format failure must apply persistent model cooldown"
3779 );
3780 }
3781
3782 #[serial_test::serial(cooldown_state)]
3785 #[tokio::test]
3786 async fn parse_review_output_missing_envelope_uses_assistant_decision() {
3787 crate::engine::cooldown::reset_global_state().await;
3788 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3789 let work_dir = TempDir::new().unwrap();
3790 let agent = "kimi";
3791 let model = "opus-3734-assistant";
3792 let ctx = test_review_context(agent, model, work_dir.path());
3793 let ndjson = r#"{"type":"assistant","message":{"content":[{"type":"text","text":"{\"decision\":\"approve\",\"notes\":\"LGTM\",\"issues\":[]}"}]}}"#;
3794 write_review_output(work_dir.path(), ndjson, 0);
3795
3796 let task = test_review_task("3734-asst");
3797 let phase = parse_review_output(
3798 &task,
3799 "owner/repo",
3800 &ctx,
3801 ReviewRun { run_id: None },
3802 &store,
3803 )
3804 .await;
3805
3806 match phase {
3807 ReviewPhase::Ready(parsed) => {
3808 assert!(matches!(parsed.decision, ReviewDecision::Approve));
3809 }
3810 ReviewPhase::EarlyReturn(_) => {
3811 panic!("expected Ready(Approve) from assistant text, got EarlyReturn")
3812 }
3813 }
3814 assert!(!crate::engine::cooldown::is_model_in_cooldown(agent, model));
3815 }
3816
3817 #[serial_test::serial(cooldown_state)]
3820 #[tokio::test]
3821 async fn parse_review_output_does_not_treat_task_done_synthesis_as_approval() {
3822 crate::engine::cooldown::reset_global_state().await;
3823 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3824 let work_dir = TempDir::new().unwrap();
3825 let agent = "kimi";
3826 let model = "opus-3734-done";
3827 let ctx = test_review_context(agent, model, work_dir.path());
3828 let ndjson = r#"{"type":"assistant","message":{"content":[{"type":"text","text":"analyzing the diff"}]}}"#;
3829 write_review_output(work_dir.path(), ndjson, 0);
3830
3831 let task = test_review_task("3734-done");
3832 let phase = parse_review_output(
3833 &task,
3834 "owner/repo",
3835 &ctx,
3836 ReviewRun { run_id: None },
3837 &store,
3838 )
3839 .await;
3840
3841 match phase {
3842 ReviewPhase::EarlyReturn(ReviewDecision::Failed(reason)) => {
3843 assert!(
3844 reason.contains("parse error"),
3845 "unfinished prose without a decision must not approve, got {reason}"
3846 );
3847 }
3848 ReviewPhase::Ready(_) => {
3849 panic!("task-path status=done synthesis must not become a review approval")
3850 }
3851 ReviewPhase::EarlyReturn(_) => panic!("expected Failed(parse error)"),
3852 }
3853 }
3854
3855 #[serial_test::serial(cooldown_state)]
3861 #[tokio::test]
3862 async fn finalize_review_run_success_resets_generic_model_failure_count() {
3863 crate::engine::cooldown::reset_global_state().await;
3864 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3865 crate::engine::cooldown::init_cooldown_store(store.clone()).await;
3866
3867 let agent = "test-3478-agent";
3868 let model = "test-3478-model";
3869 let failure_key = format!("failure_count:{agent}:{model}");
3870
3871 let err = runner::agents::AgentError::AgentFailed {
3875 message: "agent reported a generic failure".to_string(),
3876 };
3877 record_review_agent_failure("task-3478-a", agent, Some(model), &err).await;
3878 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
3879 assert_eq!(
3880 store.kv_get(&failure_key).await.unwrap().as_deref(),
3881 Some("1"),
3882 "sanity check: failure should have incremented the persisted count"
3883 );
3884
3885 let work_dir = TempDir::new().unwrap();
3886 let ctx = test_review_context(agent, model, work_dir.path());
3887 let parsed = test_parsed_review(ReviewDecision::Approve);
3888 let task = test_review_task("1");
3889
3890 finalize_review_run(&task, &ctx, &parsed, &store).await;
3891 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
3892
3893 assert_eq!(
3894 store.kv_get(&failure_key).await.unwrap().as_deref(),
3895 Some("0"),
3896 "successful review must reset failure_count:{{agent}}:{{model}} (issue #3478)"
3897 );
3898 }
3899
3900 #[serial_test::serial(cooldown_state)]
3906 #[tokio::test]
3907 async fn finalize_review_run_interleaved_success_keeps_generic_cooldown_at_base() {
3908 crate::engine::cooldown::reset_global_state().await;
3909 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3910 crate::engine::cooldown::init_cooldown_store(store.clone()).await;
3911
3912 let agent = "test-3478-interleave-agent";
3913 let model = "test-3478-interleave-model";
3914 let cooldown_key = format!("{agent}:{model}");
3915
3916 let work_dir = TempDir::new().unwrap();
3917 let ctx = test_review_context(agent, model, work_dir.path());
3918 let task = test_review_task("2");
3919
3920 for i in 0..4 {
3921 let err = runner::agents::AgentError::AgentFailed {
3922 message: format!("failure {i}"),
3923 };
3924 record_review_agent_failure("task-3478-b", agent, Some(model), &err).await;
3925
3926 let now = chrono::Utc::now().timestamp();
3927 let until = crate::engine::cooldown::cooldown_until(&cooldown_key)
3928 .expect("failure should set cooldown");
3929 let remaining = until.saturating_sub(now);
3930
3931 assert!(
3932 remaining <= crate::engine::cooldown::BACKOFF_BASE_SECS + 60,
3933 "failure #{} must stay at base backoff since the prior success reset the \
3934 counter (issue #3478); got {remaining}s, base is {}s — an escalating value \
3935 here means review successes are not resetting failure_count",
3936 i + 1,
3937 crate::engine::cooldown::BACKOFF_BASE_SECS
3938 );
3939
3940 let parsed = test_parsed_review(ReviewDecision::Approve);
3944 finalize_review_run(&task, &ctx, &parsed, &store).await;
3945 tokio::time::sleep(std::time::Duration::from_millis(20)).await;
3946 }
3947 }
3948
3949 #[serial_test::serial(cooldown_state)]
3955 #[tokio::test]
3956 async fn finalize_review_run_success_does_not_reset_persistent_model_failure_count() {
3957 crate::engine::cooldown::reset_global_state().await;
3958 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3959 crate::engine::cooldown::init_cooldown_store(store.clone()).await;
3960
3961 let agent = "test-3571-agent";
3962 let model = "test-3571-model";
3963 let cooldown_key = format!("{agent}:{model}");
3964
3965 let work_dir = TempDir::new().unwrap();
3966 let ctx = test_review_context(agent, model, work_dir.path());
3967 let task = test_review_task("3");
3968
3969 let err = runner::agents::AgentError::InvalidResponse {
3971 raw: "empty review output".to_string(),
3972 };
3973 record_review_agent_failure("task-3571-a", agent, Some(model), &err).await;
3974 tokio::time::sleep(std::time::Duration::from_millis(20)).await;
3975
3976 let parsed = test_parsed_review(ReviewDecision::Approve);
3979 finalize_review_run(&task, &ctx, &parsed, &store).await;
3980 tokio::time::sleep(std::time::Duration::from_millis(20)).await;
3981
3982 let err2 = runner::agents::AgentError::InvalidResponse {
3985 raw: "empty review output again".to_string(),
3986 };
3987 record_review_agent_failure("task-3571-b", agent, Some(model), &err2).await;
3988 tokio::time::sleep(std::time::Duration::from_millis(20)).await;
3989
3990 let now = chrono::Utc::now().timestamp();
3991 let until = crate::engine::cooldown::cooldown_until(&cooldown_key)
3992 .expect("second failure should set cooldown");
3993 let remaining = until.saturating_sub(now);
3994
3995 assert!(
3996 remaining > crate::engine::cooldown::PERSISTENT_MODEL_BACKOFF_BASE_SECS,
3997 "second persistent model failure should escalate past the 4h base despite an \
3998 intervening success (issue #3571); got {remaining}s, base is {}s",
3999 crate::engine::cooldown::PERSISTENT_MODEL_BACKOFF_BASE_SECS
4000 );
4001 }
4002}