1pub(crate) const MAX_REVIEW_AGENT_FAILURES: u64 = 3;
13
14const MAX_PR_CREATE_FAILURES: u64 = 3;
16
17fn outcome_for_agent_error(err: &runner::agents::AgentError) -> &'static str {
22 match err {
23 runner::agents::AgentError::Timeout { .. } => "timeout",
24 runner::agents::AgentError::RateLimit { message } => {
29 crate::engine::cooldown::detect_credit_exhaustion_outcome(message)
30 .unwrap_or("rate_limit")
31 }
32 runner::agents::AgentError::Auth { message } => {
33 crate::engine::cooldown::detect_credit_exhaustion_outcome(message)
34 .unwrap_or("auth_error")
35 }
36 runner::agents::AgentError::InvalidResponse { .. } => "parse_error",
37 _ => "failed",
38 }
39}
40
41fn format_review_error(agent: &str, err: &runner::agents::AgentError) -> String {
50 match err {
51 runner::agents::AgentError::RateLimit { message } => format!(
52 "{agent} rate limit: {}",
53 runner::fallback::summarize_rate_limit_error(message)
54 ),
55 runner::agents::AgentError::Auth { message } => format!("{agent} auth error: {message}"),
56 runner::agents::AgentError::Timeout { elapsed } => {
57 format!("{agent} timed out after {}s", elapsed.as_secs())
58 }
59 other => format!("{agent} agent error: {other}"),
60 }
61}
62
63pub(crate) fn parse_pr_number_from_url(url: &str) -> Result<u64, String> {
75 let url = url.trim();
76 if url.is_empty() {
77 return Err("URL is empty".to_string());
78 }
79 if !url.starts_with("https://github.com/") {
80 return Err(format!(
81 "URL does not start with 'https://github.com/': {url}"
82 ));
83 }
84 if !url.contains("/pull/") {
85 return Err(format!("URL missing '/pull/' segment: {url}"));
86 }
87 let last_segment = url
88 .trim_end_matches('/')
89 .rsplit('/')
90 .next()
91 .ok_or_else(|| format!("No segment after '/pull/' in URL: {url}"))?;
92 let number_part = last_segment
93 .split(['?', '#'])
94 .next()
95 .filter(|s| !s.is_empty())
96 .ok_or_else(|| format!("PR number segment is empty in URL: {url}"))?;
97 let pr_num: u64 = number_part.parse().map_err(|_| {
98 format!("PR number is not a valid positive integer: '{number_part}' in URL: {url}")
99 })?;
100 if pr_num == 0 {
101 return Err(format!("PR number cannot be zero in URL: {url}"));
102 }
103 Ok(pr_num)
104}
105
106fn review_started_comment(review_agent: &str, review_model: &str) -> String {
107 let footer = attribution_footer("Review started", review_agent, Some(review_model));
108 format!("🔍 Automated review started{}", footer)
109}
110
111fn should_skip_no_code_reroute_increment(last_error: &str) -> bool {
112 crate::engine::runner::git_ops::is_transient_github_api_error(last_error)
113}
114
115use crate::backends::{ExternalBackend, ExternalTask, Status};
116use crate::cmd::CommandErrorContext;
117use crate::config;
118use crate::engine::attribution_footer;
119use crate::engine::auto_merge::{auto_merge_pr, handle_review_changes};
120use crate::engine::runner;
121use crate::engine::runner::worktree;
122use crate::engine::tasks::TaskManager;
123use crate::github::http::GhHttp;
124use crate::store::store_log_activity;
125use crate::store::TaskStore;
126use crate::store::{opt_store_get_task, set_review_session_expected, store_increment};
127use crate::store::{store_set, store_set_result};
128use crate::store::{CompleteRun, RunTokenUsage, StartRun};
129use crate::tmux::TmuxManager;
130use anyhow::Context;
131use std::collections::HashSet;
132use std::sync::Arc;
133use tokio::process::Command;
134use tokio::sync::RwLock;
135
136use super::router::Router;
137
138#[derive(Debug, Clone)]
140pub(crate) enum ReviewDecision {
141 Approve,
143 RequestChanges {
145 notes: String,
146 issues: Vec<crate::engine::runner::response::ReviewIssue>,
147 },
148 Failed(String),
150 Blocked(String),
152 Skipped,
154 Rerouted,
156}
157
158enum EnsurePrResult {
160 Ready(u64),
162 EarlyReturn(ReviewDecision),
164}
165
166enum ReviewPhase<T> {
167 Ready(T),
168 EarlyReturn(ReviewDecision),
169}
170
171struct ReviewContext {
172 store_id: Option<i64>,
173 had_prev_push_failure: bool,
174 review_attempt: u32,
175 worktree_path: std::path::PathBuf,
176 branch_name: String,
177 pr_number: u64,
178 default_branch: String,
179 review_agent: String,
180 review_model: Option<String>,
181 review_task_id: String,
182 review_attempt_dir: std::path::PathBuf,
183 output_file: std::path::PathBuf,
184 invocation: runner::agent::AgentInvocation,
185 same_agent_fallback: bool,
186}
187
188impl ReviewContext {
189 fn review_model_str(&self) -> &str {
190 self.review_model.as_deref().unwrap_or("")
191 }
192}
193
194struct ReviewRun {
195 run_id: Option<i64>,
196}
197
198struct ParsedReview {
199 run_id: Option<i64>,
200 exit_code: i32,
201 stderr: String,
202 raw_output: String,
203 text_for_review: String,
204 token_usage: RunTokenUsage,
205 review_notes_for_comment: String,
206 decision: ReviewDecision,
207}
208
209fn build_review_disallowed_tools(worktree_path: &std::path::Path) -> Vec<String> {
210 let mut tools = vec![
211 "Bash(rm *)".to_string(),
212 "Bash(rm -*)".to_string(),
213 "Bash(git push*)".to_string(),
214 ];
215 let orch_yml = worktree_path.join(".orch.yml");
216 let orch_yml_str = orch_yml.to_string_lossy();
217 tools.extend([
218 format!("Write({orch_yml_str})"),
219 format!("Edit({orch_yml_str})"),
220 ]);
221 if let Ok(orch_home) = crate::home::orch_home() {
222 for path in [
223 orch_home.join("config.yml"),
224 orch_home.join("config.example.yml"),
225 ] {
226 let path_str = path.to_string_lossy();
227 tools.extend([
228 format!("Read({path_str})"),
229 format!("Write({path_str})"),
230 format!("Edit({path_str})"),
231 ]);
232 }
233 }
234 tools
235}
236
237async fn select_review_agent(
238 task_id: &str,
239 task_agent: &str,
240 store_id: Option<i64>,
241 router: &Arc<RwLock<Router>>,
242 store: &Arc<TaskStore>,
243) -> anyhow::Result<(String, Option<String>, bool)> {
244 let mut exclude_set: HashSet<String> = HashSet::new();
245 if !task_agent.is_empty() {
246 exclude_set.insert(task_agent.to_string());
247 }
248 if let Some(store_id) = store_id {
249 if let Ok(failed) = store.previous_review_agents(store_id).await {
250 for agent in failed {
251 exclude_set.insert(agent);
252 }
253 }
254 }
255
256 let mut r = router.write().await;
257 let mut tried_agents: HashSet<String> = HashSet::new();
258 let mut chosen_agent: Option<String> = None;
259 let mut chosen_model: Option<String> = None;
260 let available_count = r.available_agents.len().max(1);
261
262 loop {
263 let tmp_exclude_refs: Vec<&str> = exclude_set.iter().map(|s| s.as_str()).collect();
264 let agent = match r.next_round_robin_agent(&tmp_exclude_refs, "review") {
265 Some(a) => a,
266 None => break,
267 };
268
269 if tried_agents.contains(&agent) {
270 break;
271 }
272 tried_agents.insert(agent.clone());
273
274 let model = r.config.model_for_complexity(&agent, "review", task_id);
275 if model.is_some() {
276 chosen_agent = Some(agent);
277 chosen_model = model;
278 break;
279 }
280
281 exclude_set.insert(agent);
282 if exclude_set.len() >= available_count {
283 break;
284 }
285 }
286
287 if let Some(agent) = chosen_agent {
288 let same = !task_agent.is_empty() && agent == task_agent;
289 Ok((agent, chosen_model, same))
290 } else {
291 let final_exclude_refs: Vec<&str> = exclude_set.iter().map(|s| s.as_str()).collect();
292 let fallback_agent = r
293 .next_round_robin_agent(&final_exclude_refs, "review")
294 .unwrap_or_else(|| "claude".to_string());
295 let fallback_model = r
296 .config
297 .model_for_complexity(&fallback_agent, "review", task_id);
298 if fallback_model.is_none() {
299 anyhow::bail!(
300 "no review agent with a configured review model is available for task {}",
301 task_id
302 );
303 }
304 let same = !task_agent.is_empty() && fallback_agent == task_agent;
305 Ok((fallback_agent, fallback_model, same))
306 }
307}
308
309#[allow(clippy::too_many_arguments)]
310async fn build_review_context(
311 task: &ExternalTask,
312 repo: &str,
313 router: &Arc<RwLock<Router>>,
314 task_manager: &Arc<TaskManager>,
315 store: &Arc<TaskStore>,
316) -> anyhow::Result<ReviewPhase<ReviewContext>> {
317 let stored_task = store
318 .get_by_external_id(repo, &task.id.0)
319 .await
320 .with_context(|| format!("store lookup failed for task {}", task.id.0))?;
321
322 let store_id: Option<i64> = stored_task.as_ref().map(|t| t.id);
323 let had_prev_push_failure = stored_task
324 .as_ref()
325 .is_some_and(|t| t.last_error.contains("push failed"));
326 let review_attempt: u32 = match store_increment(
327 &Some(Arc::clone(store)),
328 repo,
329 &task.id.0,
330 "review_invocations",
331 )
332 .await
333 {
334 Ok(v) => v as u32,
335 Err(e) => {
336 tracing::warn!(task_id = task.id.0, err = %e, "failed to increment review_invocations — using fallback attempt=1");
337 1
338 }
339 };
340
341 let no_code_review = stored_task.as_ref().is_some_and(|t| {
342 t.worktree.trim().is_empty() && t.branch.trim().is_empty() && t.pr_number.is_none()
343 });
344 if no_code_review {
345 if crate::engine::tasks::is_internal_id(&task.id.0) {
346 tracing::info!(
348 task_id = task.id.0,
349 "no worktree/branch/pr for internal needs_review task — skipping review and marking done"
350 );
351 if let Err(e) = task_manager
352 .update_task_status(&task.id, Status::Done)
353 .await
354 {
355 tracing::error!(
356 task_id = task.id.0,
357 err = %e,
358 "update_task_status(Done) failed for no-code review skip"
359 );
360 }
361 return Ok(ReviewPhase::EarlyReturn(ReviewDecision::Skipped));
362 }
363
364 let max_reroutes: u32 = config::get("workflow.max_reroute_attempts")
367 .ok()
368 .and_then(|s| s.parse().ok())
369 .or_else(|| {
370 config::get("workflow.max_attempts")
371 .ok()
372 .and_then(|s| s.parse().ok())
373 })
374 .unwrap_or(3);
375
376 let reroutes = match store_increment(
377 &Some(Arc::clone(store)),
378 repo,
379 &task.id.0,
380 "no_code_reroutes",
381 )
382 .await
383 {
384 Ok(v) => v,
385 Err(e) => {
386 tracing::warn!(task_id = task.id.0, err = %e, "failed to increment no_code_reroutes — skipping reroute/block decision this tick");
387 return Ok(ReviewPhase::EarlyReturn(ReviewDecision::Failed(format!(
388 "transient store error: {e}"
389 ))));
390 }
391 };
392
393 if reroutes as u32 >= max_reroutes {
394 tracing::error!(
395 task_id = task.id.0,
396 reroutes,
397 max_reroutes,
398 "reached max reroute attempts for no-worktree/branch/pr result — blocking for human review"
399 );
400 let msg = format!(
401 "no worktree/branch/pr after {}/{} reroute attempts",
402 reroutes, max_reroutes
403 );
404 let fields = [
405 (
406 "block_reason",
407 serde_json::json!(format!(
408 "max reroute attempts ({}) reached — no worktree/branch/pr produced",
409 max_reroutes
410 )),
411 ),
412 ("last_error", serde_json::json!(msg)),
413 ("agent", serde_json::json!(null)),
414 ("model", serde_json::json!(null)),
415 ];
416 if let Err(e) = task_manager
417 .update_task_status_and_result(&task.id, Status::Blocked, &fields)
418 .await
419 {
420 tracing::error!(task_id = task.id.0, err = %e, "update_task_status_and_result(Blocked) failed — skipping block to avoid silent auto-unblock loop");
421 }
422 return Ok(ReviewPhase::EarlyReturn(ReviewDecision::Blocked(msg)));
423 }
424
425 tracing::warn!(
426 task_id = task.id.0,
427 reroutes,
428 max_reroutes,
429 "no worktree/branch/pr for external needs_review task — re-routing to a different agent"
430 );
431 if let Err(e) = store_set_result(
432 &Some(Arc::clone(store)),
433 repo,
434 &task.id.0,
435 &[
436 ("agent", serde_json::json!(null)),
437 ("model", serde_json::json!(null)),
438 (
439 "last_error",
440 serde_json::json!("no worktree/branch/pr produced"),
441 ),
442 ],
443 )
444 .await
445 {
446 tracing::error!(task_id = task.id.0, err = %e, "failed to clear agent/model for reroute — skipping status transition");
447 } else if let Err(e) = task_manager.update_task_status(&task.id, Status::New).await {
448 tracing::error!(task_id = task.id.0, err = %e, "update_task_status(New) failed — task may be stuck in InReview");
449 }
450 return Ok(ReviewPhase::EarlyReturn(ReviewDecision::Rerouted));
451 }
452
453 let mut worktree_path = match stored_task.as_ref().map(|t| t.worktree.as_str()) {
454 Some(w) if !w.is_empty() => std::path::PathBuf::from(w),
455 _ => {
456 tracing::warn!(task_id = task.id.0, "no worktree found for review");
457 return Ok(ReviewPhase::EarlyReturn(ReviewDecision::Failed(
458 "no worktree found".to_string(),
459 )));
460 }
461 };
462
463 let mut branch_name = match stored_task.as_ref().map(|t| t.branch.as_str()) {
464 Some(b) if !b.is_empty() => b.to_string(),
465 _ => {
466 tracing::warn!(task_id = task.id.0, "no branch found for review");
467 return Ok(ReviewPhase::EarlyReturn(ReviewDecision::Failed(
468 "no branch found".to_string(),
469 )));
470 }
471 };
472
473 let mut missing_worktree_recovered = false;
474 if !worktree_path.exists() {
475 match recover_missing_review_worktree(task, repo, store).await {
476 Ok(recovered) => {
477 branch_name = recovered.branch;
478 worktree_path = recovered.work_dir;
479 missing_worktree_recovered = true;
480 }
481 Err(e) => {
482 let reason = format!("missing review worktree recovery failed: {e}");
483 tracing::error!(task_id = task.id.0, error = %reason, "cannot recover review worktree");
484 store_set(
485 &Some(Arc::clone(store)),
486 repo,
487 &task.id.0,
488 &[("last_error", serde_json::json!(reason.clone()))],
489 )
490 .await;
491 return Ok(ReviewPhase::EarlyReturn(ReviewDecision::Blocked(reason)));
492 }
493 }
494 }
495
496 if missing_worktree_recovered {
497 store_set(
498 &Some(Arc::clone(store)),
499 repo,
500 &task.id.0,
501 &[("last_error", serde_json::json!(""))],
502 )
503 .await;
504 }
505
506 let agent_summary = stored_task
507 .as_ref()
508 .map(|t| t.summary.clone())
509 .unwrap_or_default();
510 let stored_pr_number = stored_task
511 .as_ref()
512 .and_then(|t| t.pr_number)
513 .map(|n| n as u64)
514 .filter(|&n| n > 0);
515
516 let pr_number = match ensure_pr_exists(
517 task,
518 &branch_name,
519 &worktree_path,
520 repo,
521 &agent_summary,
522 store,
523 task_manager,
524 stored_pr_number,
525 )
526 .await?
527 {
528 EnsurePrResult::Ready(n) => n,
529 EnsurePrResult::EarlyReturn(decision) => return Ok(ReviewPhase::EarlyReturn(decision)),
530 };
531
532 {
536 let fetch_timeout = std::time::Duration::from_secs(60);
537 let fetch_future = async {
538 let mut cmd = Command::new("git");
539 cmd.args(["fetch", "origin", "--prune"])
540 .current_dir(&worktree_path);
541 cmd.kill_on_drop(true);
543 cmd.output_with_context().await
544 };
545
546 match tokio::time::timeout(fetch_timeout, fetch_future).await {
547 Ok(Ok(o)) if o.status.success() => {
548 }
550 Ok(Ok(o)) => {
551 tracing::warn!(
552 task_id = task.id.0,
553 exit_code = ?o.status.code(),
554 stdout = %String::from_utf8_lossy(&o.stdout),
555 stderr = %String::from_utf8_lossy(&o.stderr),
556 "review: git fetch failed (non-zero exit) — diff/log may use stale remote refs"
557 );
558 }
559 Ok(Err(e)) => {
560 tracing::warn!(
561 task_id = task.id.0,
562 error = %e,
563 "review: git fetch failed — diff/log may use stale remote refs"
564 );
565 }
566 Err(_) => {
567 tracing::warn!(
568 task_id = task.id.0,
569 "review: git fetch timed out after 60s — diff/log may use stale remote refs"
570 );
571 }
572 }
573 }
574
575 let default_branch = runner::worktree::detect_default_branch(&worktree_path).await;
576 let git_diff = runner::context::build_git_diff(&worktree_path, &default_branch).await;
577 let git_log = runner::context::build_git_log(&worktree_path, &default_branch).await;
578 let review_prompt = runner::agent::build_review_prompt(
579 task,
580 &agent_summary,
581 &git_diff,
582 &git_log,
583 &default_branch,
584 pr_number,
585 );
586
587 let task_agent = stored_task
588 .as_ref()
589 .and_then(|t| t.agent.clone())
590 .unwrap_or_default();
591 let (review_agent, review_model, same_agent_fallback) =
592 select_review_agent(&task.id.0, &task_agent, store_id, router, store).await?;
593
594 anyhow::ensure!(
595 review_model.is_some(),
596 "review model missing for task {} after agent selection — refusing to dispatch",
597 task.id.0
598 );
599
600 if same_agent_fallback {
601 tracing::warn!(
602 task_id = task.id.0,
603 agent = %review_agent,
604 "no other agent available for review — falling back to the agent that wrote the code"
605 );
606 store_log_activity(
607 &Some(Arc::clone(store)),
608 repo,
609 &task.id.0,
610 "review_agent_fallback",
611 None,
612 None,
613 Some(&review_agent),
614 review_model.as_deref(),
615 Some(&serde_json::json!({
616 "reason": "all available agents excluded (author + previous reviewers)",
617 })),
618 )
619 .await;
620 }
621
622 tracing::info!(
623 task_id = task.id.0,
624 agent = %review_agent,
625 model = ?review_model,
626 "spawning review agent"
627 );
628
629 let review_task_id = format!("{}-review", task.id.0);
630 let review_attempt_dir =
631 crate::home::task_attempt_dir_async(repo, &review_task_id, review_attempt).await?;
632 let output_file = review_attempt_dir.join("output.json");
633
634 let git_name =
635 crate::config::get("git.name").unwrap_or_else(|_| format!("{review_agent}[bot]"));
636 let git_email = crate::config::get("git.email")
637 .unwrap_or_else(|_| format!("{review_agent}[bot]@users.noreply.github.com"));
638 let review_timeout_secs: u64 = crate::config::get("workflow.review_timeout_seconds")
639 .ok()
640 .and_then(|s| s.parse().ok())
641 .unwrap_or_else(|| {
642 crate::config::get("workflow.timeout_seconds")
643 .ok()
644 .and_then(|s| s.parse().ok())
645 .unwrap_or(1800)
646 })
647 .max(1800);
648
649 let invocation = runner::agent::AgentInvocation {
650 agent: review_agent.clone(),
651 model: review_model.clone(),
652 work_dir: worktree_path.clone(),
653 system_prompt: runner::agent::review_system_prompt(),
654 agent_message: review_prompt,
655 task_id: review_task_id.clone(),
656 disallowed_tools: build_review_disallowed_tools(&worktree_path),
657 git_author_name: git_name,
658 git_author_email: git_email,
659 output_file: output_file.clone(),
660 timeout_seconds: review_timeout_secs,
661 repo: repo.to_string(),
662 attempt: review_attempt,
663 };
664
665 Ok(ReviewPhase::Ready(ReviewContext {
666 store_id,
667 had_prev_push_failure,
668 review_attempt,
669 worktree_path,
670 branch_name,
671 pr_number,
672 default_branch,
673 review_agent,
674 review_model,
675 review_task_id,
676 review_attempt_dir,
677 output_file,
678 invocation,
679 same_agent_fallback,
680 }))
681}
682
683#[allow(clippy::too_many_arguments)]
684async fn complete_review_run(
685 store: &Arc<TaskStore>,
686 run_id: Option<i64>,
687 exit_code: Option<i32>,
688 stdout: &str,
689 stderr: &str,
690 parsed: &str,
691 outcome: &str,
692 error: &str,
693 tokens: RunTokenUsage,
694) {
695 if let Some(run_id) = run_id {
696 if let Err(e) = store
697 .complete_run(&CompleteRun {
698 run_id,
699 exit_code,
700 stdout,
701 stderr,
702 parsed,
703 outcome,
704 error,
705 tokens,
706 })
707 .await
708 {
709 tracing::warn!(
710 run_id,
711 error = %e,
712 "failed to record review run completion in audit trail"
713 );
714 }
715 }
716}
717
718async fn invoke_review_agent(
719 task: &ExternalTask,
720 tmux: &Arc<TmuxManager>,
721 ctx: &ReviewContext,
722 repo: &str,
723 store: &Arc<TaskStore>,
724) -> ReviewPhase<ReviewRun> {
725 let run_id = if let Some(sid) = ctx.store_id {
726 match store
727 .start_run(&StartRun {
728 task_id: sid,
729 attempt: ctx.review_attempt as i32,
730 run_type: "review",
731 agent: &ctx.review_agent,
732 model: ctx.review_model.as_deref().unwrap_or(""),
733 command: "",
734 prompt: "",
735 })
736 .await
737 {
738 Ok(run_id) => Some(run_id),
739 Err(e) => {
740 tracing::warn!(
741 task_id = task.id.0,
742 error = %e,
743 "failed to record review run start in audit trail"
744 );
745 None
746 }
747 }
748 } else {
749 None
750 };
751
752 let session = match runner::agent::spawn_in_tmux(tmux, &ctx.invocation).await {
753 Ok(session) => session,
754 Err(e) => {
755 tracing::error!(task_id = task.id.0, error = %e, "failed to spawn review agent");
756 complete_review_run(
757 store,
758 run_id,
759 None,
760 "",
761 &e.to_string(),
762 "",
763 "failed",
764 &format!("spawn failed: {e}"),
765 RunTokenUsage::default(),
766 )
767 .await;
768 return ReviewPhase::EarlyReturn(ReviewDecision::Failed(format!("spawn failed: {e}")));
769 }
770 };
771
772 let start_comment = review_started_comment(&ctx.review_agent, ctx.review_model_str());
773 match GhHttp::new() {
774 Ok(gh) => {
775 if let Err(e) = gh
776 .add_comment(repo, &ctx.pr_number.to_string(), &start_comment)
777 .await
778 {
779 tracing::debug!(
780 task_id = task.id.0,
781 pr_number = ctx.pr_number,
782 error = %e,
783 "failed to post review-started comment"
784 );
785 }
786 }
787 Err(e) => {
788 tracing::debug!(
789 task_id = task.id.0,
790 pr_number = ctx.pr_number,
791 error = %e,
792 "failed to create GitHub client for review-started comment"
793 );
794 }
795 }
796
797 let poll_interval = std::time::Duration::from_secs(5);
798 let timeout_duration = std::time::Duration::from_secs(ctx.invocation.timeout_seconds);
799 let wait_result = tokio::time::timeout(
800 timeout_duration,
801 tmux.wait_for_completion(&session, poll_interval),
802 )
803 .await;
804
805 match wait_result {
806 Ok(Ok(_)) => {
807 tracing::info!(task_id = task.id.0, "review agent completed");
808 let _ = tmux.kill_session(&session).await;
809 ReviewPhase::Ready(ReviewRun { run_id })
810 }
811 Ok(Err(e)) => {
812 tracing::warn!(task_id = task.id.0, error = %e, "review agent process exited with error");
813 let _ = tmux.kill_session(&session).await;
814
815 let exit_code: i32 = tokio::task::spawn_blocking({
820 let review_attempt_dir = ctx.review_attempt_dir.clone();
821 move || {
822 std::fs::read_to_string(review_attempt_dir.join("exit.txt"))
823 .ok()
824 .and_then(|s| s.trim().parse().ok())
825 .unwrap_or(-1)
826 }
827 })
828 .await
829 .unwrap_or(-1);
830
831 if exit_code != 0 {
832 let raw_output =
837 runner::response::read_output_file(&ctx.review_task_id, &ctx.output_file, repo)
838 .await;
839 let agent_result =
840 runner::agents::find_agent_result(&ctx.review_agent, &raw_output);
841 if agent_result.as_ref().map(|r| !r.is_error).unwrap_or(false) {
843 tracing::info!(
844 task_id = task.id.0,
845 agent = %ctx.review_agent,
846 exit_code,
847 output_found = !raw_output.is_empty(),
848 "review agent succeeded despite non-zero exit — proceeding to parse output"
849 );
850 return ReviewPhase::Ready(ReviewRun { run_id });
851 }
852 }
853
854 tracing::error!(task_id = task.id.0, error = %e, "review agent failed");
855 complete_review_run(
856 store,
857 run_id,
858 Some(exit_code),
859 "",
860 &e.to_string(),
861 "",
862 "failed",
863 &format!("agent error: {e}"),
864 RunTokenUsage::default(),
865 )
866 .await;
867 ReviewPhase::EarlyReturn(ReviewDecision::Failed(format!("agent error: {e}")))
868 }
869 Err(_) => {
870 tracing::error!(task_id = task.id.0, "review agent timed out");
871 let _ = tmux.kill_session(&session).await;
872 if let Some(model) = ctx.review_model.as_deref() {
873 crate::engine::cooldown::record_persistent_model_failure(&ctx.review_agent, model)
878 .await;
879 }
880 complete_review_run(
881 store,
882 run_id,
883 Some(-1),
884 "",
885 "timeout",
886 "",
887 "timeout",
888 "timeout",
889 RunTokenUsage::default(),
890 )
891 .await;
892 ReviewPhase::EarlyReturn(ReviewDecision::Failed("timeout".to_string()))
893 }
894 }
895}
896
897async fn record_review_agent_failure(
898 task_id: &str,
899 review_agent: &str,
900 review_model: Option<&str>,
901 err: &runner::agents::AgentError,
902) {
903 match err {
904 runner::agents::AgentError::RateLimit { message }
905 | runner::agents::AgentError::Auth { message } => {
906 tracing::warn!(
907 task_id,
908 agent = %review_agent,
909 "review agent hit error — adding to cooldown"
910 );
911 if let Some(reason) = crate::engine::cooldown::detect_credit_exhaustion(message) {
912 if reason == crate::engine::cooldown::CreditExhaustionReason::BillingCycleExhausted
913 {
914 if let Some(model) = review_model {
915 crate::engine::cooldown::record_persistent_model_failure(
917 review_agent,
918 model,
919 )
920 .await;
921 } else {
922 crate::engine::cooldown::record_credit_exhaustion(review_agent, reason)
923 .await;
924 }
925 } else {
926 crate::engine::cooldown::record_credit_exhaustion(review_agent, reason).await;
927 }
928 } else {
929 runner::response::record_agent_failure_with_message(review_agent, &err.to_string())
930 .await;
931 }
932 }
933 runner::agents::AgentError::InvalidResponse { .. } => {
934 if let Some(model) = review_model {
939 crate::engine::cooldown::record_persistent_model_failure(review_agent, model).await;
940 }
941 }
942 runner::agents::AgentError::ModelUnavailable { .. } => {
943 if let Some(model) = review_model {
948 crate::engine::cooldown::record_persistent_model_failure(review_agent, model).await;
949 }
950 }
951 _ => {
952 if let Some(model) = review_model {
955 crate::engine::cooldown::record_model_failure(review_agent, model).await;
956 }
957 }
958 }
959}
960
961fn review_decision_from_response(
962 review_response: runner::response::ReviewResponse,
963) -> ReviewDecision {
964 match review_response.decision.as_str() {
965 "approve" => ReviewDecision::Approve,
966 "request_changes" => ReviewDecision::RequestChanges {
967 notes: review_response.notes,
968 issues: review_response.issues,
969 },
970 _ => ReviewDecision::Failed(format!("unknown decision: {}", review_response.decision)),
971 }
972}
973
974fn review_decision_name(decision: &ReviewDecision) -> &'static str {
975 match decision {
976 ReviewDecision::Approve => "approve",
977 ReviewDecision::RequestChanges { .. } => "request_changes",
978 ReviewDecision::Failed(_) => "failed",
979 ReviewDecision::Blocked(_) => "blocked",
980 ReviewDecision::Skipped => "skipped",
981 ReviewDecision::Rerouted => "rerouted",
982 }
983}
984
985async fn parse_review_output(
986 task: &ExternalTask,
987 repo: &str,
988 ctx: &ReviewContext,
989 run: ReviewRun,
990 store: &Arc<TaskStore>,
991) -> ReviewPhase<ParsedReview> {
992 let (file_exists, file_size, exit_code, stderr) = {
993 let output_file_clone = ctx.output_file.clone();
994 let review_attempt_dir_clone = ctx.review_attempt_dir.clone();
995 match tokio::task::spawn_blocking(move || {
996 let file_exists = output_file_clone.exists();
997 let file_size = std::fs::metadata(&output_file_clone)
998 .map(|m| m.len())
999 .unwrap_or(0);
1000 let exit_code = std::fs::read_to_string(review_attempt_dir_clone.join("exit.txt"))
1001 .ok()
1002 .and_then(|s| s.trim().parse::<i32>().ok())
1003 .unwrap_or(-1);
1004 let stderr = std::fs::read_to_string(review_attempt_dir_clone.join("stderr.txt"))
1005 .unwrap_or_default();
1006 (file_exists, file_size, exit_code, stderr)
1007 })
1008 .await
1009 {
1010 Ok(tuple) => tuple,
1011 Err(e) => {
1012 tracing::error!(
1013 task_id = task.id.0,
1014 error = %e,
1015 "spawn_blocking panicked reading review output metadata"
1016 );
1017 (false, 0, -1, format!("spawn_blocking failed: {e}"))
1018 }
1019 }
1020 };
1021
1022 tracing::info!(
1023 task_id = task.id.0,
1024 path = %ctx.output_file.display(),
1025 file_exists,
1026 file_size,
1027 "reading review agent output"
1028 );
1029
1030 let raw_output =
1031 runner::response::read_output_file(&ctx.review_task_id, &ctx.output_file, repo).await;
1032 let agent_result = runner::agents::find_agent_result(&ctx.review_agent, &raw_output);
1033 let token_usage = RunTokenUsage {
1034 input_tokens: agent_result
1035 .as_ref()
1036 .and_then(|r| r.input_tokens)
1037 .unwrap_or(0),
1038 output_tokens: agent_result
1039 .as_ref()
1040 .and_then(|r| r.output_tokens)
1041 .unwrap_or(0),
1042 total_cost_usd: agent_result
1043 .as_ref()
1044 .and_then(|r| r.cost_usd)
1045 .unwrap_or(0.0),
1046 duration_secs: 0.0,
1047 };
1048
1049 let agent_result_is_error = agent_result.as_ref().map(|r| r.is_error).unwrap_or(false);
1050 let ndjson_completed_no_error = raw_output.contains("\"terminal_reason\":\"completed\"")
1051 && !raw_output.contains("\"is_error\":true");
1052 let is_hard_failure =
1054 !ndjson_completed_no_error && (raw_output.is_empty() || agent_result_is_error);
1055
1056 if is_hard_failure {
1057 tracing::warn!(
1058 task_id = task.id.0,
1059 exit_code,
1060 raw_output_len = raw_output.len(),
1061 stderr_len = stderr.len(),
1062 "review agent: entering error path"
1063 );
1064 let agent_runner = runner::agents::get_runner(&ctx.review_agent);
1065 let err = agent_runner.classify_error(exit_code, &raw_output, &stderr);
1066 record_review_agent_failure(
1067 &task.id.0,
1068 &ctx.review_agent,
1069 ctx.review_model.as_deref(),
1070 &err,
1071 )
1072 .await;
1073 tracing::error!(task_id = task.id.0, error = %err, "review agent failed");
1074 let stored_error = format_review_error(&ctx.review_agent, &err);
1075 complete_review_run(
1076 store,
1077 run.run_id,
1078 Some(exit_code),
1079 &raw_output,
1080 &stderr,
1081 "",
1082 outcome_for_agent_error(&err),
1083 &stored_error,
1084 token_usage,
1085 )
1086 .await;
1087 return ReviewPhase::EarlyReturn(ReviewDecision::Failed(stored_error));
1088 }
1089
1090 let text_for_review = agent_result
1091 .as_ref()
1092 .map(|r| r.result_text.clone())
1093 .filter(|text| !text.is_empty())
1094 .unwrap_or_else(|| {
1095 tracing::debug!(
1096 task_id = task.id.0,
1097 agent = %ctx.review_agent,
1098 "review agent: per-agent extractor returned no text, using raw output"
1099 );
1100 raw_output.clone()
1101 });
1102
1103 let review_response = match runner::response::parse_review_response(&text_for_review) {
1104 Ok(response) => response,
1105 Err(e) => {
1106 if let Some(response) = runner::response::infer_review_response(&text_for_review) {
1107 tracing::warn!(
1108 task_id = task.id.0,
1109 agent = %ctx.review_agent,
1110 "review response parsed via keyword fallback"
1111 );
1112 response
1113 } else {
1114 let ndjson_completed = ndjson_completed_no_error;
1120
1121 if ndjson_completed {
1126 let assistant_text = runner::agents::collect_assistant_messages_text(
1127 &ctx.review_agent,
1128 &raw_output,
1129 );
1130 if !assistant_text.is_empty() {
1131 if let Ok(response) =
1132 runner::response::parse_review_response(&assistant_text)
1133 {
1134 tracing::debug!(
1135 task_id = task.id.0,
1136 agent = %ctx.review_agent,
1137 "found review response JSON in assistant message (terminal_reason:completed)"
1138 );
1139 let review_notes_for_comment = response.notes.clone();
1140 let decision = review_decision_from_response(response);
1141 return ReviewPhase::Ready(ParsedReview {
1142 run_id: run.run_id,
1143 exit_code,
1144 stderr,
1145 raw_output,
1146 text_for_review: assistant_text,
1147 token_usage,
1148 review_notes_for_comment,
1149 decision,
1150 });
1151 }
1152 if let Some(response) =
1153 runner::response::infer_review_response(&assistant_text)
1154 {
1155 tracing::warn!(
1156 task_id = task.id.0,
1157 agent = %ctx.review_agent,
1158 "inferred review response from assistant message (terminal_reason:completed)"
1159 );
1160 let review_notes_for_comment = response.notes.clone();
1161 let decision = review_decision_from_response(response);
1162 return ReviewPhase::Ready(ParsedReview {
1163 run_id: run.run_id,
1164 exit_code,
1165 stderr,
1166 raw_output,
1167 text_for_review: assistant_text,
1168 token_usage,
1169 review_notes_for_comment,
1170 decision,
1171 });
1172 }
1173 }
1174 }
1175
1176 let scan_head = runner::agents::patterns::safe_head(
1183 &text_for_review,
1184 runner::agents::patterns::RATE_LIMIT_SCAN_HEAD_BYTES,
1185 );
1186 let scan_tail = runner::agents::patterns::safe_tail(
1187 &text_for_review,
1188 runner::agents::patterns::RATE_LIMIT_SCAN_TAIL_BYTES,
1189 );
1190
1191 if let Some(err) = (!ndjson_completed)
1192 .then(|| {
1193 runner::agents::patterns::detect_rate_limit(scan_head)
1194 .or_else(|| runner::agents::patterns::detect_rate_limit(scan_tail))
1195 })
1196 .flatten()
1197 {
1198 let runner::agents::AgentError::RateLimit { message } = &err else {
1199 unreachable!("detect_rate_limit only returns AgentError::RateLimit")
1200 };
1201 tracing::warn!(
1202 task_id = task.id.0,
1203 agent = %ctx.review_agent,
1204 "review agent hit rate limit — adding to cooldown"
1205 );
1206 let summary = runner::fallback::summarize_rate_limit_error(message);
1213 let stored_error = format!("{} rate limit: {summary}", ctx.review_agent);
1214 record_review_agent_failure(
1215 &task.id.0,
1216 &ctx.review_agent,
1217 ctx.review_model.as_deref(),
1218 &err,
1219 )
1220 .await;
1221 complete_review_run(
1222 store,
1223 run.run_id,
1224 Some(exit_code),
1225 &raw_output,
1226 &stderr,
1227 &text_for_review,
1228 outcome_for_agent_error(&err),
1229 &stored_error,
1230 token_usage,
1231 )
1232 .await;
1233 return ReviewPhase::EarlyReturn(ReviewDecision::Failed(stored_error));
1234 }
1235
1236 if let Some(err) = runner::agents::patterns::detect_auth_error(scan_tail) {
1237 let runner::agents::AgentError::Auth { message } = &err else {
1238 unreachable!("detect_auth_error only returns AgentError::Auth")
1239 };
1240 tracing::warn!(
1241 task_id = task.id.0,
1242 agent = %ctx.review_agent,
1243 "review agent hit auth error — adding to cooldown"
1244 );
1245 let stored_error = format!("auth error: {message}");
1246 record_review_agent_failure(
1247 &task.id.0,
1248 &ctx.review_agent,
1249 ctx.review_model.as_deref(),
1250 &err,
1251 )
1252 .await;
1253 complete_review_run(
1254 store,
1255 run.run_id,
1256 Some(exit_code),
1257 &raw_output,
1258 &stderr,
1259 &text_for_review,
1260 outcome_for_agent_error(&err),
1261 &stored_error,
1262 token_usage,
1263 )
1264 .await;
1265 return ReviewPhase::EarlyReturn(ReviewDecision::Failed(stored_error));
1266 }
1267
1268 if agent_result
1275 .as_ref()
1276 .map(|r| r.truncated_by_length)
1277 .unwrap_or(false)
1278 {
1279 tracing::warn!(
1280 task_id = task.id.0,
1281 agent = %ctx.review_agent,
1282 model = ?ctx.review_model,
1283 "review agent output truncated: step_finish reason=length \
1284 (token budget exceeded before completion)"
1285 );
1286 if let Some(model) = ctx.review_model.as_deref() {
1287 crate::engine::cooldown::record_persistent_model_failure(
1288 &ctx.review_agent,
1289 model,
1290 )
1291 .await;
1292 }
1293 let stored_error = "truncated: review output was cut off before completion \
1294 (output/reasoning token budget exceeded)"
1295 .to_string();
1296 complete_review_run(
1297 store,
1298 run.run_id,
1299 Some(exit_code),
1300 &raw_output,
1301 &stderr,
1302 &text_for_review,
1303 "truncated",
1304 &stored_error,
1305 token_usage,
1306 )
1307 .await;
1308 return ReviewPhase::EarlyReturn(ReviewDecision::Failed(stored_error));
1309 }
1310
1311 tracing::error!(
1312 task_id = task.id.0,
1313 error = %e,
1314 agent = %ctx.review_agent,
1315 output = %text_for_review.chars().take(300).collect::<String>(),
1316 raw_output = %raw_output.chars().take(1000).collect::<String>(),
1317 "failed to parse review response"
1318 );
1319 if let Some(model) = ctx.review_model.as_deref() {
1320 crate::engine::cooldown::record_persistent_model_failure(
1326 &ctx.review_agent,
1327 model,
1328 )
1329 .await;
1330 }
1331 complete_review_run(
1332 store,
1333 run.run_id,
1334 Some(exit_code),
1335 &raw_output,
1336 &stderr,
1337 &text_for_review,
1338 "parse_error",
1339 &format!("parse error: {e}"),
1340 token_usage,
1341 )
1342 .await;
1343 return ReviewPhase::EarlyReturn(ReviewDecision::Failed(format!(
1344 "parse error: {e}"
1345 )));
1346 }
1347 }
1348 };
1349
1350 let review_notes_for_comment = review_response.notes.clone();
1351 let decision = review_decision_from_response(review_response);
1352
1353 tracing::info!(
1354 task_id = task.id.0,
1355 pr_number = ctx.pr_number,
1356 decision = ?decision,
1357 "review agent decision received"
1358 );
1359 store_log_activity(
1360 &Some(Arc::clone(store)),
1361 repo,
1362 &task.id.0,
1363 "review_decision",
1364 None,
1365 None,
1366 Some(&ctx.review_agent),
1367 ctx.review_model.as_deref(),
1368 Some(&serde_json::json!({
1369 "decision": review_decision_name(&decision),
1370 "pr_number": ctx.pr_number,
1371 })),
1372 )
1373 .await;
1374
1375 ReviewPhase::Ready(ParsedReview {
1376 run_id: run.run_id,
1377 exit_code,
1378 stderr,
1379 raw_output,
1380 text_for_review,
1381 token_usage,
1382 review_notes_for_comment,
1383 decision,
1384 })
1385}
1386
1387async fn finalize_review_run(
1388 task: &ExternalTask,
1389 ctx: &ReviewContext,
1390 parsed: &ParsedReview,
1391 store: &Arc<TaskStore>,
1392) {
1393 let outcome = match &parsed.decision {
1394 ReviewDecision::Approve
1395 | ReviewDecision::RequestChanges { .. }
1396 | ReviewDecision::Skipped
1397 | ReviewDecision::Rerouted => "success",
1398 ReviewDecision::Failed(_) | ReviewDecision::Blocked(_) => "failed",
1399 };
1400 let error = match &parsed.decision {
1401 ReviewDecision::Failed(error) | ReviewDecision::Blocked(error) => error.as_str(),
1402 _ => "",
1403 };
1404
1405 if outcome == "success" {
1406 crate::engine::cooldown::record_agent_success(&ctx.review_agent, ctx.review_model_str())
1409 .await;
1410 }
1411
1412 complete_review_run(
1413 store,
1414 parsed.run_id,
1415 Some(parsed.exit_code),
1416 &parsed.raw_output,
1417 &parsed.stderr,
1418 &parsed.text_for_review,
1419 outcome,
1420 error,
1421 parsed.token_usage,
1422 )
1423 .await;
1424
1425 if let Some(store_id) = ctx.store_id {
1426 if parsed.token_usage.input_tokens > 0 || parsed.token_usage.output_tokens > 0 {
1427 let model = if ctx.review_model_str().is_empty() {
1428 "unknown"
1429 } else {
1430 ctx.review_model_str()
1431 };
1432 if let Err(e) = store
1433 .store_tokens(
1434 store_id,
1435 parsed.token_usage.input_tokens,
1436 parsed.token_usage.output_tokens,
1437 model,
1438 )
1439 .await
1440 {
1441 tracing::warn!(
1442 task_id = task.id.0,
1443 ?e,
1444 "failed to store review agent token usage"
1445 );
1446 }
1447 }
1448 }
1449}
1450
1451async fn restore_review_config_if_needed(
1452 task_id: &str,
1453 worktree_path: &std::path::Path,
1454 default_branch: &str,
1455) {
1456 let _ = Command::new("git")
1457 .args(["checkout", "HEAD", "--", ".orch.yml"])
1458 .current_dir(worktree_path)
1459 .status()
1460 .await;
1461
1462 let merge_base = Command::new("git")
1463 .args(["merge-base", "HEAD", default_branch])
1464 .current_dir(worktree_path)
1465 .output()
1466 .await
1467 .ok()
1468 .filter(|output| output.status.success())
1469 .and_then(|output| String::from_utf8(output.stdout).ok())
1470 .map(|s| s.trim().to_string())
1471 .unwrap_or_default();
1472
1473 if merge_base.is_empty() {
1474 return;
1475 }
1476
1477 let orch_yml_changed = Command::new("git")
1478 .args([
1479 "diff",
1480 "--name-only",
1481 &merge_base,
1482 "HEAD",
1483 "--",
1484 ".orch.yml",
1485 ])
1486 .current_dir(worktree_path)
1487 .output()
1488 .await
1489 .ok()
1490 .filter(|output| output.status.success())
1491 .map(|output| !output.stdout.is_empty())
1492 .unwrap_or(false);
1493
1494 if !orch_yml_changed {
1495 return;
1496 }
1497
1498 tracing::warn!(
1499 task_id,
1500 "review agent modified .orch.yml (forbidden); restoring to merge-base version"
1501 );
1502 let restore_ok = Command::new("git")
1503 .args(["checkout", &merge_base, "--", ".orch.yml"])
1504 .current_dir(worktree_path)
1505 .status()
1506 .await
1507 .map(|status| status.success())
1508 .unwrap_or(false);
1509
1510 if !restore_ok {
1511 return;
1512 }
1513
1514 let staged = Command::new("git")
1515 .args(["diff", "--cached", "--name-only", "--", ".orch.yml"])
1516 .current_dir(worktree_path)
1517 .output()
1518 .await
1519 .ok()
1520 .filter(|output| output.status.success())
1521 .map(|output| !output.stdout.is_empty())
1522 .unwrap_or(false);
1523
1524 if staged {
1525 let _ = Command::new("git")
1526 .args([
1527 "-c",
1528 "commit.gpgsign=false",
1529 "commit",
1530 "-m",
1531 "revert: restore .orch.yml (review agent must not modify project config)",
1532 ])
1533 .current_dir(worktree_path)
1534 .status()
1535 .await;
1536 }
1537}
1538
1539fn is_missing_path_error_message(err: &str) -> bool {
1540 err.to_ascii_lowercase()
1541 .contains("no such file or directory")
1542}
1543
1544async fn push_review_branch(
1545 task: &ExternalTask,
1546 repo: &str,
1547 ctx: &ReviewContext,
1548 store: &Arc<TaskStore>,
1549) -> Result<(), String> {
1550 let mut worktree_path = ctx.worktree_path.clone();
1551 let mut branch_name = ctx.branch_name.clone();
1552 let mut retried_missing_worktree = false;
1553
1554 let push_result = loop {
1555 match runner::git_ops::push_branch(&worktree_path, &branch_name, &ctx.default_branch).await
1556 {
1557 Ok(ok) => break Ok(ok),
1558 Err(e)
1559 if !retried_missing_worktree
1560 && !worktree_path.exists()
1561 && is_missing_path_error_message(&e.to_string()) =>
1562 {
1563 retried_missing_worktree = true;
1564 tracing::warn!(
1565 task_id = task.id.0,
1566 worktree = %worktree_path.display(),
1567 branch = %branch_name,
1568 "review push failed because worktree disappeared; attempting one-time recovery"
1569 );
1570 match recover_missing_review_worktree(task, repo, store).await {
1571 Ok(recovered) => {
1572 worktree_path = recovered.work_dir;
1573 branch_name = recovered.branch;
1574 continue;
1575 }
1576 Err(recover_err) => {
1577 break Err(anyhow::anyhow!(
1578 "missing review worktree recovery failed after push error: {recover_err}"
1579 ));
1580 }
1581 }
1582 }
1583 Err(e) => break Err(e),
1584 }
1585 };
1586
1587 match push_result {
1588 Ok(_) => {
1589 store_log_activity(
1590 &Some(Arc::clone(store)),
1591 repo,
1592 &task.id.0,
1593 "push",
1594 None,
1595 None,
1596 Some(&ctx.review_agent),
1597 ctx.review_model.as_deref(),
1598 Some(&serde_json::json!({
1599 "status": "ok",
1600 "branch": branch_name.clone(),
1601 "default_branch": ctx.default_branch.clone(),
1602 "phase": "review",
1603 })),
1604 )
1605 .await;
1606 if ctx.had_prev_push_failure {
1607 if let Err(e) = store_set_result(
1608 &Some(Arc::clone(store)),
1609 repo,
1610 &task.id.0,
1611 &[
1612 ("last_error", serde_json::json!("")),
1613 ("push_failures", serde_json::json!(0)),
1614 ],
1615 )
1616 .await
1617 {
1618 tracing::warn!(task_id = task.id.0, err = %e, "failed to reset push_failures — counter may drift");
1619 }
1620 }
1621 Ok(())
1622 }
1623 Err(e) => {
1624 store_log_activity(
1625 &Some(Arc::clone(store)),
1626 repo,
1627 &task.id.0,
1628 "push",
1629 None,
1630 None,
1631 Some(&ctx.review_agent),
1632 ctx.review_model.as_deref(),
1633 Some(&serde_json::json!({
1634 "status": "error",
1635 "branch": branch_name.clone(),
1636 "default_branch": ctx.default_branch.clone(),
1637 "phase": "review",
1638 "error": e.to_string(),
1639 })),
1640 )
1641 .await;
1642 tracing::error!(
1643 task_id = task.id.0,
1644 branch = %ctx.branch_name,
1645 error = %e,
1646 "review push failed"
1647 );
1648 store_set(
1649 &Some(Arc::clone(store)),
1650 repo,
1651 &task.id.0,
1652 &[("last_error", serde_json::json!(format!("push failed: {e}")))],
1653 )
1654 .await;
1655 Err(e.to_string())
1656 }
1657 }
1658}
1659
1660fn build_pr_review_comment(decision: &ReviewDecision, review_notes_for_comment: &str) -> String {
1661 match decision {
1662 ReviewDecision::Approve => {
1663 format!(
1664 "## Automated Review \u{2014} Approve\n\n{}",
1665 review_notes_for_comment
1666 )
1667 }
1668 ReviewDecision::RequestChanges { notes, issues } => {
1669 let mut body = format!(
1670 "## Automated Review \u{2014} Changes Requested\n\n{}\n",
1671 notes
1672 );
1673 if !issues.is_empty() {
1674 body.push_str("\n**Issues Found:**\n");
1675 for issue in issues {
1676 body.push_str(&format!(
1677 "- `{}` line {}: {} [{}]\n",
1678 issue.file,
1679 issue
1680 .line
1681 .map(|line| line.to_string())
1682 .unwrap_or_else(|| "?".to_string()),
1683 issue.description,
1684 issue.severity
1685 ));
1686 }
1687 }
1688 body
1689 }
1690 _ => String::new(),
1691 }
1692}
1693
1694async fn post_review_comment(
1695 task: &ExternalTask,
1696 repo: &str,
1697 ctx: &ReviewContext,
1698 decision: &ReviewDecision,
1699 review_notes_for_comment: &str,
1700) -> anyhow::Result<()> {
1701 let gh = match GhHttp::new() {
1702 Ok(gh) => gh,
1703 Err(e) => {
1704 tracing::warn!(
1705 task_id = task.id.0,
1706 error = %e,
1707 "failed to create GH client for review comment"
1708 );
1709 return Ok(());
1710 }
1711 };
1712 let mut pr_comment = build_pr_review_comment(decision, review_notes_for_comment);
1713 if ctx.same_agent_fallback && !pr_comment.is_empty() {
1714 pr_comment.push_str(
1715 "\n> Note: no other agent was available, so this PR was reviewed by the same agent that wrote the code.\n",
1716 );
1717 }
1718
1719 if !pr_comment.is_empty() {
1720 let footer =
1721 attribution_footer("Reviewed", &ctx.review_agent, Some(ctx.review_model_str()));
1722 let pr_comment_with_footer = format!("{}{}", pr_comment, footer);
1723 if let Err(e) = gh
1724 .add_comment(repo, &ctx.pr_number.to_string(), &pr_comment_with_footer)
1725 .await
1726 {
1727 tracing::warn!(
1728 task_id = task.id.0,
1729 pr_number = ctx.pr_number,
1730 error = %e,
1731 "failed to post automated review comment on PR"
1732 );
1733 }
1734 }
1735
1736 Ok(())
1737}
1738
1739#[allow(clippy::too_many_arguments)]
1740async fn apply_review_decision(
1741 task: &ExternalTask,
1742 backend: &Arc<dyn ExternalBackend>,
1743 repo: &str,
1744 task_manager: &Arc<TaskManager>,
1745 store: &Arc<TaskStore>,
1746 ctx: &ReviewContext,
1747 parsed: &ParsedReview,
1748 router_config: &crate::engine::router::RouterConfig,
1749) -> anyhow::Result<ReviewDecision> {
1750 restore_review_config_if_needed(&task.id.0, &ctx.worktree_path, &ctx.default_branch).await;
1751
1752 if let Err(e) = push_review_branch(task, repo, ctx, store).await {
1753 return Ok(ReviewDecision::Failed(format!("push failed: {e}")));
1754 }
1755
1756 post_review_comment(
1757 task,
1758 repo,
1759 ctx,
1760 &parsed.decision,
1761 &parsed.review_notes_for_comment,
1762 )
1763 .await?;
1764
1765 match &parsed.decision {
1766 ReviewDecision::Approve => {
1767 if let Err(e) = auto_merge_pr(
1772 task,
1773 &ctx.branch_name,
1774 backend,
1775 repo,
1776 &ctx.review_agent,
1777 ctx.review_model_str(),
1778 task_manager,
1779 store,
1780 )
1781 .await
1782 {
1783 let error_msg = e.to_string();
1784 if error_msg.contains("not yet computed") {
1785 tracing::warn!(
1786 task_id = task.id.0,
1787 pr_number = ctx.pr_number,
1788 branch = %ctx.branch_name,
1789 error = %e,
1790 "auto-merge deferred — PR mergeability not yet computed"
1791 );
1792 } else {
1793 tracing::error!(
1794 task_id = task.id.0,
1795 pr_number = ctx.pr_number,
1796 branch = %ctx.branch_name,
1797 error = %e,
1798 "auto-merge failed"
1799 );
1800 }
1801 return Ok(ReviewDecision::Failed(format!("merge failed: {e}")));
1802 }
1803 Ok(ReviewDecision::Approve)
1804 }
1805 ReviewDecision::RequestChanges { notes, issues } => {
1806 handle_review_changes(
1807 task,
1808 notes,
1809 issues,
1810 backend,
1811 repo,
1812 ctx.pr_number,
1813 &ctx.review_agent,
1814 ctx.review_model_str(),
1815 task_manager,
1816 store,
1817 router_config,
1818 )
1819 .await?;
1820 Ok(parsed.decision.clone())
1821 }
1822 _ => Ok(parsed.decision.clone()),
1823 }
1824}
1825
1826async fn recover_missing_review_worktree(
1827 task: &ExternalTask,
1828 repo: &str,
1829 store: &Arc<TaskStore>,
1830) -> anyhow::Result<worktree::WorktreeSetup> {
1831 let project_dir = crate::config::get_projects_with_paths()?
1832 .into_iter()
1833 .find(|(candidate_repo, _)| candidate_repo == repo)
1834 .map(|(_, project_dir)| project_dir)
1835 .ok_or_else(|| anyhow::anyhow!("no project directory configured for repo {repo}"))?;
1836
1837 tracing::warn!(
1838 task_id = task.id.0,
1839 repo,
1840 project_dir = %project_dir.display(),
1841 "review worktree missing, recreating via normal setup"
1842 );
1843
1844 let store_ref = Some(Arc::clone(store));
1845 let wt = worktree::setup_worktree(&task.id.0, &task.title, &project_dir, &store_ref, repo)
1846 .await
1847 .context("recreating missing review worktree")?;
1848
1849 if !wt.work_dir.exists() {
1850 anyhow::bail!(
1851 "recreated review worktree at {} for task {} is still missing",
1852 wt.work_dir.display(),
1853 task.id.0
1854 );
1855 }
1856
1857 Ok(wt)
1858}
1859
1860async fn count_ahead_commits(
1866 worktree_path: &std::path::Path,
1867 default_branch: &str,
1868) -> Result<u64, String> {
1869 let worktree_str = worktree_path.to_str().ok_or_else(|| {
1870 format!(
1871 "worktree path contains non-UTF-8 characters: {:?}",
1872 worktree_path
1873 )
1874 })?;
1875
1876 let output = tokio::process::Command::new("git")
1877 .args([
1878 "-C",
1879 worktree_str,
1880 "rev-list",
1881 "--count",
1882 &format!("origin/{default_branch}..HEAD"),
1883 ])
1884 .output()
1885 .await
1886 .map_err(|e| format!("git rev-list command failed to start: {e}"))?;
1887
1888 if !output.status.success() {
1889 let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string();
1890 return Err(format!(
1891 "git rev-list exited with {:?}: {}",
1892 output.status.code(),
1893 if stderr.is_empty() {
1894 String::from_utf8_lossy(&output.stdout).trim().to_string()
1895 } else {
1896 stderr
1897 }
1898 ));
1899 }
1900
1901 String::from_utf8_lossy(&output.stdout)
1902 .trim()
1903 .parse::<u64>()
1904 .map_err(|e| format!("could not parse git rev-list output as u64: {e}"))
1905}
1906
1907#[allow(clippy::too_many_arguments)]
1914async fn ensure_pr_exists(
1915 task: &ExternalTask,
1916 branch_name: &str,
1917 worktree_path: &std::path::Path,
1918 repo: &str,
1919 agent_summary: &str,
1920 store: &Arc<TaskStore>,
1921 task_manager: &Arc<TaskManager>,
1922 stored_pr_number: Option<u64>,
1923) -> anyhow::Result<EnsurePrResult> {
1924 let gh_check = GhHttp::new()?;
1925
1926 if let Some(n) = stored_pr_number {
1927 tracing::info!(
1928 task_id = task.id.0,
1929 pr_number = n,
1930 branch = %branch_name,
1931 "open PR found in store, proceeding with review"
1932 );
1933 return Ok(EnsurePrResult::Ready(n));
1934 }
1935
1936 match gh_check.get_pr_number(repo, branch_name).await {
1937 Ok(Some(n)) => {
1938 tracing::info!(
1939 task_id = task.id.0,
1940 pr_number = n,
1941 branch = %branch_name,
1942 "open PR found, proceeding with review"
1943 );
1944 Ok(EnsurePrResult::Ready(n))
1945 }
1946 Ok(None) => {
1947 let default_branch = worktree::detect_default_branch(worktree_path).await;
1949 let has_commits = match count_ahead_commits(worktree_path, &default_branch).await {
1950 Ok(count) => count > 0,
1951 Err(e) => {
1952 tracing::error!(
1953 task_id = task.id.0,
1954 branch = %branch_name,
1955 default_branch = %default_branch,
1956 err = %e,
1957 "could not determine if branch has commits ahead of default"
1958 );
1959 return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
1960 format!("git ancestry check failed: {e}"),
1961 )));
1962 }
1963 };
1964
1965 if has_commits {
1966 tracing::warn!(
1968 task_id = task.id.0,
1969 branch = %branch_name,
1970 "no open PR but branch has commits — attempting to create PR"
1971 );
1972 let worktree_str = worktree_path.to_str().ok_or_else(|| {
1973 anyhow::anyhow!(
1974 "worktree path contains non-UTF-8 characters: {:?}",
1975 worktree_path
1976 )
1977 })?;
1978 let auth_env = runner::git_ops::build_git_auth_env();
1980
1981 let push_timeout = std::time::Duration::from_secs(120);
1986 match tokio::time::timeout(
1987 push_timeout,
1988 tokio::process::Command::new("git")
1989 .args(["-C", worktree_str, "push", "-u", "origin", branch_name])
1990 .envs(auth_env.iter().map(|(k, v)| (k.as_str(), v.as_str())))
1991 .output(),
1992 )
1993 .await
1994 {
1995 Err(_elapsed) => {
1996 tracing::warn!(
1997 task_id = task.id.0,
1998 branch = %branch_name,
1999 timeout_secs = push_timeout.as_secs(),
2000 "review gate fallback push timed out"
2001 );
2002 }
2003 Ok(Ok(o)) => {
2004 if !o.status.success() {
2005 tracing::warn!(
2006 task_id = task.id.0,
2007 branch = %branch_name,
2008 exit_code = ?o.status.code(),
2009 stdout = %String::from_utf8_lossy(&o.stdout),
2010 stderr = %String::from_utf8_lossy(&o.stderr),
2011 "review gate fallback push failed"
2012 );
2013 } else {
2014 tracing::info!(task_id = task.id.0, branch = %branch_name, "review gate fallback push succeeded");
2015 }
2016 }
2017 Ok(Err(e)) => {
2018 tracing::error!(
2019 task_id = task.id.0,
2020 branch = %branch_name,
2021 error = %e,
2022 "review gate fallback push command failed to run"
2023 );
2024 }
2025 }
2026
2027 let task_ref = runner::git_ops::format_task_ref(&task.id.0);
2028 let pr_body = format!(
2029 "Resolves {task_ref}\n\nAuto-created by orch review gate (agent forgot to open PR)"
2030 );
2031 let gh = GhHttp::new()?;
2032 match gh
2033 .create_pr(repo, &task.title, &pr_body, branch_name, &default_branch)
2034 .await
2035 {
2036 Ok(url) => match parse_pr_number_from_url(&url) {
2037 Ok(pr_num) => {
2038 store_set_result(
2039 &Some(Arc::clone(store)),
2040 repo,
2041 &task.id.0,
2042 &[("pr_number", serde_json::json!(pr_num as i64))],
2043 )
2044 .await?;
2045 tracing::info!(
2046 task_id = task.id.0,
2047 branch = %branch_name,
2048 pr_url = %url,
2049 "created missing PR via GhHttp — continuing review"
2050 );
2051 Ok(EnsurePrResult::Ready(pr_num))
2052 }
2053 Err(e) => {
2054 tracing::warn!(
2055 task_id = task.id.0,
2056 branch = %branch_name,
2057 pr_url = %url,
2058 error = %e,
2059 "created missing PR via GhHttp, but failed to parse PR number"
2060 );
2061 Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2062 format!("created missing PR, but could not parse PR number: {e}"),
2063 )))
2064 }
2065 },
2066 Err(e) => {
2067 let e_str = format!("{e}");
2068 if e_str.contains("already exists") {
2069 let backoff_delays = [2u64, 4u64];
2073 let mut found_pr: Option<u64> = None;
2074 for delay_secs in backoff_delays {
2075 tokio::time::sleep(tokio::time::Duration::from_secs(delay_secs))
2076 .await;
2077 if let Ok(Some(n)) = gh_check.get_pr_number(repo, branch_name).await
2078 {
2079 found_pr = Some(n);
2080 break;
2081 }
2082 }
2083 if let Some(n) = found_pr {
2084 tracing::info!(
2085 task_id = task.id.0,
2086 pr_number = n,
2087 branch = %branch_name,
2088 "found existing PR after create_pr 422 — retrying review"
2089 );
2090 store_set_result(
2091 &Some(Arc::clone(store)),
2092 repo,
2093 &task.id.0,
2094 &[("pr_number", serde_json::json!(n as i64))],
2095 )
2096 .await?;
2097 return Ok(EnsurePrResult::Ready(n));
2098 }
2099 }
2100 tracing::warn!(
2101 task_id = task.id.0,
2102 branch = %branch_name,
2103 error = %e,
2104 "create_pr failed via GhHttp, falling back to CLI"
2105 );
2106 let pr_timeout = std::time::Duration::from_secs(120);
2109 let mut gh_cmd = tokio::process::Command::new("gh");
2110 gh_cmd.args([
2111 "pr",
2112 "create",
2113 "--repo",
2114 repo,
2115 "--head",
2116 branch_name,
2117 "--title",
2118 &task.title,
2119 "--body",
2120 &pr_body,
2121 ]);
2122 gh_cmd.current_dir(worktree_path);
2123 gh_cmd.kill_on_drop(true);
2124
2125 let pr_result = tokio::time::timeout(pr_timeout, gh_cmd.output()).await;
2126 match pr_result {
2127 Ok(Ok(o)) if o.status.success() => {
2128 let stdout = String::from_utf8_lossy(&o.stdout);
2129 match parse_pr_number_from_url(stdout.trim()) {
2130 Ok(pr_num) => {
2131 store_set_result(
2132 &Some(Arc::clone(store)),
2133 repo,
2134 &task.id.0,
2135 &[("pr_number", serde_json::json!(pr_num as i64))],
2136 )
2137 .await?;
2138 tracing::info!(
2139 task_id = task.id.0,
2140 branch = %branch_name,
2141 "created missing PR via CLI — continuing review"
2142 );
2143 Ok(EnsurePrResult::Ready(pr_num))
2144 }
2145 Err(e) => {
2146 tracing::warn!(
2147 task_id = task.id.0,
2148 branch = %branch_name,
2149 stdout = %stdout,
2150 error = %e,
2151 "created missing PR via CLI, but failed to parse PR number"
2152 );
2153 Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2154 format!("created missing PR via CLI, but could not parse PR number: {e}")
2155 )))
2156 }
2157 }
2158 }
2159 Ok(Ok(o)) => {
2160 let stderr = String::from_utf8_lossy(&o.stderr);
2161 if stderr.contains("already exists") {
2162 if let Some(pr_url) =
2163 stderr.lines().find(|l| l.trim().starts_with("https://"))
2164 {
2165 match parse_pr_number_from_url(pr_url.trim()) {
2166 Ok(pr_num) => {
2167 if let Err(e) = store_set_result(
2168 &Some(Arc::clone(store)),
2169 repo,
2170 &task.id.0,
2171 &[(
2172 "pr_number",
2173 serde_json::json!(pr_num as i64),
2174 )],
2175 )
2176 .await
2177 {
2178 tracing::warn!(
2179 task_id = task.id.0,
2180 pr_num,
2181 err = %e,
2182 "failed to persist pr_number from already-exists recovery — downstream may retry PR creation"
2183 );
2184 }
2185 tracing::info!(
2186 task_id = task.id.0,
2187 branch = %branch_name,
2188 pr_url = %pr_url,
2189 "PR already exists (from CLI stderr) — retrying review"
2190 );
2191 return Ok(EnsurePrResult::Ready(pr_num));
2192 }
2193 Err(e) => {
2194 tracing::warn!(
2195 task_id = task.id.0,
2196 branch = %branch_name,
2197 pr_url = %pr_url,
2198 error = %e,
2199 "found existing PR URL in CLI stderr, but failed to parse PR number"
2200 );
2201 }
2203 }
2204 }
2205 }
2206 tracing::error!(
2207 task_id = task.id.0,
2208 branch = %branch_name,
2209 stderr = %stderr,
2210 "failed to create missing PR — work may be stuck"
2211 );
2212
2213 if crate::engine::runner::git_ops::is_transient_github_api_error(
2220 &stderr,
2221 ) {
2222 tracing::warn!(
2223 task_id = task.id.0,
2224 branch = %branch_name,
2225 "transient GitHub error creating PR; will retry later without incrementing persistent failure counter"
2226 );
2227 return Ok(EnsurePrResult::EarlyReturn(
2228 ReviewDecision::Failed(format!(
2229 "transient github error creating PR: {stderr}"
2230 )),
2231 ));
2232 }
2233
2234 match store_increment(
2235 &Some(Arc::clone(store)),
2236 repo,
2237 &task.id.0,
2238 "pr_create_failures",
2239 )
2240 .await
2241 {
2242 Ok(failures) if failures >= MAX_PR_CREATE_FAILURES => {
2243 return Ok(EnsurePrResult::EarlyReturn(
2244 ReviewDecision::Blocked(format!(
2245 "no PR, create failed {failures} times: {stderr}"
2246 )),
2247 ));
2248 }
2249 Ok(_) => {}
2250 Err(e) => {
2251 tracing::warn!(task_id = task.id.0, err = %e, "failed to increment pr_create_failures — skipping blocking decision this tick");
2252 }
2253 }
2254 Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2255 format!("no PR, create failed: {stderr}"),
2256 )))
2257 }
2258 Ok(Err(e)) => {
2259 tracing::error!(
2260 task_id = task.id.0,
2261 error = %e,
2262 "failed to run gh pr create"
2263 );
2264 let e_str = format!("{e}");
2265 if crate::engine::runner::git_ops::is_transient_github_api_error(
2266 &e_str,
2267 ) {
2268 tracing::warn!(
2269 task_id = task.id.0,
2270 branch = %branch_name,
2271 "transient GitHub error from gh CLI fallback; will retry later without incrementing persistent failure counter"
2272 );
2273 return Ok(EnsurePrResult::EarlyReturn(
2274 ReviewDecision::Failed(format!("transient gh error: {e}")),
2275 ));
2276 }
2277
2278 match store_increment(
2279 &Some(Arc::clone(store)),
2280 repo,
2281 &task.id.0,
2282 "pr_create_failures",
2283 )
2284 .await
2285 {
2286 Ok(failures) if failures >= MAX_PR_CREATE_FAILURES => {
2287 return Ok(EnsurePrResult::EarlyReturn(
2288 ReviewDecision::Blocked(format!(
2289 "no PR, gh error {failures} times: {e}"
2290 )),
2291 ));
2292 }
2293 Ok(_) => {}
2294 Err(e) => {
2295 tracing::warn!(task_id = task.id.0, err = %e, "failed to increment pr_create_failures — skipping blocking decision this tick");
2296 }
2297 }
2298 Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2299 format!("no PR, gh error: {e}"),
2300 )))
2301 }
2302 Err(_elapsed) => {
2303 tracing::error!(
2305 task_id = task.id.0,
2306 timeout_secs = pr_timeout.as_secs(),
2307 "gh pr create CLI fallback timed out"
2308 );
2309 let e_str = format!(
2310 "gh pr create timed out after {}s",
2311 pr_timeout.as_secs()
2312 );
2313 if crate::engine::runner::git_ops::is_transient_github_api_error(
2314 &e_str,
2315 ) {
2316 tracing::warn!(
2317 task_id = task.id.0,
2318 branch = %branch_name,
2319 "transient GitHub timeout from gh CLI fallback; will retry later without incrementing persistent failure counter"
2320 );
2321 return Ok(EnsurePrResult::EarlyReturn(
2322 ReviewDecision::Failed(format!(
2323 "transient gh error: {e_str}"
2324 )),
2325 ));
2326 }
2327
2328 match store_increment(
2329 &Some(Arc::clone(store)),
2330 repo,
2331 &task.id.0,
2332 "pr_create_failures",
2333 )
2334 .await
2335 {
2336 Ok(failures) if failures >= MAX_PR_CREATE_FAILURES => {
2337 return Ok(EnsurePrResult::EarlyReturn(
2338 ReviewDecision::Blocked(format!(
2339 "no PR, gh timeout {failures} times: {e_str}"
2340 )),
2341 ));
2342 }
2343 Ok(_) => {}
2344 Err(e) => {
2345 tracing::warn!(task_id = task.id.0, err = %e, "failed to increment pr_create_failures — skipping blocking decision this tick");
2346 }
2347 }
2348 Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2349 format!("no PR, gh timeout: {e_str}"),
2350 )))
2351 }
2352 }
2353 }
2354 }
2355 } else {
2356 let merged = match gh_check.is_pr_merged(repo, branch_name).await {
2358 Ok(v) => v,
2359 Err(e) => {
2360 tracing::warn!(task_id = task.id.0, branch = %branch_name, err = %e, "merge check failed, skipping task this tick");
2361 return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2362 format!("merge check failed: {e}"),
2363 )));
2364 }
2365 };
2366 if merged {
2367 tracing::info!(
2368 task_id = task.id.0,
2369 branch = %branch_name,
2370 "PR already merged, marking done"
2371 );
2372 if let Err(e) = task_manager
2373 .update_task_status(&task.id, Status::Done)
2374 .await
2375 {
2376 tracing::error!(task_id = task.id.0, err = %e, "update_task_status(Done) failed — task may be stuck in InReview");
2377 }
2378 return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Skipped));
2379 }
2380
2381 let last_error = opt_store_get_task(&Some(Arc::clone(store)), repo, &task.id.0)
2382 .await
2383 .map(|t| t.last_error)
2384 .unwrap_or_default();
2385 let reason = if !agent_summary.is_empty() {
2386 agent_summary.to_string()
2387 } else {
2388 last_error.clone()
2389 };
2390
2391 if last_error.contains("exceeded max attempts") {
2392 tracing::warn!(
2393 task_id = task.id.0,
2394 branch = %branch_name,
2395 "no PR and no commits after max attempts — marking blocked to stop loop"
2396 );
2397 let fields = [
2401 (
2402 "block_reason",
2403 serde_json::json!("max attempts exceeded — no PR or commits produced"),
2404 ),
2405 ("last_error", serde_json::json!(last_error)),
2406 ];
2407 if let Err(e) = task_manager
2408 .update_task_status_and_result(&task.id, Status::Blocked, &fields)
2409 .await
2410 {
2411 tracing::error!(task_id = task.id.0, err = %e, "update_task_status_and_result(Blocked) failed — skipping block to avoid silent auto-unblock loop");
2412 return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2413 format!("failed to write block_reason: {e}"),
2414 )));
2415 }
2416 return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Skipped));
2417 }
2418
2419 if last_error.contains("422") && last_error.contains("head") {
2420 tracing::info!(
2421 task_id = task.id.0,
2422 branch = %branch_name,
2423 "no PR — 422/head-invalid means work already merged, marking done"
2424 );
2425 if let Err(e) = task_manager
2426 .update_task_status(&task.id, Status::Done)
2427 .await
2428 {
2429 tracing::error!(task_id = task.id.0, err = %e, "update_task_status(Done) failed");
2430 }
2431 return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Skipped));
2432 }
2433
2434 if last_error.contains("no fallback agents")
2435 || last_error.contains("all agents exhausted")
2436 {
2437 tracing::warn!(
2438 task_id = task.id.0,
2439 branch = %branch_name,
2440 last_error = %last_error,
2441 "no PR and no commits after failover exhaustion — marking blocked to stop loop"
2442 );
2443 let fields = [
2447 (
2448 "block_reason",
2449 serde_json::json!("all agents exhausted — no PR or commits produced"),
2450 ),
2451 ("last_error", serde_json::json!(last_error)),
2452 ];
2453 if let Err(e) = task_manager
2454 .update_task_status_and_result(&task.id, Status::Blocked, &fields)
2455 .await
2456 {
2457 tracing::error!(task_id = task.id.0, err = %e, "update_task_status_and_result(Blocked) failed — skipping block to avoid silent auto-unblock loop");
2458 return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2459 format!("failed to write block_reason: {e}"),
2460 )));
2461 }
2462 return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Skipped));
2463 }
2464
2465 let max_reroutes: u32 = config::get("workflow.max_reroute_attempts")
2472 .ok()
2473 .and_then(|s| s.parse().ok())
2474 .or_else(|| {
2475 config::get("workflow.max_attempts")
2476 .ok()
2477 .and_then(|s| s.parse().ok())
2478 })
2479 .unwrap_or(3);
2480
2481 tracing::warn!(
2482 task_id = task.id.0,
2483 branch = %branch_name,
2484 reason = %reason,
2485 "no PR and no commits — re-routing for retry"
2486 );
2487
2488 if should_skip_no_code_reroute_increment(&last_error) {
2492 tracing::warn!(
2493 task_id = task.id.0,
2494 last_error = %last_error,
2495 "transient GitHub error recorded; skipping persistent no_pr_reroutes increment"
2496 );
2497 return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2498 format!("transient github error: {last_error}"),
2499 )));
2500 }
2501
2502 let reroutes = match store_increment(
2504 &Some(Arc::clone(store)),
2505 repo,
2506 &task.id.0,
2507 "no_code_reroutes",
2508 )
2509 .await
2510 {
2511 Ok(v) => v,
2512 Err(e) => {
2513 tracing::warn!(task_id = task.id.0, err = %e, "failed to increment no_code_reroutes — skipping reroute/block decision this tick");
2514 return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2516 format!("transient store error: {e}"),
2517 )));
2518 }
2519 };
2520
2521 if reroutes as u32 >= max_reroutes {
2522 tracing::error!(
2523 task_id = task.id.0,
2524 reroutes,
2525 max_reroutes,
2526 "reached max reroute attempts for no-pr-result — blocking for human review"
2527 );
2528 let msg = format!(
2529 "no PR or code changes after {}/{} reroute attempts",
2530 reroutes, max_reroutes
2531 );
2532 let fields = [
2536 (
2537 "block_reason",
2538 serde_json::json!(format!(
2539 "max reroute attempts ({}) reached — no PR or code changes produced",
2540 max_reroutes
2541 )),
2542 ),
2543 ("last_error", serde_json::json!(msg)),
2544 ("agent", serde_json::json!(null)),
2545 ("model", serde_json::json!(null)),
2546 ];
2547 if let Err(e) = task_manager
2548 .update_task_status_and_result(&task.id, Status::Blocked, &fields)
2549 .await
2550 {
2551 tracing::error!(task_id = task.id.0, err = %e, "update_task_status_and_result(Blocked) failed — skipping block to avoid silent auto-unblock loop");
2552 }
2553 Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Blocked(msg)))
2554 } else if task.id.0.starts_with("internal:") {
2555 tracing::info!(
2562 task_id = task.id.0,
2563 "no PR and no commits for internal task — marking done"
2564 );
2565 if let Err(e) = task_manager
2566 .update_task_status(&task.id, Status::Done)
2567 .await
2568 {
2569 tracing::error!(task_id = task.id.0, err = %e, "update_task_status(Done) failed — task may be stuck in InReview");
2570 }
2571 Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Skipped))
2572 } else {
2573 if let Err(e) = store_set_result(
2578 &Some(Arc::clone(store)),
2579 repo,
2580 &task.id.0,
2581 &[
2582 ("agent", serde_json::json!(null)),
2583 ("model", serde_json::json!(null)),
2584 (
2585 "last_error",
2586 serde_json::json!("no PR or code changes produced"),
2587 ),
2588 ],
2589 )
2590 .await
2591 {
2592 tracing::error!(task_id = task.id.0, err = %e, "failed to clear agent/model for reroute — skipping status transition");
2593 } else if let Err(e) =
2595 task_manager.update_task_status(&task.id, Status::New).await
2596 {
2597 tracing::error!(task_id = task.id.0, err = %e, "update_task_status(New) failed — task may be stuck in InReview");
2598 }
2599 Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Rerouted))
2600 }
2601 }
2602 }
2603 Err(e) => {
2604 tracing::warn!(
2605 task_id = task.id.0,
2606 branch = %branch_name,
2607 error = %e,
2608 "failed to check PR status"
2609 );
2610 Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2611 format!("PR check failed: {e}"),
2612 )))
2613 }
2614 }
2615}
2616
2617pub(crate) async fn review_and_merge(
2623 task: &ExternalTask,
2624 backend: &Arc<dyn ExternalBackend>,
2625 tmux: &Arc<TmuxManager>,
2626 repo: &str,
2627 router: &Arc<RwLock<Router>>,
2628 task_manager: &Arc<TaskManager>,
2629 store: &Arc<TaskStore>,
2630) -> anyhow::Result<ReviewDecision> {
2631 set_review_session_expected(store, repo, &task.id.0, true).await;
2632
2633 let ctx = match build_review_context(task, repo, router, task_manager, store).await? {
2634 ReviewPhase::Ready(ctx) => ctx,
2635 ReviewPhase::EarlyReturn(decision) => return Ok(decision),
2636 };
2637
2638 let run = match invoke_review_agent(task, tmux, &ctx, repo, store).await {
2639 ReviewPhase::Ready(run) => run,
2640 ReviewPhase::EarlyReturn(decision) => return Ok(decision),
2641 };
2642
2643 let parsed = match parse_review_output(task, repo, &ctx, run, store).await {
2644 ReviewPhase::Ready(parsed) => parsed,
2645 ReviewPhase::EarlyReturn(decision) => return Ok(decision),
2646 };
2647
2648 finalize_review_run(task, &ctx, &parsed, store).await;
2649
2650 let router_config = router.read().await.config.clone();
2651 apply_review_decision(
2652 task,
2653 backend,
2654 repo,
2655 task_manager,
2656 store,
2657 &ctx,
2658 &parsed,
2659 &router_config,
2660 )
2661 .await
2662}
2663
2664#[cfg(test)]
2665mod tests {
2666 use super::*;
2667
2668 use crate::backends::ExternalTask;
2669 use crate::engine::router::RouterConfig;
2670 use crate::github::types::{GitHubReview, GitHubReviewComment, GitHubUser, PullRequestReview};
2671 use crate::store::{TaskStatus, TaskStore};
2672 use tempfile::TempDir;
2673
2674 #[test]
2677 fn outcome_for_agent_error_kimi_billing_cycle_is_not_rate_limit() {
2678 let err = runner::agents::AgentError::RateLimit {
2682 message: "Failed to authenticate. API Error: 403 You've reached your usage limit \
2683 for this billing cycle. Your quota will be refreshed soon."
2684 .to_string(),
2685 };
2686 assert_eq!(outcome_for_agent_error(&err), "billing_cycle_exhausted");
2687 }
2688
2689 #[test]
2690 fn outcome_for_agent_error_generic_rate_limit_unaffected() {
2691 let err = runner::agents::AgentError::RateLimit {
2692 message: "429 Too Many Requests".to_string(),
2693 };
2694 assert_eq!(outcome_for_agent_error(&err), "rate_limit");
2695 }
2696
2697 #[test]
2698 fn outcome_for_agent_error_generic_auth_unaffected() {
2699 let err = runner::agents::AgentError::Auth {
2700 message: "401 Unauthorized".to_string(),
2701 };
2702 assert_eq!(outcome_for_agent_error(&err), "auth_error");
2703 }
2704
2705 #[test]
2708 fn parse_pr_number_standard_url() {
2709 assert_eq!(
2710 parse_pr_number_from_url("https://github.com/owner/repo/pull/42"),
2711 Ok(42)
2712 );
2713 }
2714
2715 #[test]
2716 fn parse_pr_number_trailing_slash() {
2717 assert_eq!(
2718 parse_pr_number_from_url("https://github.com/owner/repo/pull/42/"),
2719 Ok(42)
2720 );
2721 }
2722
2723 #[test]
2724 fn parse_pr_number_with_query_string() {
2725 assert_eq!(
2726 parse_pr_number_from_url("https://github.com/owner/repo/pull/42?tab=files"),
2727 Ok(42)
2728 );
2729 }
2730
2731 #[test]
2732 fn parse_pr_number_with_hash_fragment() {
2733 assert_eq!(
2734 parse_pr_number_from_url("https://github.com/owner/repo/pull/42#issuecomment-123"),
2735 Ok(42)
2736 );
2737 }
2738
2739 #[test]
2740 fn parse_pr_number_with_query_and_hash() {
2741 assert_eq!(
2742 parse_pr_number_from_url("https://github.com/owner/repo/pull/42?quick_pull=1#top"),
2743 Ok(42)
2744 );
2745 }
2746
2747 #[test]
2748 fn parse_pr_number_with_leading_whitespace() {
2749 assert_eq!(
2750 parse_pr_number_from_url(" https://github.com/owner/repo/pull/42\n"),
2751 Ok(42)
2752 );
2753 }
2754
2755 #[test]
2756 fn parse_pr_number_wrong_domain() {
2757 let url = "https://gitlab.com/owner/repo/merge_requests/42";
2758 let result = parse_pr_number_from_url(url);
2759 assert!(result.is_err());
2760 assert!(result
2761 .unwrap_err()
2762 .contains("does not start with 'https://github.com/'"));
2763 }
2764
2765 #[test]
2766 fn parse_pr_number_missing_pull_segment() {
2767 let url = "https://github.com/owner/repo/issues/42";
2769 let result = parse_pr_number_from_url(url);
2770 assert!(result.is_err());
2771 assert!(result.unwrap_err().contains("missing '/pull/' segment"));
2772 }
2773
2774 #[test]
2775 fn parse_pr_number_zero_is_rejected() {
2776 let url = "https://github.com/owner/repo/pull/0";
2777 let result = parse_pr_number_from_url(url);
2778 assert!(result.is_err());
2779 assert!(result.unwrap_err().contains("cannot be zero"));
2780 }
2781
2782 #[test]
2783 fn parse_pr_number_non_numeric_segment() {
2784 let url = "https://github.com/owner/repo/pull/abc";
2785 let result = parse_pr_number_from_url(url);
2786 assert!(result.is_err());
2787 assert!(result.unwrap_err().contains("not a valid positive integer"));
2788 }
2789
2790 #[test]
2791 fn parse_pr_number_empty_string() {
2792 let result = parse_pr_number_from_url("");
2793 assert!(result.is_err());
2794 assert!(result.unwrap_err().contains("empty"));
2795 }
2796
2797 #[test]
2798 fn parse_pr_number_http_not_https() {
2799 let url = "http://github.com/owner/repo/pull/42";
2801 let result = parse_pr_number_from_url(url);
2802 assert!(result.is_err());
2803 assert!(result
2804 .unwrap_err()
2805 .contains("does not start with 'https://github.com/'"));
2806 }
2807
2808 #[test]
2816 fn format_review_error_summarises_rate_limit_fragment_with_api_retry() {
2817 let raw_fragment = "{\"attempt\":6,\"max_retries\":10,\"retry_delay_ms\":17266.789,\"error_status\":429,\"error\":\"rate_limit\",\"session_id\":\"5c7148ee\"}\n\
2821 {\"type\":\"system\",\"subtype\":\"api_retry\",\"attempt\":7,\"max_retries\":10,\"retry_delay_ms\":35162.66,\"error_status\":429,\"error\":\"rate_limit\",\"session_id\":\"5c7148ee\"}";
2822 let err = runner::agents::AgentError::RateLimit {
2823 message: raw_fragment.to_string(),
2824 };
2825
2826 let stored = format_review_error("minimax", &err);
2827
2828 assert!(
2831 !stored.contains("{\"type\""),
2832 "stored error must not contain raw NDJSON envelope, got: {stored}"
2833 );
2834 assert!(
2835 !stored.contains("subtype"),
2836 "stored error must not contain raw api_retry fragment, got: {stored}"
2837 );
2838 assert!(
2839 !stored.contains("session_id"),
2840 "stored error must not contain session_id from provider payload, got: {stored}"
2841 );
2842 assert_eq!(
2843 stored,
2844 "minimax rate limit: status=429 after 7 attempts (last delay 35s)"
2845 );
2846 }
2847
2848 #[test]
2849 fn format_review_error_summarises_truncated_fragment_without_subtype_marker() {
2850 let raw_fragment = ",\"attempt\":5,\"max_retries\":10,\"retry_delay_ms\":9034.81,\"error_status\":429,\"error\":\"rate_limit\",\"session_id\":\"992e17d3\"";
2855 let err = runner::agents::AgentError::RateLimit {
2856 message: raw_fragment.to_string(),
2857 };
2858
2859 let stored = format_review_error("glm", &err);
2860
2861 assert!(stored.starts_with("glm rate limit:"));
2862 assert!(
2863 !stored.contains("session_id"),
2864 "stored error must not leak session_id from truncated fragment, got: {stored}"
2865 );
2866 assert!(
2869 stored.len() <= 200,
2870 "stored error should be compact (<=200 chars), got {} chars: {stored}",
2871 stored.len()
2872 );
2873 }
2874
2875 #[test]
2876 fn format_review_error_preserves_non_rate_limit_variants() {
2877 let err = runner::agents::AgentError::Auth {
2878 message: "invalid bearer token".to_string(),
2879 };
2880 assert_eq!(
2881 format_review_error("kimi", &err),
2882 "kimi auth error: invalid bearer token"
2883 );
2884
2885 let err = runner::agents::AgentError::Timeout {
2886 elapsed: std::time::Duration::from_secs(1801),
2887 };
2888 assert_eq!(
2889 format_review_error("kimi", &err),
2890 "kimi timed out after 1801s"
2891 );
2892 }
2893
2894 fn git(dir: &std::path::Path, args: &[&str]) {
2895 let mut cmd = std::process::Command::new("git");
2896 cmd.current_dir(dir);
2897 if args.first() == Some(&"commit") {
2899 cmd.args(["-c", "commit.gpgsign=false"]);
2900 }
2901 let status = cmd.args(args).status().unwrap();
2902 assert!(status.success(), "git {:?} failed", args);
2903 }
2904
2905 #[test]
2906 fn test_pull_request_review_requests_changes() {
2907 let review = PullRequestReview {
2908 review: GitHubReview {
2909 id: 1,
2910 user: GitHubUser {
2911 login: "reviewer".to_string(),
2912 },
2913 body: Some("Please fix".to_string()),
2914 state: "CHANGES_REQUESTED".to_string(),
2915 html_url: None,
2916 submitted_at: "2024-01-01T00:00:00Z".to_string(),
2917 commit_id: None,
2918 },
2919 comments: vec![],
2920 };
2921 assert!(review.requests_changes());
2922 }
2923
2924 #[test]
2925 fn test_pull_request_review_does_not_request_changes() {
2926 let review = PullRequestReview {
2927 review: GitHubReview {
2928 id: 1,
2929 user: GitHubUser {
2930 login: "reviewer".to_string(),
2931 },
2932 body: Some("LGTM".to_string()),
2933 state: "APPROVED".to_string(),
2934 html_url: None,
2935 submitted_at: "2024-01-01T00:00:00Z".to_string(),
2936 commit_id: None,
2937 },
2938 comments: vec![],
2939 };
2940 assert!(!review.requests_changes());
2941 }
2942
2943 #[test]
2944 fn test_pull_request_review_actionable_comments_filters_empty_and_replies() {
2945 let review = PullRequestReview {
2946 review: GitHubReview {
2947 id: 1,
2948 user: GitHubUser {
2949 login: "reviewer".to_string(),
2950 },
2951 body: None,
2952 state: "CHANGES_REQUESTED".to_string(),
2953 html_url: None,
2954 submitted_at: "2024-01-01T00:00:00Z".to_string(),
2955 commit_id: None,
2956 },
2957 comments: vec![
2958 GitHubReviewComment {
2959 id: 1,
2960 user: GitHubUser {
2961 login: "reviewer".to_string(),
2962 },
2963 body: "Fix this issue".to_string(),
2964 path: "src/main.rs".to_string(),
2965 line: Some(10),
2966 original_line: Some(10),
2967 commit_id: "abc".to_string(),
2968 original_commit_id: "abc".to_string(),
2969 html_url: "url".to_string(),
2970 created_at: "2024-01-01T00:00:00Z".to_string(),
2971 updated_at: "2024-01-01T00:00:00Z".to_string(),
2972 in_reply_to_id: None,
2973 diff_hunk: Some(
2974 "@@ -8,5 +8,5 @@ fn main() {\n- let x = 1;\n+ let x = 2;".to_string(),
2975 ),
2976 },
2977 GitHubReviewComment {
2978 id: 2,
2979 user: GitHubUser {
2980 login: "reviewer".to_string(),
2981 },
2982 body: "".to_string(),
2983 path: "src/lib.rs".to_string(),
2984 line: Some(20),
2985 original_line: Some(20),
2986 commit_id: "abc".to_string(),
2987 original_commit_id: "abc".to_string(),
2988 html_url: "url".to_string(),
2989 created_at: "2024-01-01T00:00:00Z".to_string(),
2990 updated_at: "2024-01-01T00:00:00Z".to_string(),
2991 in_reply_to_id: None,
2992 diff_hunk: None,
2993 },
2994 GitHubReviewComment {
2995 id: 3,
2996 user: GitHubUser {
2997 login: "reviewer".to_string(),
2998 },
2999 body: "Reply to this".to_string(),
3000 path: "src/lib.rs".to_string(),
3001 line: Some(30),
3002 original_line: Some(30),
3003 commit_id: "abc".to_string(),
3004 original_commit_id: "abc".to_string(),
3005 html_url: "url".to_string(),
3006 created_at: "2024-01-01T00:00:00Z".to_string(),
3007 updated_at: "2024-01-01T00:00:00Z".to_string(),
3008 in_reply_to_id: Some(1),
3009 diff_hunk: None,
3010 },
3011 ],
3012 };
3013 let actionable = review.actionable_comments();
3014 assert_eq!(actionable.len(), 1);
3015 assert_eq!(actionable[0].id, 1);
3016 assert_eq!(actionable[0].body, "Fix this issue");
3017 assert_eq!(actionable[0].path, "src/main.rs");
3018 assert_eq!(
3019 actionable[0].diff_hunk.as_ref().unwrap(),
3020 "@@ -8,5 +8,5 @@ fn main() {\n- let x = 1;\n+ let x = 2;"
3021 );
3022 }
3023
3024 #[test]
3025 fn test_router_config_model_for_complexity_returns_none_without_config() {
3026 let cfg = RouterConfig::default();
3029 assert!(cfg.model_for_complexity("claude", "simple", "").is_none());
3030 assert!(cfg.model_for_complexity("opencode", "review", "").is_none());
3031 assert!(cfg
3032 .model_for_complexity("unknown_agent_xyz", "simple", "")
3033 .is_none());
3034 }
3035
3036 #[test]
3037 fn review_started_comment_includes_agent_and_model() {
3038 let expected_footer = attribution_footer("Review started", "kimi", Some("opus"));
3039 let expected = format!("🔍 Automated review started{}", expected_footer);
3040 assert_eq!(review_started_comment("kimi", "opus"), expected);
3041 }
3042
3043 fn no_code_review_task(id: &str) -> ExternalTask {
3046 ExternalTask {
3047 id: crate::backends::ExternalId(id.to_string()),
3048 title: "opencode discovery timeout".to_string(),
3049 body: "".to_string(),
3050 state: "open".to_string(),
3051 labels: vec![],
3052 author: "tester".to_string(),
3053 created_at: "2026-01-01T00:00:00Z".to_string(),
3054 updated_at: "2026-01-01T00:00:00Z".to_string(),
3055 url: "https://example.com/issues/1".to_string(),
3056 }
3057 }
3058
3059 #[tokio::test]
3060 async fn build_review_context_reroutes_external_task_with_no_code() {
3061 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3062 let repo = "owner/repo";
3063 let ext = no_code_review_task("3617");
3064 store.ensure_external_task(repo, &ext).await.unwrap();
3065
3066 let backend = Arc::new(crate::backends::test_helpers::NoopBackend);
3067 let task_manager = Arc::new(TaskManager::with_store(
3068 backend,
3069 Arc::clone(&store),
3070 repo.to_string(),
3071 ));
3072 let router = Arc::new(RwLock::new(Router::new(RouterConfig::default())));
3073
3074 let phase = build_review_context(&ext, repo, &router, &task_manager, &store)
3075 .await
3076 .unwrap();
3077
3078 assert!(
3079 matches!(phase, ReviewPhase::EarlyReturn(ReviewDecision::Rerouted)),
3080 "external task with no worktree/branch/pr must reroute, not skip-to-done"
3081 );
3082
3083 let stored = store
3084 .get_by_external_id(repo, &ext.id.0)
3085 .await
3086 .unwrap()
3087 .expect("task must still exist");
3088 assert_eq!(stored.status, TaskStatus::New);
3089 assert!(stored.agent.is_none());
3090 assert!(stored.model.is_none());
3091 }
3092
3093 #[tokio::test]
3094 async fn build_review_context_blocks_external_task_after_max_reroutes() {
3095 let temp_home = TempDir::new().unwrap();
3098 let orch_dir = temp_home.path().join(".orch");
3099 std::fs::create_dir_all(&orch_dir).unwrap();
3100 let old_orch_home = std::env::var_os("ORCH_HOME");
3101 std::env::set_var("ORCH_HOME", &orch_dir);
3102
3103 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3104 let repo = "owner/repo";
3105 let ext = no_code_review_task("3618");
3106 store.ensure_external_task(repo, &ext).await.unwrap();
3107
3108 let backend = Arc::new(crate::backends::test_helpers::NoopBackend);
3109 let task_manager = Arc::new(TaskManager::with_store(
3110 backend,
3111 Arc::clone(&store),
3112 repo.to_string(),
3113 ));
3114 let router = Arc::new(RwLock::new(Router::new(RouterConfig::default())));
3115
3116 for _ in 0..3 {
3118 let phase = build_review_context(&ext, repo, &router, &task_manager, &store)
3119 .await
3120 .unwrap();
3121 if let ReviewPhase::EarlyReturn(ReviewDecision::Blocked(_)) = phase {
3122 break;
3123 }
3124 }
3125
3126 let stored = store
3127 .get_by_external_id(repo, &ext.id.0)
3128 .await
3129 .unwrap()
3130 .expect("task must still exist");
3131 assert_eq!(stored.status, TaskStatus::Blocked);
3132
3133 if let Some(old) = old_orch_home {
3134 std::env::set_var("ORCH_HOME", old);
3135 } else {
3136 std::env::remove_var("ORCH_HOME");
3137 }
3138 }
3139
3140 #[tokio::test]
3141 async fn build_review_context_marks_internal_task_done_with_no_code() {
3142 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3143 let repo = "owner/repo";
3144 let internal_id = store
3145 .create_internal(repo, "Daily retrospective", "", "job", "job-1", None)
3146 .await
3147 .unwrap();
3148 let ext_id = format!("internal:{internal_id}");
3149 let ext = no_code_review_task(&ext_id);
3150
3151 let backend = Arc::new(crate::backends::test_helpers::NoopBackend);
3152 let task_manager = Arc::new(TaskManager::with_store(
3153 backend,
3154 Arc::clone(&store),
3155 repo.to_string(),
3156 ));
3157 let router = Arc::new(RwLock::new(Router::new(RouterConfig::default())));
3158
3159 let phase = build_review_context(&ext, repo, &router, &task_manager, &store)
3160 .await
3161 .unwrap();
3162
3163 assert!(matches!(
3164 phase,
3165 ReviewPhase::EarlyReturn(ReviewDecision::Skipped)
3166 ));
3167 let stored = store.get(internal_id).await.unwrap();
3168 assert_eq!(stored.status, TaskStatus::Done);
3169 }
3170
3171 #[test]
3172 fn no_code_reroute_skip_check_rejects_agent_stream_disconnects() {
3173 let last_error =
3174 "codex failed: Reconnecting... (stream disconnected before completion: Broken pipe)";
3175 assert!(!should_skip_no_code_reroute_increment(last_error));
3176 }
3177
3178 #[test]
3187 fn review_fallback_scan_ignores_rate_limit_word_buried_in_long_report() {
3188 let padding = "x".repeat(10_000);
3189 let text_for_review = format!(
3190 "{padding}\n+| codex | gpt-5.4 | failed | 1 |\n+| codex | gpt-5.5 | failed / aborted / (empty) | 1 each |\n+| kimi | opus | rate_limit | 1 |\n{padding}"
3191 );
3192
3193 let scan_head = runner::agents::patterns::safe_head(
3194 &text_for_review,
3195 runner::agents::patterns::RATE_LIMIT_SCAN_HEAD_BYTES,
3196 );
3197 let scan_tail = runner::agents::patterns::safe_tail(
3198 &text_for_review,
3199 runner::agents::patterns::RATE_LIMIT_SCAN_TAIL_BYTES,
3200 );
3201
3202 assert!(runner::agents::patterns::detect_rate_limit(scan_head).is_none());
3203 assert!(runner::agents::patterns::detect_rate_limit(scan_tail).is_none());
3204 }
3205
3206 #[test]
3207 fn review_fallback_scan_still_detects_real_rate_limit_at_tail() {
3208 let padding = "x".repeat(10_000);
3209 let text_for_review =
3210 format!("{padding}\nError: rate limit exceeded, please try again later");
3211
3212 let scan_tail = runner::agents::patterns::safe_tail(
3213 &text_for_review,
3214 runner::agents::patterns::RATE_LIMIT_SCAN_TAIL_BYTES,
3215 );
3216
3217 assert!(runner::agents::patterns::detect_rate_limit(scan_tail).is_some());
3218 }
3219
3220 #[tokio::test]
3221 async fn recover_missing_review_worktree_recreates_worktree() {
3222 let temp_home = TempDir::new().unwrap();
3223 let orch_dir = temp_home.path().join(".orch");
3224 std::fs::create_dir_all(&orch_dir).unwrap();
3225 let old_orch_home = std::env::var_os("ORCH_HOME");
3226 std::env::set_var("ORCH_HOME", &orch_dir);
3227
3228 let project_dir = temp_home.path().join("project");
3229 std::fs::create_dir_all(&project_dir).unwrap();
3230 git(&project_dir, &["init", "-b", "main"]);
3231 git(&project_dir, &["config", "user.name", "Test User"]);
3232 git(&project_dir, &["config", "user.email", "test@example.com"]);
3233 std::fs::write(project_dir.join("README.md"), "hello").unwrap();
3234 git(&project_dir, &["add", "README.md"]);
3235 git(&project_dir, &["commit", "-m", "init"]);
3236
3237 std::fs::write(
3238 orch_dir.join("config.yml"),
3239 format!("projects:\n - {}\n", project_dir.display()),
3240 )
3241 .unwrap();
3242 std::fs::write(project_dir.join(".orch.yml"), "gh:\n repo: owner/repo\n").unwrap();
3243
3244 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3245 let repo = "owner/repo";
3246 store
3247 .create_internal(repo, "Fix review recovery", "", "review", "review-1", None)
3248 .await
3249 .unwrap();
3250 let ext = ExternalTask {
3251 id: crate::backends::ExternalId("1".to_string()),
3252 title: "Fix review recovery".to_string(),
3253 body: "".to_string(),
3254 state: "open".to_string(),
3255 labels: vec![],
3256 author: "tester".to_string(),
3257 created_at: "2026-01-01T00:00:00Z".to_string(),
3258 updated_at: "2026-01-01T00:00:00Z".to_string(),
3259 url: "https://example.com/issues/1".to_string(),
3260 };
3261 store.ensure_external_task(repo, &ext).await.unwrap();
3262
3263 let recovered = recover_missing_review_worktree(&ext, repo, &store)
3264 .await
3265 .unwrap();
3266
3267 assert!(recovered.work_dir.exists());
3268 assert!(recovered.branch.starts_with("gh-issue-1-"));
3269
3270 if let Some(old) = old_orch_home {
3271 std::env::set_var("ORCH_HOME", old);
3272 } else {
3273 std::env::remove_var("ORCH_HOME");
3274 }
3275 }
3276
3277 #[tokio::test]
3280 async fn count_ahead_commits_returns_zero_when_no_ahead_commits() {
3281 let temp = TempDir::new().unwrap();
3282 let dir = temp.path();
3283
3284 let bare = temp.path().join("bare.git");
3286 std::fs::create_dir_all(&bare).unwrap();
3287 std::process::Command::new("git")
3288 .args(["init", "--bare"])
3289 .current_dir(&bare)
3290 .output()
3291 .unwrap();
3292
3293 git(dir, &["init", "-b", "main"]);
3294 git(dir, &["config", "user.name", "Test"]);
3295 git(dir, &["config", "user.email", "test@test.com"]);
3296 std::fs::write(dir.join("f"), "a").unwrap();
3297 git(dir, &["add", "f"]);
3298 git(dir, &["commit", "-m", "init"]);
3299 std::process::Command::new("git")
3300 .args(["remote", "add", "origin", bare.to_str().unwrap()])
3301 .current_dir(dir)
3302 .output()
3303 .unwrap();
3304 std::process::Command::new("git")
3305 .args(["push", "origin", "main"])
3306 .current_dir(dir)
3307 .output()
3308 .unwrap();
3309
3310 let result = count_ahead_commits(dir, "main").await;
3312 assert!(
3313 result.is_ok(),
3314 "expected Ok when origin/main exists, got {result:?}"
3315 );
3316 assert_eq!(
3317 result.unwrap(),
3318 0,
3319 "expected 0 commits ahead when branch matches origin/main"
3320 );
3321 }
3322
3323 #[tokio::test]
3324 async fn count_ahead_commits_returns_error_when_remote_ref_missing() {
3325 let temp = TempDir::new().unwrap();
3326 let dir = temp.path();
3327 git(dir, &["init", "-b", "main"]);
3328 git(dir, &["config", "user.name", "Test"]);
3329 git(dir, &["config", "user.email", "test@test.com"]);
3330 std::fs::write(dir.join("f"), "a").unwrap();
3331 git(dir, &["add", "f"]);
3332 git(dir, &["commit", "-m", "init"]);
3333 let result = count_ahead_commits(dir, "main").await;
3335 assert!(
3336 result.is_err(),
3337 "expected Err when origin/main does not exist, got {result:?}"
3338 );
3339 let err = result.unwrap_err();
3340 assert!(
3341 err.contains("git rev-list exited"),
3342 "error should mention exit failure, got: {err}"
3343 );
3344 }
3345
3346 #[tokio::test]
3347 async fn count_ahead_commits_returns_error_when_git_cannot_start() {
3348 let temp = TempDir::new().unwrap();
3350 let dir = temp.path();
3351 let result = count_ahead_commits(dir, "main").await;
3353 assert!(
3354 result.is_err(),
3355 "expected Err when directory is not a git repo, got {result:?}"
3356 );
3357 }
3358
3359 #[serial_test::serial(cooldown_state)]
3369 #[tokio::test]
3370 async fn review_invalid_response_applies_persistent_model_cooldown() {
3371 crate::engine::cooldown::reset_global_state().await;
3372 let agent = "opencode-3314-review-invalid";
3373 let model = "opencode/north-mini-code-free";
3374 let key = format!("{agent}:{model}");
3375
3376 assert!(
3377 !crate::engine::cooldown::is_model_in_cooldown(agent, model),
3378 "model should not be in cooldown before record_review_agent_failure"
3379 );
3380
3381 let err = runner::agents::AgentError::InvalidResponse {
3382 raw: "empty review output".to_string(),
3383 };
3384 record_review_agent_failure("test-3314-a", agent, Some(model), &err).await;
3385
3386 let now = chrono::Utc::now().timestamp();
3387 let until = crate::engine::cooldown::cooldown_until(&key)
3388 .expect("review InvalidResponse should set model cooldown");
3389 let remaining = until.saturating_sub(now);
3390
3391 assert!(
3394 remaining >= crate::engine::cooldown::PERSISTENT_MODEL_BACKOFF_BASE_SECS - 30,
3395 "review InvalidResponse must apply ~4h persistent cooldown (issue #3314), \
3396 got {remaining}s — may have used 5-min standard backoff instead"
3397 );
3398 }
3399
3400 #[serial_test::serial(cooldown_state)]
3408 #[tokio::test]
3409 async fn review_persistent_cooldown_survives_after_first_failure() {
3410 crate::engine::cooldown::reset_global_state().await;
3411 let agent = "opencode-3314-review-parse";
3412 let model = "opencode/north-mini-code-free";
3413 let key = format!("{agent}:{model}");
3414
3415 let err = runner::agents::AgentError::InvalidResponse {
3417 raw: "failed to parse review response".to_string(),
3418 };
3419 record_review_agent_failure("test-3314-b", agent, Some(model), &err).await;
3420
3421 let now = chrono::Utc::now().timestamp();
3422 let until1 = crate::engine::cooldown::cooldown_until(&key)
3423 .expect("first failure should set cooldown");
3424 let remaining1 = until1.saturating_sub(now);
3425
3426 crate::engine::cooldown::record_agent_success(agent, model).await;
3429
3430 assert!(
3436 remaining1 >= crate::engine::cooldown::PERSISTENT_MODEL_BACKOFF_BASE_SECS - 30,
3437 "after success reset, first-failure cooldown should still be ~4h, got {remaining1}s"
3438 );
3439 }
3440
3441 #[serial_test::serial(cooldown_state)]
3447 #[tokio::test]
3448 async fn review_inline_scan_billing_cycle_message_gets_model_scoped_cooldown() {
3449 crate::engine::cooldown::reset_global_state().await;
3450 let agent = "kimi-3551-review-inline";
3451 let model = "opus";
3452 let key = format!("{agent}:{model}");
3453
3454 let text_for_review = "kimi rate limit: Failed to authenticate. API Error: 403 \
3456 You've reached your usage limit for this billing cycle. Your quota will be \
3457 refreshed next cycle.";
3458
3459 let scan_tail = runner::agents::patterns::safe_tail(
3460 text_for_review,
3461 runner::agents::patterns::RATE_LIMIT_SCAN_TAIL_BYTES,
3462 );
3463 let err = runner::agents::patterns::detect_rate_limit(scan_tail)
3464 .expect("billing-cycle message must still be caught by the rate-limit scan");
3465
3466 assert_eq!(
3467 outcome_for_agent_error(&err),
3468 "billing_cycle_exhausted",
3469 "inline-scanned billing-cycle message must not be stored as plain rate_limit"
3470 );
3471
3472 record_review_agent_failure("test-3551", agent, Some(model), &err).await;
3473
3474 let now = chrono::Utc::now().timestamp();
3475 let until = crate::engine::cooldown::cooldown_until(&key)
3476 .expect("billing-cycle exhaustion must set a model-scoped cooldown");
3477 let remaining = until.saturating_sub(now);
3478 assert!(
3483 remaining >= crate::engine::cooldown::PERSISTENT_MODEL_BACKOFF_BASE_SECS - 30,
3484 "billing-cycle exhaustion for a known model must apply the ~4h persistent \
3485 model cooldown, got {remaining}s — may have used the generic short agent \
3486 cooldown instead"
3487 );
3488 }
3489
3490 fn test_review_task(id: &str) -> ExternalTask {
3493 ExternalTask {
3494 id: crate::backends::ExternalId(id.to_string()),
3495 title: "Test review task".to_string(),
3496 body: "".to_string(),
3497 state: "open".to_string(),
3498 labels: vec![],
3499 author: "tester".to_string(),
3500 created_at: "2026-01-01T00:00:00Z".to_string(),
3501 updated_at: "2026-01-01T00:00:00Z".to_string(),
3502 url: "https://example.com/issues/1".to_string(),
3503 }
3504 }
3505
3506 fn test_review_context(agent: &str, model: &str, work_dir: &std::path::Path) -> ReviewContext {
3507 ReviewContext {
3508 store_id: None,
3509 had_prev_push_failure: false,
3510 review_attempt: 1,
3511 worktree_path: work_dir.to_path_buf(),
3512 branch_name: "gh-issue-1-test".to_string(),
3513 pr_number: 1,
3514 default_branch: "main".to_string(),
3515 review_agent: agent.to_string(),
3516 review_model: Some(model.to_string()),
3517 review_task_id: "review-1".to_string(),
3518 review_attempt_dir: work_dir.to_path_buf(),
3519 output_file: work_dir.join("output.txt"),
3520 invocation: runner::agent::AgentInvocation {
3521 agent: agent.to_string(),
3522 model: Some(model.to_string()),
3523 work_dir: work_dir.to_path_buf(),
3524 system_prompt: String::new(),
3525 agent_message: String::new(),
3526 task_id: "review-1".to_string(),
3527 disallowed_tools: vec![],
3528 git_author_name: "Test".to_string(),
3529 git_author_email: "test@example.com".to_string(),
3530 output_file: work_dir.join("output.txt"),
3531 timeout_seconds: 0,
3532 repo: "owner/repo".to_string(),
3533 attempt: 1,
3534 },
3535 same_agent_fallback: false,
3536 }
3537 }
3538
3539 fn test_parsed_review(decision: ReviewDecision) -> ParsedReview {
3540 ParsedReview {
3541 run_id: None,
3542 exit_code: 0,
3543 stderr: String::new(),
3544 raw_output: String::new(),
3545 text_for_review: String::new(),
3546 token_usage: RunTokenUsage::default(),
3547 review_notes_for_comment: String::new(),
3548 decision,
3549 }
3550 }
3551
3552 #[serial_test::serial(cooldown_state)]
3558 #[tokio::test]
3559 async fn finalize_review_run_success_resets_generic_model_failure_count() {
3560 crate::engine::cooldown::reset_global_state().await;
3561 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3562 crate::engine::cooldown::init_cooldown_store(store.clone()).await;
3563
3564 let agent = "test-3478-agent";
3565 let model = "test-3478-model";
3566 let failure_key = format!("failure_count:{agent}:{model}");
3567
3568 let err = runner::agents::AgentError::AgentFailed {
3572 message: "agent reported a generic failure".to_string(),
3573 };
3574 record_review_agent_failure("task-3478-a", agent, Some(model), &err).await;
3575 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
3576 assert_eq!(
3577 store.kv_get(&failure_key).await.unwrap().as_deref(),
3578 Some("1"),
3579 "sanity check: failure should have incremented the persisted count"
3580 );
3581
3582 let work_dir = TempDir::new().unwrap();
3583 let ctx = test_review_context(agent, model, work_dir.path());
3584 let parsed = test_parsed_review(ReviewDecision::Approve);
3585 let task = test_review_task("1");
3586
3587 finalize_review_run(&task, &ctx, &parsed, &store).await;
3588 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
3589
3590 assert_eq!(
3591 store.kv_get(&failure_key).await.unwrap().as_deref(),
3592 Some("0"),
3593 "successful review must reset failure_count:{{agent}}:{{model}} (issue #3478)"
3594 );
3595 }
3596
3597 #[serial_test::serial(cooldown_state)]
3603 #[tokio::test]
3604 async fn finalize_review_run_interleaved_success_keeps_generic_cooldown_at_base() {
3605 crate::engine::cooldown::reset_global_state().await;
3606 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3607 crate::engine::cooldown::init_cooldown_store(store.clone()).await;
3608
3609 let agent = "test-3478-interleave-agent";
3610 let model = "test-3478-interleave-model";
3611 let cooldown_key = format!("{agent}:{model}");
3612
3613 let work_dir = TempDir::new().unwrap();
3614 let ctx = test_review_context(agent, model, work_dir.path());
3615 let task = test_review_task("2");
3616
3617 for i in 0..4 {
3618 let err = runner::agents::AgentError::AgentFailed {
3619 message: format!("failure {i}"),
3620 };
3621 record_review_agent_failure("task-3478-b", agent, Some(model), &err).await;
3622
3623 let now = chrono::Utc::now().timestamp();
3624 let until = crate::engine::cooldown::cooldown_until(&cooldown_key)
3625 .expect("failure should set cooldown");
3626 let remaining = until.saturating_sub(now);
3627
3628 assert!(
3629 remaining <= crate::engine::cooldown::BACKOFF_BASE_SECS + 60,
3630 "failure #{} must stay at base backoff since the prior success reset the \
3631 counter (issue #3478); got {remaining}s, base is {}s — an escalating value \
3632 here means review successes are not resetting failure_count",
3633 i + 1,
3634 crate::engine::cooldown::BACKOFF_BASE_SECS
3635 );
3636
3637 let parsed = test_parsed_review(ReviewDecision::Approve);
3641 finalize_review_run(&task, &ctx, &parsed, &store).await;
3642 tokio::time::sleep(std::time::Duration::from_millis(20)).await;
3643 }
3644 }
3645
3646 #[serial_test::serial(cooldown_state)]
3652 #[tokio::test]
3653 async fn finalize_review_run_success_does_not_reset_persistent_model_failure_count() {
3654 crate::engine::cooldown::reset_global_state().await;
3655 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3656 crate::engine::cooldown::init_cooldown_store(store.clone()).await;
3657
3658 let agent = "test-3571-agent";
3659 let model = "test-3571-model";
3660 let cooldown_key = format!("{agent}:{model}");
3661
3662 let work_dir = TempDir::new().unwrap();
3663 let ctx = test_review_context(agent, model, work_dir.path());
3664 let task = test_review_task("3");
3665
3666 let err = runner::agents::AgentError::InvalidResponse {
3668 raw: "empty review output".to_string(),
3669 };
3670 record_review_agent_failure("task-3571-a", agent, Some(model), &err).await;
3671 tokio::time::sleep(std::time::Duration::from_millis(20)).await;
3672
3673 let parsed = test_parsed_review(ReviewDecision::Approve);
3676 finalize_review_run(&task, &ctx, &parsed, &store).await;
3677 tokio::time::sleep(std::time::Duration::from_millis(20)).await;
3678
3679 let err2 = runner::agents::AgentError::InvalidResponse {
3682 raw: "empty review output again".to_string(),
3683 };
3684 record_review_agent_failure("task-3571-b", agent, Some(model), &err2).await;
3685 tokio::time::sleep(std::time::Duration::from_millis(20)).await;
3686
3687 let now = chrono::Utc::now().timestamp();
3688 let until = crate::engine::cooldown::cooldown_until(&cooldown_key)
3689 .expect("second failure should set cooldown");
3690 let remaining = until.saturating_sub(now);
3691
3692 assert!(
3693 remaining > crate::engine::cooldown::PERSISTENT_MODEL_BACKOFF_BASE_SECS,
3694 "second persistent model failure should escalate past the 4h base despite an \
3695 intervening success (issue #3571); got {remaining}s, base is {}s",
3696 crate::engine::cooldown::PERSISTENT_MODEL_BACKOFF_BASE_SECS
3697 );
3698 }
3699}