1use std::{
4 fs,
5 io::{self, Read, Write},
6 path::{Path, PathBuf},
7 process::{Command, ExitCode, Stdio},
8 sync::{
9 Arc,
10 atomic::{AtomicBool, AtomicU64, Ordering},
11 },
12 time::{Duration, Instant, SystemTime, UNIX_EPOCH},
13};
14
15use anyhow::Result;
16use serde::{Deserialize, Serialize};
17use thiserror::Error;
18
19use crate::{
20 claim::{Claim, EvidenceRef},
21 cli::{self, Agent, ReviewScope, ReviewerHarness},
22 config::{self, Effort},
23 ledger::{
24 Disposition, LedgerEntry, LedgerError, LedgerStore, MemorySkillClassification,
25 MemorySkillClassificationKind, ResolutionKind, ReviewerConfig, StructuredFinding,
26 TransitionProvenance, TransitionProvenanceKind, Verdict,
27 },
28 provenance::{self, EntireCheckpointRef},
29 state::{StateTransaction, atomic_replace},
30 surface,
31 time::unix_now,
32};
33
34pub const REVIEW_QUEUE_FILE: &str = "review-queue.jsonl";
35pub const REVIEW_RUNS_DIR: &str = "runs";
36const BATCHES_DIR: &str = "batches";
37const ENQUEUE_INFLIGHT_FILE: &str = "enqueue-inflight.json";
38const CLAIMED_PHASE: &str = "claimed";
40const PROVIDER_BACKOFF_PHASE: &str = "provider_backoff";
42const DEFAULT_PROVIDER_BACKOFF_SECS: u64 = 60;
44const PETITION_FAILURES_FILE: &str = "petition-failures.jsonl";
45const REVIEW_LEDGER_PHASE: &str = "reviewing";
46const MAX_INLINE_DIFF_FILES: usize = 2;
47const MAX_INLINE_DIFF_BYTES: usize = 256 * 1024;
48const MAX_UNTRACKED_FILE_BYTES: u64 = 16 * 1024;
49
50#[derive(Clone, Debug, Eq, PartialEq)]
51pub struct ReviewRequest {
52 pub watched_agent: Agent,
53 pub watched_model: String,
54 pub reviewer_harness: ReviewerHarness,
55 pub reviewer_model: String,
56 pub reviewer_effort: Effort,
57 pub allow_same_model: bool,
58 pub prompt: String,
59}
60
61impl ReviewRequest {
62 pub fn new(
63 watched_agent: Agent,
64 watched_model: impl Into<String>,
65 reviewer_harness: ReviewerHarness,
66 reviewer_model: impl Into<String>,
67 allow_same_model: bool,
68 prompt: impl Into<String>,
69 ) -> Self {
70 Self {
71 watched_agent,
72 watched_model: watched_model.into(),
73 reviewer_harness,
74 reviewer_model: reviewer_model.into(),
75 reviewer_effort: Effort::highest(),
76 allow_same_model,
77 prompt: prompt.into(),
78 }
79 }
80
81 pub fn with_effort(mut self, effort: Effort) -> Self {
82 self.reviewer_effort = effort;
83 self
84 }
85}
86
87#[derive(Clone, Debug, Eq, PartialEq)]
90pub struct ReviewSelection {
91 pub watched_agent: Agent,
92 pub watched_model: String,
93 pub reviewer_harness: ReviewerHarness,
94 pub reviewer_model: String,
95 pub reviewer_effort: Effort,
96 pub allow_same_model: bool,
97 pub strict: Option<StrictReviewConfig>,
98}
99
100impl ReviewSelection {
101 #[allow(clippy::too_many_arguments)]
104 pub fn resolve(
105 watched_agent: Option<Agent>,
106 watched_model: Option<String>,
107 reviewer_harness: Option<ReviewerHarness>,
108 reviewer_model: Option<String>,
109 reviewer_effort: Option<Effort>,
110 allow_same_model: bool,
111 config: &config::TruthMirrorConfig,
112 ) -> Result<Self, ReviewerError> {
113 let watched_agent = match watched_agent {
114 Some(agent) => agent,
115 None => agent_from_slug(&config.default_writer)?,
116 };
117 let writer_slug = surface::agent_slug(watched_agent);
118 let pair = config.pair_for(writer_slug);
119
120 let harness_from_cli = reviewer_harness.is_some();
121 let reviewer_harness = match reviewer_harness {
122 Some(harness) => harness,
123 None => {
124 let slug = pair
125 .map(|pair| pair.reviewer.harness.as_str())
126 .ok_or_else(|| ReviewerError::NoPairForWriter {
127 writer: writer_slug.to_owned(),
128 })?;
129 harness_from_slug(slug)?
130 }
131 };
132 let reviewer_model = match reviewer_model {
133 Some(model) => model,
134 None => {
135 let pair = pair.ok_or_else(|| ReviewerError::NoPairForWriter {
136 writer: writer_slug.to_owned(),
137 })?;
138 if harness_from_cli
141 && !pair
142 .reviewer
143 .harness
144 .eq_ignore_ascii_case(harness_slug(reviewer_harness))
145 {
146 return Err(ReviewerError::OverrideNeedsModel {
147 role: "reviewer".to_owned(),
148 harness: harness_slug(reviewer_harness).to_owned(),
149 });
150 }
151 pair.reviewer.model.clone()
152 }
153 };
154 let reviewer_effort = reviewer_effort
155 .or_else(|| pair.map(|pair| pair.reviewer.effort))
156 .unwrap_or_else(Effort::highest);
157
158 Ok(Self {
159 watched_agent,
160 watched_model: watched_model.unwrap_or_default(),
161 reviewer_harness,
162 reviewer_model,
163 reviewer_effort,
164 allow_same_model: allow_same_model || config.allow_same_model,
166 strict: None,
167 })
168 }
169
170 pub fn resolve_arbiter(
173 watched_agent: Agent,
174 arbiter_harness: Option<ReviewerHarness>,
175 arbiter_model: Option<String>,
176 arbiter_effort: Option<Effort>,
177 config: &config::TruthMirrorConfig,
178 ) -> Result<StrictReviewConfig, ReviewerError> {
179 let pair_arbiter = config
180 .pair_for(surface::agent_slug(watched_agent))
181 .and_then(|pair| pair.arbiter.clone());
182
183 let harness_from_cli = arbiter_harness.is_some();
184 let harness = match arbiter_harness {
185 Some(harness) => harness,
186 None => {
187 let slug = pair_arbiter
188 .as_ref()
189 .map(|arbiter| arbiter.harness.as_str())
190 .ok_or(ReviewerError::MissingArbiter)?;
191 harness_from_slug(slug)?
192 }
193 };
194 let model = match arbiter_model {
195 Some(model) => model,
196 None => {
197 let arbiter = pair_arbiter.as_ref().ok_or(ReviewerError::MissingArbiter)?;
198 if harness_from_cli && !arbiter.harness.eq_ignore_ascii_case(harness_slug(harness))
199 {
200 return Err(ReviewerError::OverrideNeedsModel {
201 role: "arbiter".to_owned(),
202 harness: harness_slug(harness).to_owned(),
203 });
204 }
205 arbiter.model.clone()
206 }
207 };
208 let effort = arbiter_effort
209 .or_else(|| pair_arbiter.as_ref().map(|arbiter| arbiter.effort))
210 .unwrap_or_else(Effort::highest);
211
212 Ok(StrictReviewConfig {
213 arbiter_harness: harness,
214 arbiter_model: model,
215 arbiter_effort: effort,
216 })
217 }
218
219 fn request_for(&self, prompt: String) -> ReviewRequest {
220 ReviewRequest::new(
221 self.watched_agent,
222 self.watched_model.clone(),
223 self.reviewer_harness,
224 self.reviewer_model.clone(),
225 self.allow_same_model,
226 prompt,
227 )
228 .with_effort(self.reviewer_effort)
229 }
230}
231
232#[derive(Clone, Debug, Eq, PartialEq)]
233pub struct ReviewPlan {
234 pub watched_agent: Agent,
235 pub watched_model: String,
236 pub reviewer_harness: ReviewerHarness,
237 pub reviewer_model: String,
238 pub allow_same_model: bool,
239 pub invocation: InvocationPlan,
240}
241
242impl ReviewPlan {
243 pub fn build(request: ReviewRequest) -> Result<Self, ReviewerError> {
244 validate_model_present("reviewer", &request.reviewer_model)?;
245
246 if request.watched_model.trim().is_empty() {
249 if !request.allow_same_model {
250 eprintln!(
253 "{} warning: writer model is unknown; model opposition is only pair-based \
254 for this review. Pass --watched-model to enforce per-model opposition; \
255 truth-mirror has no persistent writer-model config key yet.",
256 crate::messages::diagnostic_prefix()
257 );
258 }
259 } else if !request.allow_same_model
260 && normalized_model(&request.watched_model) == normalized_model(&request.reviewer_model)
261 {
262 return Err(ReviewerError::SameModelWithoutWaiver {
263 watched_model: request.watched_model,
264 reviewer_model: request.reviewer_model,
265 });
266 }
267
268 let invocation = InvocationPlan::for_harness(
269 request.reviewer_harness,
270 &request.reviewer_model,
271 request.reviewer_effort,
272 )?;
273
274 Ok(Self {
275 watched_agent: request.watched_agent,
276 watched_model: request.watched_model,
277 reviewer_harness: request.reviewer_harness,
278 reviewer_model: request.reviewer_model,
279 allow_same_model: request.allow_same_model,
280 invocation,
281 })
282 }
283
284 pub fn run_with<R: ProcessRunner>(
285 &self,
286 prompt: &str,
287 runner: &R,
288 timeout: Option<Duration>,
289 ) -> Result<ProcessOutput, ReviewerError> {
290 runner.run(&self.invocation, prompt, timeout)
291 }
292
293 pub fn run_with_dir<R: ProcessRunner>(
294 &self,
295 prompt: &str,
296 runner: &R,
297 timeout: Option<Duration>,
298 current_dir: Option<&Path>,
299 ) -> Result<ProcessOutput, ReviewerError> {
300 runner.run_in_dir(&self.invocation, prompt, timeout, current_dir)
301 }
302 fn reviewer_config(&self) -> ReviewerConfig {
303 ReviewerConfig::new(
304 harness_slug(self.reviewer_harness),
305 self.reviewer_model.clone(),
306 self.allow_same_model,
307 )
308 }
309}
310
311#[derive(Clone, Debug, Eq, PartialEq)]
312pub struct InvocationPlan {
313 pub program: String,
314 pub args: Vec<String>,
315 pub prompt_delivery: PromptDelivery,
316}
317
318impl InvocationPlan {
319 pub fn for_harness(
320 harness: ReviewerHarness,
321 model: &str,
322 effort: Effort,
323 ) -> Result<Self, ReviewerError> {
324 validate_model_present("reviewer", model)?;
325 let model = model.trim();
326 let e = effort.as_str();
327
328 let plan = match harness {
332 ReviewerHarness::Claude => Self {
333 program: "claude".to_owned(),
334 args: vec![
335 "--print".to_owned(),
336 "--model".to_owned(),
337 model.to_owned(),
338 "--effort".to_owned(),
339 effort.claude_value().to_owned(),
341 ],
342 prompt_delivery: PromptDelivery::Stdin,
343 },
344 ReviewerHarness::Codex => Self {
345 program: "codex".to_owned(),
346 args: vec![
347 "exec".to_owned(),
348 "-m".to_owned(),
349 model.to_owned(),
350 "-c".to_owned(),
351 format!("model_reasoning_effort={e}"),
352 ],
353 prompt_delivery: PromptDelivery::PositionalArgument,
354 },
355 ReviewerHarness::Pi => Self {
356 program: "pi".to_owned(),
357 args: vec![
358 "--model".to_owned(),
359 model.to_owned(),
360 "--thinking".to_owned(),
361 e.to_owned(),
362 "--tools".to_owned(),
365 "read,grep,find,ls".to_owned(),
366 "-p".to_owned(),
367 ],
368 prompt_delivery: PromptDelivery::Stdin,
369 },
370 ReviewerHarness::Gemini => Self {
371 program: "gemini".to_owned(),
372 args: vec!["-m".to_owned(), model.to_owned()],
373 prompt_delivery: PromptDelivery::FlagValue("-p".to_owned()),
374 },
375 ReviewerHarness::Opencode => Self {
376 program: "opencode".to_owned(),
377 args: vec!["run".to_owned(), "--model".to_owned(), model.to_owned()],
378 prompt_delivery: PromptDelivery::PositionalArgument,
379 },
380 ReviewerHarness::Custom => return Err(ReviewerError::UnsupportedCustomHarness),
381 };
382
383 Ok(plan)
384 }
385
386 pub fn args_for_prompt(&self, prompt: &str) -> Vec<String> {
387 let mut args = self.args.clone();
388 match &self.prompt_delivery {
389 PromptDelivery::Stdin => {}
390 PromptDelivery::PositionalArgument => args.push(prompt.to_owned()),
391 PromptDelivery::FlagValue(flag) => {
392 args.push(flag.clone());
393 args.push(prompt.to_owned());
394 }
395 }
396 args
397 }
398}
399
400#[derive(Clone, Debug, Eq, PartialEq)]
401pub enum PromptDelivery {
402 Stdin,
403 PositionalArgument,
404 FlagValue(String),
405}
406
407#[derive(Clone, Debug, Eq, PartialEq)]
408pub struct ProcessOutput {
409 pub status_code: Option<i32>,
410 pub stdout: String,
411 pub stderr: String,
412}
413
414pub trait ProcessRunner {
415 fn run(
423 &self,
424 invocation: &InvocationPlan,
425 prompt: &str,
426 timeout: Option<Duration>,
427 ) -> Result<ProcessOutput, ReviewerError>;
428
429 fn run_in_dir(
433 &self,
434 invocation: &InvocationPlan,
435 prompt: &str,
436 timeout: Option<Duration>,
437 current_dir: Option<&Path>,
438 ) -> Result<ProcessOutput, ReviewerError> {
439 let _ = current_dir;
440 self.run(invocation, prompt, timeout)
441 }
442}
443
444#[derive(Clone, Copy, Debug, Default)]
445pub struct StdProcessRunner;
446
447impl ProcessRunner for StdProcessRunner {
448 fn run(
449 &self,
450 invocation: &InvocationPlan,
451 prompt: &str,
452 timeout: Option<Duration>,
453 ) -> Result<ProcessOutput, ReviewerError> {
454 self.run_in_dir(invocation, prompt, timeout, None)
455 }
456
457 fn run_in_dir(
458 &self,
459 invocation: &InvocationPlan,
460 prompt: &str,
461 timeout: Option<Duration>,
462 current_dir: Option<&Path>,
463 ) -> Result<ProcessOutput, ReviewerError> {
464 let mut command = Command::new(&invocation.program);
465 command.args(invocation.args_for_prompt(prompt));
466 command.stdout(Stdio::piped()).stderr(Stdio::piped());
467 if let Some(current_dir) = current_dir {
468 command.current_dir(current_dir);
469 }
470
471 if invocation.prompt_delivery == PromptDelivery::Stdin {
472 command.stdin(Stdio::piped());
473 }
474
475 #[cfg(unix)]
480 {
481 use std::os::unix::process::CommandExt as _;
482 command.process_group(0);
483 }
484
485 let deadline = timeout.and_then(|limit| {
492 Instant::now()
493 .checked_add(limit)
494 .map(|instant| (instant, limit))
495 });
496
497 let mut child = command.spawn().map_err(ReviewerError::Spawn)?;
498
499 let mut stdin_writer = if invocation.prompt_delivery == PromptDelivery::Stdin {
505 let pipe = child.stdin.take().ok_or(ReviewerError::MissingStdinPipe)?;
506 Some(StdinWriter::spawn(pipe, prompt.as_bytes().to_vec()))
507 } else {
508 None
509 };
510
511 let mut stdout_reader = child.stdout.take().map(spawn_pipe_reader);
516 let mut stderr_reader = child.stderr.take().map(spawn_pipe_reader);
517
518 let child_pid = child.id();
519 let status = match wait_for_child(child, deadline) {
520 Ok(status) => status,
521 Err(error) => {
522 let cleanup_deadline = deadline.map(|_| Instant::now() + PIPE_DRAIN_GRACE);
531 if let Some(reader) = stdout_reader.as_mut() {
532 let _ =
533 reader.try_collect(remaining_until(cleanup_deadline), child_pid, "stdout");
534 }
535 if let Some(reader) = stderr_reader.as_mut() {
536 let _ =
537 reader.try_collect(remaining_until(cleanup_deadline), child_pid, "stderr");
538 }
539 if let Some(writer) = stdin_writer.as_mut() {
540 let _ =
541 writer.try_collect(remaining_until(cleanup_deadline), child_pid, "stdin");
542 }
543 return Err(error);
544 }
545 };
546
547 let cleanup_deadline = deadline.map(|_| Instant::now() + PIPE_DRAIN_GRACE);
552 let mut prompt_write = stdin_writer.as_mut().map(|writer| {
553 writer.try_collect(remaining_until(cleanup_deadline), child_pid, "stdin")
554 });
555 let mut stdout = stdout_reader.as_mut().map(|reader| {
556 reader.try_collect(remaining_until(cleanup_deadline), child_pid, "stdout")
557 });
558 let mut stderr = stderr_reader.as_mut().map(|reader| {
559 reader.try_collect(remaining_until(cleanup_deadline), child_pid, "stderr")
560 });
561
562 if pipes_wedged(&prompt_write, &stdout, &stderr) {
563 let _ = platform_tree_kill(child_pid);
570 let retry_deadline = deadline.map(|_| Instant::now() + PIPE_DRAIN_GRACE);
571 if prompt_write.as_ref().is_some_and(WriteState::is_wedged) {
572 prompt_write = stdin_writer.as_mut().map(|writer| {
573 writer.try_collect(remaining_until(retry_deadline), child_pid, "stdin")
574 });
575 }
576 if stdout.as_ref().is_some_and(DrainState::is_wedged) {
577 stdout = stdout_reader.as_mut().map(|reader| {
578 reader.try_collect(remaining_until(retry_deadline), child_pid, "stdout")
579 });
580 }
581 if stderr.as_ref().is_some_and(DrainState::is_wedged) {
582 stderr = stderr_reader.as_mut().map(|reader| {
583 reader.try_collect(remaining_until(retry_deadline), child_pid, "stderr")
584 });
585 }
586 }
587 if pipes_wedged(&prompt_write, &stdout, &stderr) {
588 return Err(ReviewerError::ReviewerOutputWedged {
589 grace_secs: PIPE_DRAIN_GRACE.as_secs(),
590 });
591 }
592
593 if let Some(WriteState::Finished(Err(source))) = prompt_write {
597 return Err(ReviewerError::WritePrompt(source));
598 }
599
600 Ok(ProcessOutput {
601 status_code: status.code(),
602 stdout: String::from_utf8_lossy(&stdout.and_then(drain_bytes).unwrap_or_default())
603 .into_owned(),
604 stderr: String::from_utf8_lossy(&stderr.and_then(drain_bytes).unwrap_or_default())
605 .into_owned(),
606 })
607 }
608}
609
610enum DrainState {
612 Drained(Option<Vec<u8>>),
616 Wedged,
619}
620
621impl DrainState {
622 fn is_wedged(&self) -> bool {
623 matches!(self, Self::Wedged)
624 }
625}
626
627fn drain_bytes(state: DrainState) -> Option<Vec<u8>> {
630 match state {
631 DrainState::Drained(bytes) => bytes,
632 DrainState::Wedged => None,
633 }
634}
635
636fn pipes_wedged(
638 prompt_write: &Option<WriteState>,
639 stdout: &Option<DrainState>,
640 stderr: &Option<DrainState>,
641) -> bool {
642 prompt_write.as_ref().is_some_and(WriteState::is_wedged)
643 || stdout.as_ref().is_some_and(DrainState::is_wedged)
644 || stderr.as_ref().is_some_and(DrainState::is_wedged)
645}
646
647enum WriteState {
649 Finished(io::Result<()>),
650 Wedged,
651}
652
653impl WriteState {
654 fn is_wedged(&self) -> bool {
655 matches!(self, Self::Wedged)
656 }
657}
658
659struct PipeReader {
664 handle: Option<std::thread::JoinHandle<()>>,
665 receiver: std::sync::mpsc::Receiver<io::Result<Vec<u8>>>,
666}
667
668fn spawn_pipe_reader(mut pipe: impl Read + Send + 'static) -> PipeReader {
669 let (sender, receiver) = std::sync::mpsc::channel();
670 let handle = std::thread::spawn(move || {
671 let mut buffer = Vec::new();
672 let result = pipe.read_to_end(&mut buffer).map(|_| buffer);
673 let _ = sender.send(result);
674 });
675 PipeReader {
676 handle: Some(handle),
677 receiver,
678 }
679}
680
681impl PipeReader {
682 fn try_collect(
689 &mut self,
690 grace: Option<Duration>,
691 pid: u32,
692 label: &'static str,
693 ) -> DrainState {
694 use std::sync::mpsc::RecvTimeoutError;
695 let received = match grace {
696 Some(grace) => self.receiver.recv_timeout(grace),
697 None => self
698 .receiver
699 .recv()
700 .map_err(|_| RecvTimeoutError::Disconnected),
701 };
702 match received {
703 Ok(result) => {
704 if let Some(handle) = self.handle.take() {
705 let _ = handle.join();
706 }
707 DrainState::Drained(result.ok())
708 }
709 Err(RecvTimeoutError::Timeout | RecvTimeoutError::Disconnected) => {
710 if let Some(handle) = self.handle.take() {
711 reap_wedged_pipe_thread(pid, label, handle);
712 }
713 DrainState::Wedged
714 }
715 }
716 }
717}
718
719struct StdinWriter {
722 handle: Option<std::thread::JoinHandle<()>>,
723 receiver: std::sync::mpsc::Receiver<io::Result<()>>,
724}
725
726impl StdinWriter {
727 fn spawn(mut pipe: std::process::ChildStdin, bytes: Vec<u8>) -> Self {
728 let (sender, receiver) = std::sync::mpsc::channel();
729 let handle = std::thread::spawn(move || {
730 let result = pipe.write_all(&bytes);
731 drop(pipe);
733 let _ = sender.send(result);
734 });
735 Self {
736 handle: Some(handle),
737 receiver,
738 }
739 }
740
741 fn try_collect(
742 &mut self,
743 grace: Option<Duration>,
744 pid: u32,
745 label: &'static str,
746 ) -> WriteState {
747 use std::sync::mpsc::RecvTimeoutError;
748 let received = match grace {
749 Some(grace) => self.receiver.recv_timeout(grace),
750 None => self
751 .receiver
752 .recv()
753 .map_err(|_| RecvTimeoutError::Disconnected),
754 };
755 match received {
756 Ok(result) => {
757 if let Some(handle) = self.handle.take() {
758 let _ = handle.join();
759 }
760 WriteState::Finished(result)
761 }
762 Err(RecvTimeoutError::Timeout | RecvTimeoutError::Disconnected) => {
763 if let Some(handle) = self.handle.take() {
764 reap_wedged_pipe_thread(pid, label, handle);
765 }
766 WriteState::Wedged
767 }
768 }
769 }
770}
771
772fn remaining_until(deadline: Option<Instant>) -> Option<Duration> {
778 deadline.map(|deadline| deadline.saturating_duration_since(Instant::now()))
779}
780
781fn reap_wedged_pipe_thread(pid: u32, label: &'static str, handle: std::thread::JoinHandle<()>) {
791 tracing::warn!(
806 pid,
807 label,
808 "reviewer pipe helper thread still blocked after its grace window (a descendant likely \
809 escaped the process-tree kill and still holds the pipe open); not spawning a reaper for \
810 it, since that reaper's own join could block forever too — leaving it to exit on its own"
811 );
812 drop(handle);
813}
814
815const CHILD_POLL_INTERVAL: Duration = Duration::from_millis(100);
819
820const KILL_REAP_GRACE: Duration = Duration::from_secs(1);
824
825const PIPE_DRAIN_GRACE: Duration = Duration::from_secs(5);
830
831fn wait_for_child(
840 mut child: std::process::Child,
841 deadline: Option<(Instant, Duration)>,
842) -> Result<std::process::ExitStatus, ReviewerError> {
843 let Some((deadline, limit)) = deadline else {
844 return child.wait().map_err(ReviewerError::Wait);
845 };
846 loop {
847 if Instant::now() >= deadline {
857 if let Some(status) = child.try_wait().map_err(ReviewerError::Wait)? {
870 return Ok(status);
871 }
872 return kill_and_reap(child, limit);
873 }
874 if let Some(status) = child.try_wait().map_err(ReviewerError::Wait)? {
875 return Ok(status);
876 }
877 std::thread::sleep(CHILD_POLL_INTERVAL);
878 }
879}
880
881fn kill_and_reap(
886 child: std::process::Child,
887 limit: Duration,
888) -> Result<std::process::ExitStatus, ReviewerError> {
889 let mut child = child;
890 kill_process_tree(&mut child)?;
891 let status = match wait_for_reap(&mut child, KILL_REAP_GRACE) {
892 Ok(status) => status,
893 Err(error) => return reap_after_grace(child, error, limit.as_secs()),
894 };
895 outcome_after_kill(status, limit.as_secs())
896}
897
898fn reap_after_grace(
899 mut child: std::process::Child,
900 grace_error: io::Error,
901 timeout_secs: u64,
902) -> Result<std::process::ExitStatus, ReviewerError> {
903 let pid = child.id();
904 tracing::warn!(
905 pid,
906 %grace_error,
907 "reviewer kill grace expired; handing direct-child reap to background cleanup"
908 );
909 match child.kill() {
910 Ok(()) => {}
911 Err(error) if error.kind() == io::ErrorKind::InvalidInput => {}
912 Err(source) => {
913 return Err(ReviewerError::KillReviewer {
914 pid,
915 tree_error: "direct-child reap fallback".to_owned(),
916 source,
917 });
918 }
919 }
920 if let Some(status) = child.try_wait().map_err(ReviewerError::Wait)? {
921 return outcome_after_kill(status, timeout_secs);
922 }
923
924 let reaper = std::thread::Builder::new()
925 .name(format!("truth-mirror-reaper-{pid}"))
926 .spawn(move || {
927 if let Err(error) = child.wait() {
928 tracing::debug!(pid, %error, "background reviewer reaper stopped");
929 }
930 });
931 if let Err(error) = reaper {
932 tracing::warn!(pid, %error, "could not spawn background reviewer reaper");
933 }
934 Err(ReviewerError::ReviewerTimeout { timeout_secs })
935}
936
937fn wait_for_reap(
938 child: &mut std::process::Child,
939 grace: Duration,
940) -> io::Result<std::process::ExitStatus> {
941 let started = Instant::now();
942 loop {
943 if let Some(status) = child.try_wait()? {
944 return Ok(status);
945 }
946 if started.elapsed() >= grace {
947 return Err(io::Error::other(format!(
948 "killed child did not exit within {grace:?} (pid={})",
949 child.id()
950 )));
951 }
952 std::thread::sleep(CHILD_POLL_INTERVAL);
953 }
954}
955
956fn outcome_after_kill(
970 status: std::process::ExitStatus,
971 timeout_secs: u64,
972) -> Result<std::process::ExitStatus, ReviewerError> {
973 tracing::debug!(
974 ?status,
975 timeout_secs,
976 "reviewer reaped after deadline kill; reporting timeout regardless of reap outcome"
977 );
978 Err(ReviewerError::ReviewerTimeout { timeout_secs })
979}
980
981fn kill_process_tree(child: &mut std::process::Child) -> Result<(), ReviewerError> {
998 let pid = child.id();
999 if let Err(tree_error) = platform_tree_kill(pid) {
1000 match child.kill() {
1004 Ok(()) => {}
1005 Err(direct_error) => {
1006 if direct_error.kind() != io::ErrorKind::InvalidInput {
1010 return Err(ReviewerError::KillReviewer {
1011 pid,
1012 tree_error: tree_error.to_string(),
1013 source: direct_error,
1014 });
1015 }
1016 }
1017 }
1018 }
1019 Ok(())
1020}
1021
1022#[cfg(unix)]
1023fn platform_tree_kill(pid: u32) -> io::Result<()> {
1024 let Some(group) = rustix::process::Pid::from_raw(pid as i32) else {
1030 return Err(io::Error::new(
1031 io::ErrorKind::InvalidInput,
1032 format!("invalid reviewer process group pid: {pid}"),
1033 ));
1034 };
1035 rustix::process::kill_process_group(group, rustix::process::Signal::KILL)
1036 .map_err(io::Error::from)
1037}
1038
1039#[cfg(windows)]
1040fn platform_tree_kill(pid: u32) -> io::Result<()> {
1041 let status = Command::new("taskkill")
1043 .args(["/F", "/T", "/PID", &pid.to_string()])
1044 .stdin(Stdio::null())
1045 .stdout(Stdio::null())
1046 .stderr(Stdio::null())
1047 .status()?;
1048 if status.success() {
1049 Ok(())
1050 } else {
1051 Err(io::Error::other(format!(
1052 "taskkill /F /T /PID {pid} exited with {status}"
1053 )))
1054 }
1055}
1056
1057#[cfg(not(any(unix, windows)))]
1058fn platform_tree_kill(_pid: u32) -> io::Result<()> {
1059 Err(io::Error::other(
1062 "process-tree kill is not supported on this platform",
1063 ))
1064}
1065
1066#[derive(Clone, Debug, Eq, PartialEq)]
1067pub struct ReviewJob {
1068 pub commit_sha: String,
1069 pub claim: Claim,
1070 pub diff: String,
1071 pub context: String,
1073 pub request: ReviewRequest,
1074 pub strict: Option<StrictReviewConfig>,
1075 pub petition: Option<PetitionContext>,
1082}
1083
1084#[derive(Clone, Debug, Eq, PartialEq)]
1086pub struct PetitionContext {
1087 pub original_sha: String,
1088 pub fix_sha: String,
1089 pub original_claim: String,
1090 pub original_summary: String,
1091 pub original_findings: Vec<String>,
1092 pub original_structured_findings: Vec<StructuredFinding>,
1093 pub original_reviewer_model: String,
1094 pub attempts_so_far: u32,
1095}
1096
1097#[derive(Clone, Debug, Eq, PartialEq)]
1098pub struct StrictReviewConfig {
1099 pub arbiter_harness: ReviewerHarness,
1100 pub arbiter_model: String,
1101 pub arbiter_effort: Effort,
1102}
1103
1104#[derive(Clone, Debug, Eq, PartialEq)]
1105pub struct ReviewExecution {
1106 pub entries: Vec<LedgerEntry>,
1107}
1108
1109fn materialize_petition_prompt(mut job: ReviewJob) -> ReviewJob {
1114 let Some(petition) = &job.petition else {
1115 return job;
1116 };
1117 job.request.prompt = petition_prompt(petition, &job.claim, &job.diff, &job.context);
1118 job
1119}
1120
1121pub fn execute_review_job<R: ProcessRunner>(
1122 job: ReviewJob,
1123 runner: &R,
1124 store: &LedgerStore,
1125 timeout: Option<Duration>,
1126) -> Result<ReviewExecution, ReviewerError> {
1127 let job = materialize_petition_prompt(job);
1128 let execution = draft_review_job(&job, runner, timeout)?;
1129 apply_review_execution(&job, &execution, store)?;
1130 Ok(execution)
1131}
1132
1133fn strict_pass_needed(job: &ReviewJob, first_verdict: &ParsedVerdict) -> bool {
1134 job.strict.is_some()
1135 && first_verdict.verdict == Verdict::Pass
1136 && first_verdict.findings.is_empty()
1137}
1138
1139fn draft_review_first_pass_in_dir<R: ProcessRunner>(
1140 job: &ReviewJob,
1141 runner: &R,
1142 timeout: Option<Duration>,
1143 current_dir: Option<&Path>,
1144) -> Result<(LedgerEntry, String), ReviewerError> {
1145 let first_plan = ReviewPlan::build(job.request.clone())?;
1146 let first_output =
1147 first_plan.run_with_dir(&job.request.prompt, runner, timeout, current_dir)?;
1148 ensure_process_success(&first_output)?;
1149 let first_verdict = ParsedVerdict::parse(&first_output.stdout)?;
1150 let first_entry = entry_from_verdict(job, &first_plan, &first_verdict);
1151 Ok((first_entry, first_output.stdout))
1152}
1153
1154fn draft_review_strict_pass_in_dir<R: ProcessRunner>(
1155 job: &ReviewJob,
1156 runner: &R,
1157 first_stdout: &str,
1158 timeout: Option<Duration>,
1159 current_dir: Option<&Path>,
1160) -> Result<LedgerEntry, ReviewerError> {
1161 let strict = job
1162 .strict
1163 .as_ref()
1164 .expect("strict pass requested without strict config");
1165 validate_strict_arbiter(&job.request, strict)?;
1166 let strict_prompt = strict_second_pass_prompt(job, first_stdout);
1167 let strict_request = ReviewRequest::new(
1168 job.request.watched_agent,
1169 job.request.watched_model.clone(),
1170 strict.arbiter_harness,
1171 strict.arbiter_model.clone(),
1172 false,
1173 strict_prompt,
1174 )
1175 .with_effort(strict.arbiter_effort);
1176 let strict_plan = ReviewPlan::build(strict_request.clone())?;
1177 let strict_output =
1178 strict_plan.run_with_dir(&strict_request.prompt, runner, timeout, current_dir)?;
1179 ensure_process_success(&strict_output)?;
1180 let strict_verdict = ParsedVerdict::parse(&strict_output.stdout)?;
1181 Ok(entry_from_verdict(job, &strict_plan, &strict_verdict))
1182}
1183
1184pub(crate) fn draft_review_job<R: ProcessRunner>(
1185 job: &ReviewJob,
1186 runner: &R,
1187 timeout: Option<Duration>,
1188) -> Result<ReviewExecution, ReviewerError> {
1189 draft_review_job_in_dir(job, runner, timeout, None)
1190}
1191
1192fn draft_review_job_in_dir<R: ProcessRunner>(
1193 job: &ReviewJob,
1194 runner: &R,
1195 timeout: Option<Duration>,
1196 current_dir: Option<&Path>,
1197) -> Result<ReviewExecution, ReviewerError> {
1198 let (first_entry, first_stdout) =
1199 draft_review_first_pass_in_dir(job, runner, timeout, current_dir)?;
1200 let first_verdict = ParsedVerdict::parse(&first_stdout)?;
1201 let mut entries = vec![first_entry];
1202 if strict_pass_needed(job, &first_verdict) {
1203 entries.push(draft_review_strict_pass_in_dir(
1204 job,
1205 runner,
1206 &first_stdout,
1207 timeout,
1208 current_dir,
1209 )?);
1210 }
1211 Ok(ReviewExecution { entries })
1212}
1213
1214fn ledger_entry_canonical_line(entry: &LedgerEntry) -> Result<String, ReviewerError> {
1215 Ok(
1216 String::from_utf8_lossy(&serde_json::to_vec(entry).map_err(ReviewerError::RunJson)?)
1217 .into_owned(),
1218 )
1219}
1220
1221fn ledger_contains_exact_entry(
1222 store: &LedgerStore,
1223 entry: &LedgerEntry,
1224) -> Result<bool, ReviewerError> {
1225 let needle = ledger_entry_canonical_line(entry)?;
1226 Ok(store
1227 .read_history()
1228 .map_err(ReviewerError::Ledger)?
1229 .iter()
1230 .any(|existing| {
1231 ledger_entry_canonical_line(existing)
1232 .ok()
1233 .is_some_and(|line| line == needle)
1234 }))
1235}
1236
1237fn append_entry_idempotent(store: &LedgerStore, entry: &LedgerEntry) -> Result<(), ReviewerError> {
1238 if ledger_contains_exact_entry(store, entry)? {
1239 return Ok(());
1240 }
1241 store.append_entry(entry).map_err(ReviewerError::Ledger)
1242}
1243
1244fn petition_provenance_matches(
1245 original: &LedgerEntry,
1246 petition: &PetitionContext,
1247 attempts: u32,
1248) -> bool {
1249 if original.petition_attempts != attempts {
1250 return false;
1251 }
1252 if let Some(provenance) = &original.transition_provenance {
1253 return provenance.kind == crate::ledger::TransitionProvenanceKind::PetitionReview
1254 && provenance.fix_sha == petition.fix_sha
1255 && provenance.original_sha == petition.original_sha
1256 && provenance.attempts == attempts;
1257 }
1258 original.resolution.as_ref().is_some_and(|resolution| {
1259 resolution.reason.contains("petition review of fix")
1260 && resolution.reason.contains(&petition.fix_sha)
1261 && resolution.reason.contains(&petition.original_sha)
1262 })
1263}
1264
1265fn reviewer_petition_open_transition_applied(
1266 original: &LedgerEntry,
1267 petition: &PetitionContext,
1268 attempts: u32,
1269) -> bool {
1270 original.is_unresolved_rejection() && petition_provenance_matches(original, petition, attempts)
1271}
1272
1273#[derive(Clone, Copy, Debug, Eq, PartialEq)]
1274enum ExpectedPetitionOutcome {
1275 Resolved,
1276 NeedsHuman,
1277 OpenAttempt,
1278}
1279
1280fn expected_petition_outcome(
1281 verdict_entry: &LedgerEntry,
1282) -> Result<ExpectedPetitionOutcome, ReviewerError> {
1283 let max_attempts = crate::resolve::MAX_PETITION_ATTEMPTS;
1284 if crate::resolve::petition_accepts(verdict_entry.verdict) {
1285 return Ok(ExpectedPetitionOutcome::Resolved);
1286 }
1287 if verdict_entry.petition_attempts >= max_attempts {
1288 return Ok(ExpectedPetitionOutcome::NeedsHuman);
1289 }
1290 Ok(ExpectedPetitionOutcome::OpenAttempt)
1291}
1292
1293fn petition_outcome_already_applied(
1294 original: &LedgerEntry,
1295 petition: &PetitionContext,
1296 verdict_entry: &LedgerEntry,
1297) -> Result<bool, ReviewerError> {
1298 let expected = expected_petition_outcome(verdict_entry)?;
1299 match expected {
1300 ExpectedPetitionOutcome::Resolved => Ok(original.disposition == Disposition::Resolved
1301 && petition_provenance_matches(original, petition, verdict_entry.petition_attempts)),
1302 ExpectedPetitionOutcome::NeedsHuman => Ok(original.is_needs_human()
1303 && petition_provenance_matches(original, petition, verdict_entry.petition_attempts)),
1304 ExpectedPetitionOutcome::OpenAttempt => Ok(reviewer_petition_open_transition_applied(
1305 original,
1306 petition,
1307 verdict_entry.petition_attempts,
1308 )),
1309 }
1310}
1311
1312fn apply_petition_transition_idempotent(
1313 job: &ReviewJob,
1314 entry: &LedgerEntry,
1315 store: &LedgerStore,
1316) -> Result<(), ReviewerError> {
1317 let Some(petition) = &job.petition else {
1318 return Ok(());
1319 };
1320 apply_petition_context_transition_idempotent(petition, entry, store)
1321}
1322
1323fn apply_petition_context_transition_idempotent(
1324 petition: &PetitionContext,
1325 entry: &LedgerEntry,
1326 store: &LedgerStore,
1327) -> Result<(), ReviewerError> {
1328 let original = store
1329 .show(&petition.original_sha)
1330 .map_err(ReviewerError::Ledger)?;
1331 if petition_outcome_already_applied(&original, petition, entry)? {
1332 return Ok(());
1333 }
1334 if original.is_unresolved_rejection() || original.is_needs_human() {
1335 return apply_petition_context_transition(petition, entry, store);
1336 }
1337 Ok(())
1338}
1339
1340fn apply_review_execution(
1341 job: &ReviewJob,
1342 execution: &ReviewExecution,
1343 store: &LedgerStore,
1344) -> Result<(), ReviewerError> {
1345 for entry in &execution.entries {
1346 append_entry_idempotent(store, entry)?;
1347 }
1348 if let Some(last) = execution.entries.last() {
1349 apply_petition_transition_idempotent(job, last, store)?;
1350 }
1351 Ok(())
1352}
1353
1354fn apply_petition_batch_execution(
1355 petition: &PetitionContext,
1356 execution: &ReviewExecution,
1357 store: &LedgerStore,
1358) -> Result<(), ReviewerError> {
1359 for entry in &execution.entries {
1360 append_entry_idempotent(store, entry)?;
1361 }
1362 if let Some(last) = execution.entries.last() {
1363 apply_petition_context_transition_idempotent(petition, last, store)?;
1364 }
1365 Ok(())
1366}
1367
1368fn apply_petition_context_transition(
1373 petition: &PetitionContext,
1374 entry: &LedgerEntry,
1375 store: &LedgerStore,
1376) -> Result<(), ReviewerError> {
1377 let max_attempts = crate::resolve::MAX_PETITION_ATTEMPTS;
1378 let next_attempts = entry.petition_attempts;
1379 let reason = format!(
1380 "petition review of fix {} against rejection {} (attempts {})",
1381 petition.fix_sha, petition.original_sha, next_attempts
1382 );
1383 let provenance = Some(crate::ledger::TransitionProvenance {
1384 kind: crate::ledger::TransitionProvenanceKind::PetitionReview,
1385 fix_sha: petition.fix_sha.clone(),
1386 original_sha: petition.original_sha.clone(),
1387 attempts: next_attempts,
1388 });
1389
1390 if crate::resolve::petition_accepts(entry.verdict) {
1391 store
1394 .append_petition_transition_with_provenance(
1395 &petition.original_sha,
1396 crate::ledger::Disposition::Resolved,
1397 crate::ledger::ResolutionKind::Resolved,
1398 &reason,
1399 next_attempts,
1400 provenance.clone(),
1401 )
1402 .map_err(ReviewerError::Ledger)?;
1403 } else if next_attempts >= max_attempts {
1404 store
1405 .escalate_to_needs_human_with_attempts(
1406 &petition.original_sha,
1407 &reason,
1408 next_attempts,
1409 provenance.clone(),
1410 )
1411 .map_err(ReviewerError::Ledger)?;
1412 } else {
1413 store
1414 .append_petition_transition_with_provenance(
1415 &petition.original_sha,
1416 crate::ledger::Disposition::Open,
1417 crate::ledger::ResolutionKind::Resolved,
1418 &reason,
1419 next_attempts,
1420 provenance,
1421 )
1422 .map_err(ReviewerError::Ledger)?;
1423 }
1424 Ok(())
1425}
1426
1427fn petition_context_from_ledger(
1433 original_sha: &str,
1434 fix_sha: &str,
1435 store: &LedgerStore,
1436) -> Result<PetitionContext, ReviewerError> {
1437 let original = store.show(original_sha).map_err(ReviewerError::Ledger)?;
1438 let attempts_so_far = store
1439 .petition_attempts_for(original_sha)
1440 .map_err(ReviewerError::Ledger)?;
1441 Ok(PetitionContext {
1442 original_sha: original_sha.to_owned(),
1443 fix_sha: fix_sha.to_owned(),
1444 original_claim: original.claim.clone(),
1445 original_summary: original.summary.clone(),
1446 original_findings: original.findings.clone(),
1447 original_structured_findings: original.structured_findings.clone(),
1448 original_reviewer_model: original.reviewer.model.clone(),
1449 attempts_so_far,
1450 })
1451}
1452
1453fn petition_target_is_actionable(
1460 store: &LedgerStore,
1461 original_sha: &str,
1462) -> Result<bool, ReviewerError> {
1463 match store.show(original_sha) {
1464 Ok(entry) => Ok(entry.is_unresolved_rejection() || entry.is_needs_human()),
1465 Err(LedgerError::NotFound { .. }) => Ok(false),
1466 Err(error) => Err(ReviewerError::Ledger(error)),
1467 }
1468}
1469
1470#[derive(Clone, Debug, Eq, PartialEq)]
1471pub struct ParsedVerdict {
1472 pub verdict: Verdict,
1473 pub summary: String,
1474 pub findings: Vec<String>,
1475 pub structured_findings: Vec<StructuredFinding>,
1476 pub next_steps: Vec<String>,
1477 pub memory_skill_classification: MemorySkillClassification,
1478 pub raw: String,
1479}
1480
1481impl ParsedVerdict {
1482 pub fn parse(output: &str) -> Result<Self, ReviewerError> {
1483 let candidate = extract_verdict_json(output);
1486 let parsed: ReviewerJsonOutput =
1487 serde_json::from_str(candidate).map_err(|source| ReviewerError::VerdictJson {
1488 source,
1489 output: output.to_owned(),
1490 })?;
1491 Self::from_json(parsed, output.to_owned())
1492 }
1493
1494 fn from_json(mut parsed: ReviewerJsonOutput, raw: String) -> Result<Self, ReviewerError> {
1495 if let Err(error) = parsed.validate() {
1496 if !parsed.salvage_disallowed_pass_skill_proposal() {
1497 return Err(error);
1498 }
1499 tracing::warn!(
1500 verdict = %parsed.verdict,
1501 "reviewer returned a PASS verdict with a disallowed memory-skill proposal; stripped the proposal and accepted the verdict"
1502 );
1503 parsed.validate()?;
1504 }
1505 let findings = parsed
1506 .findings
1507 .iter()
1508 .map(StructuredFinding::display_line)
1509 .collect();
1510
1511 Ok(Self {
1512 verdict: parsed.verdict,
1513 summary: parsed.summary,
1514 findings,
1515 structured_findings: parsed.findings,
1516 next_steps: parsed.next_steps,
1517 memory_skill_classification: parsed.memory_skill,
1518 raw,
1519 })
1520 }
1521}
1522
1523#[derive(Clone, Debug, Eq, PartialEq)]
1524enum ParsedPetitionBatchMember {
1525 Valid(ParsedVerdict),
1526 Invalid(String),
1527}
1528
1529#[derive(Clone, Debug, Eq, PartialEq)]
1530struct ParsedPetitionBatch {
1531 members: std::collections::BTreeMap<String, ParsedPetitionBatchMember>,
1532 unknown_item_ids: Vec<String>,
1533}
1534
1535impl ParsedPetitionBatch {
1536 fn parse(
1537 output: &str,
1538 expected_batch_id: &str,
1539 expected_fix_sha: &str,
1540 requested_shas: &[String],
1541 ) -> Result<Self, ReviewerError> {
1542 let candidate = extract_verdict_json(output);
1543 let document: serde_json::Value =
1544 serde_json::from_str(candidate).map_err(|source| ReviewerError::VerdictJson {
1545 source,
1546 output: output.to_owned(),
1547 })?;
1548 let object = document
1549 .as_object()
1550 .ok_or_else(|| ReviewerError::BatchVerdictSchema {
1551 message: "batch verdict must be a JSON object".to_owned(),
1552 })?;
1553 let batch_id = object
1554 .get("batch_id")
1555 .and_then(serde_json::Value::as_str)
1556 .ok_or_else(|| ReviewerError::BatchVerdictSchema {
1557 message: "batch_id must be a string".to_owned(),
1558 })?;
1559 if batch_id != expected_batch_id {
1560 return Err(ReviewerError::BatchVerdictSchema {
1561 message: format!(
1562 "batch_id mismatch: expected {expected_batch_id:?}, got {batch_id:?}"
1563 ),
1564 });
1565 }
1566 let fix_sha = object
1567 .get("fix_sha")
1568 .and_then(serde_json::Value::as_str)
1569 .ok_or_else(|| ReviewerError::BatchVerdictSchema {
1570 message: "fix_sha must be a string".to_owned(),
1571 })?;
1572 if fix_sha != expected_fix_sha {
1573 return Err(ReviewerError::BatchVerdictSchema {
1574 message: format!(
1575 "fix_sha mismatch: expected {expected_fix_sha:?}, got {fix_sha:?}"
1576 ),
1577 });
1578 }
1579 let results = object
1580 .get("results")
1581 .and_then(serde_json::Value::as_array)
1582 .ok_or_else(|| ReviewerError::BatchVerdictSchema {
1583 message: "results must be an array".to_owned(),
1584 })?;
1585
1586 let requested: std::collections::BTreeSet<&str> =
1587 requested_shas.iter().map(String::as_str).collect();
1588 let mut candidates = std::collections::BTreeMap::<&str, Vec<&serde_json::Value>>::new();
1589 let mut unknown_item_ids = Vec::new();
1590 for result in results {
1591 match result
1592 .get("original_rejection_sha")
1593 .and_then(serde_json::Value::as_str)
1594 {
1595 Some(original_sha) if requested.contains(original_sha) => {
1596 candidates.entry(original_sha).or_default().push(result);
1597 }
1598 Some(original_sha) => unknown_item_ids.push(original_sha.to_owned()),
1599 None => unknown_item_ids.push("<malformed original_rejection_sha>".to_owned()),
1600 }
1601 }
1602
1603 let mut members = std::collections::BTreeMap::new();
1604 for requested_sha in requested_shas {
1605 let outcome = match candidates.get(requested_sha.as_str()).map(Vec::as_slice) {
1606 None | Some([]) => ParsedPetitionBatchMember::Invalid(format!(
1607 "missing batch result for original rejection {requested_sha}"
1608 )),
1609 Some([_, _, ..]) => ParsedPetitionBatchMember::Invalid(format!(
1610 "duplicate batch results for original rejection {requested_sha}"
1611 )),
1612 Some([result]) => {
1613 let raw = serde_json::to_string(result).map_err(ReviewerError::RunJson)?;
1614 match serde_json::from_value::<ReviewerJsonOutput>((*result).clone()) {
1615 Ok(parsed) => match ParsedVerdict::from_json(parsed, raw) {
1616 Ok(verdict) => ParsedPetitionBatchMember::Valid(verdict),
1617 Err(error) => ParsedPetitionBatchMember::Invalid(format!(
1618 "invalid batch result for original rejection {requested_sha}: {error}"
1619 )),
1620 },
1621 Err(error) => ParsedPetitionBatchMember::Invalid(format!(
1622 "invalid batch result for original rejection {requested_sha}: {error}"
1623 )),
1624 }
1625 }
1626 };
1627 members.insert(requested_sha.clone(), outcome);
1628 }
1629
1630 Ok(Self {
1631 members,
1632 unknown_item_ids,
1633 })
1634 }
1635}
1636
1637fn extract_verdict_json(output: &str) -> &str {
1656 fn parses(candidate: &str) -> bool {
1657 serde_json::from_str::<serde_json::Value>(candidate).is_ok()
1658 }
1659 fn json_prefix(text: &str) -> Option<(usize, serde_json::Value)> {
1662 let mut stream = serde_json::Deserializer::from_str(text).into_iter::<serde_json::Value>();
1663 match stream.next() {
1664 Some(Ok(value)) => Some((stream.byte_offset(), value)),
1665 _ => None,
1666 }
1667 }
1668 fn json_prefix_len(text: &str) -> Option<usize> {
1669 json_prefix(text).map(|(len, _)| len)
1670 }
1671 fn value_has_verdict(value: &serde_json::Value) -> bool {
1672 matches!(value, serde_json::Value::Object(map) if map.contains_key("verdict"))
1673 }
1674
1675 let trimmed = output.trim();
1676 if trimmed.starts_with('{') {
1677 if parses(trimmed) {
1678 return trimmed;
1679 }
1680 if let Some(end) = json_prefix_len(trimmed) {
1681 return trimmed[..end].trim();
1682 }
1683 }
1684
1685 let mut search_from = 0;
1690 let mut first_valid_fence: Option<&str> = None;
1691 while let Some(open_rel) = output[search_from..].find("```") {
1692 let after_open = search_from + open_rel + 3;
1693 let body_start = match output[after_open..].find('\n') {
1694 Some(newline_rel) => after_open + newline_rel + 1,
1695 None => break,
1696 };
1697 let Some(close_rel) = output[body_start..].find("```") else {
1698 break;
1699 };
1700 let body = output[body_start..body_start + close_rel].trim();
1701 if body.starts_with('{')
1702 && let Some((end, ref value)) = json_prefix(body)
1703 {
1704 let candidate = body[..end].trim();
1705 if value_has_verdict(value) {
1706 return candidate;
1707 }
1708 if first_valid_fence.is_none() {
1709 first_valid_fence = Some(candidate);
1710 }
1711 }
1712 search_from = body_start + close_rel + 3;
1713 }
1714
1715 let mut first_valid_brace: Option<&str> = None;
1719 for (brace_idx, _) in output.match_indices('{') {
1720 let tail = &output[brace_idx..];
1721 if let Some((end, value)) = json_prefix(tail) {
1722 let candidate = tail[..end].trim();
1723 if value_has_verdict(&value) {
1724 return candidate;
1725 }
1726 if first_valid_brace.is_none() {
1727 first_valid_brace = Some(candidate);
1728 }
1729 }
1730 }
1731
1732 first_valid_fence.or(first_valid_brace).unwrap_or(trimmed)
1735}
1736
1737#[derive(Clone, Debug, Deserialize, Eq, PartialEq)]
1738struct ReviewerJsonOutput {
1739 verdict: Verdict,
1740 summary: String,
1741 #[serde(default)]
1742 findings: Vec<StructuredFinding>,
1743 #[serde(default)]
1744 next_steps: Vec<String>,
1745 memory_skill: MemorySkillClassification,
1746}
1747
1748impl ReviewerJsonOutput {
1749 fn salvage_disallowed_pass_skill_proposal(&mut self) -> bool {
1753 if self.verdict != Verdict::Pass
1754 || !matches!(
1755 self.memory_skill.kind,
1756 MemorySkillClassificationKind::AntiPatternSkill
1757 | MemorySkillClassificationKind::RemediationSkill
1758 )
1759 {
1760 return false;
1761 }
1762
1763 self.memory_skill.kind = MemorySkillClassificationKind::None;
1764 self.memory_skill.learning_source.clear();
1765 true
1766 }
1767
1768 fn validate(&self) -> Result<(), ReviewerError> {
1769 if self.summary.trim().is_empty() {
1770 return Err(ReviewerError::VerdictSchema {
1771 message: "summary must not be empty".to_owned(),
1772 });
1773 }
1774 self.memory_skill
1775 .validate_for_verdict(self.verdict)
1776 .map_err(|message| ReviewerError::VerdictSchema { message })?;
1777
1778 for finding in &self.findings {
1779 if finding.title.trim().is_empty() {
1780 return Err(ReviewerError::VerdictSchema {
1781 message: "finding title must not be empty".to_owned(),
1782 });
1783 }
1784 if finding.body.trim().is_empty() {
1785 return Err(ReviewerError::VerdictSchema {
1786 message: "finding body must not be empty".to_owned(),
1787 });
1788 }
1789 if finding.file.trim().is_empty() {
1790 return Err(ReviewerError::VerdictSchema {
1791 message: "finding file must not be empty".to_owned(),
1792 });
1793 }
1794 if finding.line_start == 0 || finding.line_end == 0 {
1795 return Err(ReviewerError::VerdictSchema {
1796 message: "finding lines must be one-based".to_owned(),
1797 });
1798 }
1799 if finding.line_end < finding.line_start {
1800 return Err(ReviewerError::VerdictSchema {
1801 message: "finding line_end must be greater than or equal to line_start"
1802 .to_owned(),
1803 });
1804 }
1805 if finding.confidence > 100 {
1806 return Err(ReviewerError::VerdictSchema {
1807 message: "finding confidence must be between 0 and 100".to_owned(),
1808 });
1809 }
1810 if finding.recommendation.trim().is_empty() {
1811 return Err(ReviewerError::VerdictSchema {
1812 message: "finding recommendation must not be empty".to_owned(),
1813 });
1814 }
1815 }
1816
1817 if self.verdict == Verdict::Pass && !self.findings.is_empty() {
1818 return Err(ReviewerError::VerdictSchema {
1819 message: "PASS verdict must not include findings".to_owned(),
1820 });
1821 }
1822 if self.verdict == Verdict::Reject && self.findings.is_empty() {
1823 return Err(ReviewerError::VerdictSchema {
1824 message: "REJECT verdict must include at least one finding".to_owned(),
1825 });
1826 }
1827 if self.verdict == Verdict::Flag && self.findings.is_empty() {
1828 return Err(ReviewerError::VerdictSchema {
1829 message: "FLAG verdict must include at least one finding (the debt being surfaced, in the findings array)"
1830 .to_owned(),
1831 });
1832 }
1833
1834 Ok(())
1835 }
1836}
1837
1838#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
1840pub struct PendingReviewSpool {
1841 pub entries: Vec<LedgerEntry>,
1842 #[serde(default)]
1844 pub petition_transition_pending: bool,
1845 #[serde(default, skip_serializing_if = "Option::is_none")]
1847 pub strict_pass_stdout: Option<String>,
1848}
1849
1850#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
1851#[serde(rename_all = "kebab-case")]
1852pub enum ReviewRunStatus {
1853 Queued,
1854 Running,
1855 Completed,
1856 Failed,
1857 Cancelled,
1858}
1859
1860impl std::fmt::Display for ReviewRunStatus {
1861 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1862 match self {
1863 Self::Queued => formatter.write_str("queued"),
1864 Self::Running => formatter.write_str("running"),
1865 Self::Completed => formatter.write_str("completed"),
1866 Self::Failed => formatter.write_str("failed"),
1867 Self::Cancelled => formatter.write_str("cancelled"),
1868 }
1869 }
1870}
1871
1872#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
1873#[serde(rename_all = "kebab-case")]
1874pub enum BatchKind {
1875 Petition,
1876}
1877
1878#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
1879pub struct BatchRunRef {
1880 pub batch_id: String,
1881 pub kind: BatchKind,
1882 pub member_index: u32,
1883 pub member_count: u32,
1884 pub fix_sha: String,
1885}
1886#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
1888pub struct BatchReviewRecord {
1889 pub batch_id: String,
1890 pub fix_sha: String,
1891 pub phase: String,
1892 pub member_run_ids: Vec<String>,
1893 pub member_original_shas: Vec<String>,
1894 pub reviewer_harness: String,
1895 pub reviewer_model: String,
1896 pub raw_stdout: Option<String>,
1897 pub created_at_unix: u64,
1898 pub updated_at_unix: u64,
1899}
1900
1901#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
1902pub struct ReviewRun {
1903 pub id: String,
1904 pub commit_sha: String,
1905 pub target: String,
1906 pub status: ReviewRunStatus,
1907 pub phase: String,
1908 pub ledger_entries: usize,
1909 pub error: Option<String>,
1910 #[serde(default)]
1915 pub worker_pid: Option<u32>,
1916 #[serde(default, skip_serializing_if = "Option::is_none")]
1920 pub worker_start_token: Option<String>,
1921 pub created_at_unix: u64,
1922 pub updated_at_unix: u64,
1923 pub started_at_unix: Option<u64>,
1924 pub completed_at_unix: Option<u64>,
1925 #[serde(default, skip_serializing_if = "Option::is_none")]
1926 pub entire_checkpoint: Option<EntireCheckpointRef>,
1927 #[serde(default, skip_serializing_if = "Option::is_none")]
1929 pub petition_for: Option<String>,
1930 #[serde(default, skip_serializing_if = "Option::is_none")]
1933 pub ledger_applied_phase: Option<String>,
1934 #[serde(default, skip_serializing_if = "Option::is_none")]
1937 pub pending_review: Option<PendingReviewSpool>,
1938 #[serde(default, skip_serializing_if = "Option::is_none")]
1940 pub batch: Option<BatchRunRef>,
1941 #[serde(default, skip_serializing_if = "Option::is_none")]
1943 pub claimed_at_unix: Option<u64>,
1944 #[serde(default, skip_serializing_if = "Option::is_none")]
1946 pub next_attempt_at_unix: Option<u64>,
1947 #[serde(default, skip_serializing_if = "Option::is_none")]
1949 pub worktree_path: Option<String>,
1950}
1951
1952#[derive(Clone, Debug, Default, Eq, PartialEq)]
1953pub struct ReviewRunStatusCounts {
1954 pub queued: usize,
1955 pub running: usize,
1956 pub completed: usize,
1957 pub failed: usize,
1958 pub cancelled: usize,
1959 pub skipped_records: usize,
1960}
1961
1962#[derive(Clone, Debug, Default, Eq, PartialEq)]
1964pub struct SchedulerStatusSnapshot {
1965 pub owner_pid: Option<u32>,
1966 pub claimed: usize,
1967 pub inflight: usize,
1968 pub active_workers: usize,
1969 pub batches: usize,
1970}
1971
1972impl ReviewRunStatusCounts {
1973 fn add(&mut self, status: ReviewRunStatus) {
1974 match status {
1975 ReviewRunStatus::Queued => self.queued += 1,
1976 ReviewRunStatus::Running => self.running += 1,
1977 ReviewRunStatus::Completed => self.completed += 1,
1978 ReviewRunStatus::Failed => self.failed += 1,
1979 ReviewRunStatus::Cancelled => self.cancelled += 1,
1980 }
1981 }
1982}
1983
1984#[derive(Deserialize)]
1985struct ReviewRunStatusRecord {
1986 status: ReviewRunStatus,
1987}
1988
1989impl ReviewRun {
1990 #[cfg(test)]
1991 fn queued(
1992 id: impl Into<String>,
1993 commit_sha: impl Into<String>,
1994 target: impl Into<String>,
1995 ) -> Self {
1996 Self::queued_with_provenance(id, commit_sha, target, None, None)
1997 }
1998
1999 fn queued_with_provenance(
2000 id: impl Into<String>,
2001 commit_sha: impl Into<String>,
2002 target: impl Into<String>,
2003 entire_checkpoint: Option<EntireCheckpointRef>,
2004 petition_for: Option<String>,
2005 ) -> Self {
2006 let timestamp = unix_now();
2007 Self {
2008 id: id.into(),
2009 commit_sha: commit_sha.into(),
2010 target: target.into(),
2011 status: ReviewRunStatus::Queued,
2012 phase: "queued".to_owned(),
2013 ledger_entries: 0,
2014 error: None,
2015 worker_pid: None,
2016 worker_start_token: None,
2017 created_at_unix: timestamp,
2018 updated_at_unix: timestamp,
2019 started_at_unix: None,
2020 completed_at_unix: None,
2021 entire_checkpoint,
2022 petition_for,
2023 ledger_applied_phase: None,
2024 pending_review: None,
2025 batch: None,
2026 claimed_at_unix: None,
2027 next_attempt_at_unix: None,
2028 worktree_path: None,
2029 }
2030 }
2031
2032 fn clear_pending_review(&mut self) {
2033 self.pending_review = None;
2034 self.updated_at_unix = unix_now();
2035 }
2036
2037 fn mark_running(&mut self, phase: impl Into<String>) {
2038 let timestamp = unix_now();
2039 let identity = crate::watcher::ProcessIdentity::current();
2040 self.status = ReviewRunStatus::Running;
2041 self.phase = phase.into();
2042 self.error = None;
2043 self.worker_pid = Some(identity.pid);
2044 self.worker_start_token =
2045 (!identity.start_token.is_empty()).then_some(identity.start_token);
2046 self.updated_at_unix = timestamp;
2047 if self.started_at_unix.is_none() {
2048 self.started_at_unix = Some(timestamp);
2049 }
2050 self.completed_at_unix = None;
2051 self.next_attempt_at_unix = None;
2053 }
2054
2055 fn mark_claimed(&mut self) {
2057 let timestamp = unix_now();
2058 self.status = ReviewRunStatus::Running;
2059 self.phase = CLAIMED_PHASE.to_owned();
2060 self.error = None;
2061 self.worker_pid = Some(std::process::id());
2062 self.claimed_at_unix = Some(timestamp);
2063 self.next_attempt_at_unix = None;
2064 self.updated_at_unix = timestamp;
2065 self.started_at_unix = Some(timestamp);
2066 self.completed_at_unix = None;
2067 }
2068
2069 fn mark_provider_backoff(&mut self, error: impl Into<String>, backoff_secs: u64) {
2071 let timestamp = unix_now();
2072 self.status = ReviewRunStatus::Queued;
2073 self.phase = PROVIDER_BACKOFF_PHASE.to_owned();
2074 self.error = Some(error.into());
2075 self.worker_pid = None;
2076 self.claimed_at_unix = None;
2077 self.worktree_path = None;
2078 self.next_attempt_at_unix = Some(timestamp.saturating_add(backoff_secs));
2079 self.updated_at_unix = timestamp;
2080 self.completed_at_unix = None;
2081 }
2082
2083 fn release_stale_claim(&mut self, reason: impl Into<String>) {
2085 let timestamp = unix_now();
2086 self.status = ReviewRunStatus::Queued;
2087 self.phase = "queued".to_owned();
2088 self.error = Some(reason.into());
2089 self.worker_pid = None;
2090 self.claimed_at_unix = None;
2091 self.worktree_path = None;
2092 self.updated_at_unix = timestamp;
2093 self.completed_at_unix = None;
2094 }
2095
2096 fn is_claimed(&self) -> bool {
2097 self.status == ReviewRunStatus::Running && self.claimed_at_unix.is_some()
2098 }
2099
2100 fn is_inflight(&self) -> bool {
2101 self.status == ReviewRunStatus::Running
2102 && self.claimed_at_unix.is_some()
2103 && self.phase != CLAIMED_PHASE
2104 }
2105
2106 fn backoff_blocks_attempt(&self, now: u64) -> bool {
2107 self.next_attempt_at_unix
2108 .map(|eligible| eligible > now)
2109 .unwrap_or(false)
2110 }
2111
2112 fn mark_ledger_applied(&mut self, phase: impl Into<String>, ledger_entries: usize) {
2113 let timestamp = unix_now();
2114 self.ledger_applied_phase = Some(phase.into());
2115 self.ledger_entries = ledger_entries;
2116 self.updated_at_unix = timestamp;
2117 }
2118
2119 fn mark_completed(&mut self, ledger_entries: usize) {
2120 let timestamp = unix_now();
2121 self.status = ReviewRunStatus::Completed;
2122 self.phase = "completed".to_owned();
2123 self.ledger_entries = ledger_entries;
2124 self.error = None;
2125 self.worker_pid = None;
2126 self.worker_start_token = None;
2127 self.claimed_at_unix = None;
2128 self.next_attempt_at_unix = None;
2129 self.worktree_path = None;
2130 self.updated_at_unix = timestamp;
2131 self.completed_at_unix = Some(timestamp);
2132 }
2133
2134 fn mark_failed(&mut self, error: impl Into<String>) {
2135 let timestamp = unix_now();
2136 self.status = ReviewRunStatus::Failed;
2137 self.phase = "failed".to_owned();
2138 self.error = Some(error.into());
2139 self.worker_pid = None;
2140 self.worker_start_token = None;
2141 self.claimed_at_unix = None;
2142 self.next_attempt_at_unix = None;
2143 self.worktree_path = None;
2144 self.updated_at_unix = timestamp;
2145 self.completed_at_unix = Some(timestamp);
2146 }
2147
2148 fn mark_cancelled(&mut self) {
2149 let timestamp = unix_now();
2150 self.status = ReviewRunStatus::Cancelled;
2151 self.phase = "cancelled".to_owned();
2152 self.error = None;
2153 self.worker_pid = None;
2154 self.worker_start_token = None;
2155 self.claimed_at_unix = None;
2156 self.next_attempt_at_unix = None;
2157 self.worktree_path = None;
2158 self.updated_at_unix = timestamp;
2159 self.completed_at_unix = Some(timestamp);
2160 }
2161
2162 fn reconcile_liveness(
2174 &mut self,
2175 probe: impl Fn(&crate::watcher::ProcessIdentity) -> crate::watcher::WorkerLiveness,
2176 ) -> ReconcileOutcome {
2177 if self.status != ReviewRunStatus::Running {
2178 return ReconcileOutcome::Unchanged;
2179 }
2180 let Some(pid) = self.worker_pid else {
2181 return ReconcileOutcome::Unchanged;
2182 };
2183 let identity = crate::watcher::ProcessIdentity {
2184 pid,
2185 start_token: self.worker_start_token.clone().unwrap_or_default(),
2186 };
2187 match probe(&identity) {
2188 crate::watcher::WorkerLiveness::Dead => {
2189 self.mark_failed(stale_worker_reason(pid));
2190 ReconcileOutcome::Reaped
2191 }
2192 crate::watcher::WorkerLiveness::Alive => ReconcileOutcome::Unchanged,
2193 crate::watcher::WorkerLiveness::Unknown => ReconcileOutcome::ProbeUnknown,
2194 }
2195 }
2196}
2197
2198#[derive(Clone, Copy, Debug, Eq, PartialEq)]
2200pub enum ReconcileOutcome {
2201 Reaped,
2203 Unchanged,
2205 ProbeUnknown,
2208}
2209
2210fn stale_worker_reason(pid: u32) -> String {
2212 format!("worker process {pid} exited without recording a verdict (stale run)")
2213}
2214
2215fn kill_pid(pid: u32) -> Result<(), ReviewerError> {
2218 let status = Command::new("kill")
2219 .arg("-KILL")
2220 .arg(pid.to_string())
2221 .stdout(Stdio::null())
2222 .stderr(Stdio::null())
2223 .status()
2224 .map_err(ReviewerError::KillWorker)?;
2225 if status.success() || !crate::watcher::pid_is_alive(pid) {
2226 Ok(())
2227 } else {
2228 Err(ReviewerError::KillWorkerFailed { pid })
2229 }
2230}
2231
2232#[derive(Clone, Debug)]
2233pub struct ReviewRunStore {
2234 root: PathBuf,
2235}
2236
2237const RUN_STORE_LOCK_DIR: &str = "review-runs.lock";
2239const RUN_STORE_LOCK_OWNER_FILE: &str = "review-runs.lock.owner";
2240const RUN_STORE_LOCK_HEARTBEAT_FILE: &str = "review-runs.lock.heartbeat";
2241const RUN_STORE_LOCK_LEASE_PREFIX: &str = ".review-runs.lock.lease";
2242const RUN_STORE_LOCK_RECLAIM_FILE: &str = ".review-runs.lock.reclaim";
2245const RUN_STORE_LOCK_HEARTBEAT_INTERVAL: Duration = Duration::from_millis(100);
2250const RUN_STORE_LOCK_STALE_INTERVALS: u64 = 3;
2251const RUN_STORE_LOCK_WAIT: Duration = Duration::from_secs(10);
2255const RUN_STORE_LOCK_POLL: Duration = Duration::from_millis(25);
2256static RUN_STORE_LOCK_TOKEN_COUNTER: AtomicU64 = AtomicU64::new(0);
2257
2258#[derive(Clone, Debug, Deserialize, Serialize)]
2259struct RunStoreLockRecord {
2260 owner: crate::watcher::ProcessIdentity,
2261 acquisition_token: String,
2262}
2263
2264struct RunStoreLock {
2270 path: PathBuf,
2271 lease_path: PathBuf,
2272 owner: crate::watcher::ProcessIdentity,
2273 acquisition_token: String,
2274 heartbeat_stop: Arc<AtomicBool>,
2275 heartbeat: Option<std::thread::JoinHandle<()>>,
2276}
2277
2278impl Drop for RunStoreLock {
2279 fn drop(&mut self) {
2280 self.heartbeat_stop.store(true, Ordering::Release);
2281 if let Some(heartbeat) = self.heartbeat.take() {
2282 let _ = heartbeat.join();
2283 }
2284 release_run_store_lock(
2285 &self.path,
2286 &self.lease_path,
2287 &self.owner,
2288 &self.acquisition_token,
2289 );
2290 }
2291}
2292
2293#[derive(Clone, Copy, Debug, Eq, PartialEq)]
2294enum RunStoreLockLiveness {
2295 Alive,
2297 Dead,
2299 Unknown,
2302}
2303
2304fn run_store_lock_owner_path(path: &Path) -> PathBuf {
2305 path.join(RUN_STORE_LOCK_OWNER_FILE)
2306}
2307
2308fn run_store_lock_heartbeat_path(path: &Path) -> PathBuf {
2309 path.join(RUN_STORE_LOCK_HEARTBEAT_FILE)
2310}
2311
2312fn run_store_lock_is_owned_by_current_process(
2313 path: &Path,
2314 owner: &crate::watcher::ProcessIdentity,
2315 acquisition_token: &str,
2316) -> bool {
2317 let Ok(record) = read_run_store_lock_record(path) else {
2318 return false;
2319 };
2320 identity_matches_owner(&record.owner, owner) && record.acquisition_token == acquisition_token
2321}
2322
2323fn identity_matches_owner(
2324 recorded: &crate::watcher::ProcessIdentity,
2325 current: &crate::watcher::ProcessIdentity,
2326) -> bool {
2327 recorded.pid == current.pid
2328 && (recorded.start_token.is_empty()
2329 || current.start_token.is_empty()
2330 || recorded.start_token == current.start_token)
2331}
2332
2333fn decide_run_store_lock_liveness(
2334 recorded: &crate::watcher::ProcessIdentity,
2335) -> RunStoreLockLiveness {
2336 match crate::watcher::probe_worker_identity(recorded) {
2337 crate::watcher::WorkerLiveness::Alive => RunStoreLockLiveness::Alive,
2338 crate::watcher::WorkerLiveness::Dead => RunStoreLockLiveness::Dead,
2339 crate::watcher::WorkerLiveness::Unknown => RunStoreLockLiveness::Unknown,
2340 }
2341}
2342
2343fn lock_is_stale_by_heartbeat(path: &Path) -> bool {
2344 let heartbeat_path = if path.is_dir() {
2345 run_store_lock_heartbeat_path(path)
2346 } else {
2347 path.to_path_buf()
2348 };
2349 let Ok(metadata) = fs::metadata(heartbeat_path) else {
2350 return false;
2351 };
2352 let Ok(modified) = metadata.modified() else {
2353 return false;
2354 };
2355 match SystemTime::now().duration_since(modified) {
2356 Ok(age) => age >= RUN_STORE_LOCK_HEARTBEAT_INTERVAL * RUN_STORE_LOCK_STALE_INTERVALS as u32,
2357 Err(_) => false,
2358 }
2359}
2360
2361fn read_run_store_lock_record(path: &Path) -> Result<RunStoreLockRecord, io::Error> {
2362 let owner_path = if path.is_dir() {
2363 run_store_lock_owner_path(path)
2364 } else {
2365 path.to_path_buf()
2366 };
2367 let contents = fs::read_to_string(owner_path)?;
2368 if path.is_dir() {
2369 let owner = serde_json::from_str::<crate::watcher::ProcessIdentity>(&contents)
2370 .map_err(io::Error::other)?;
2371 Ok(RunStoreLockRecord {
2372 owner,
2373 acquisition_token: String::new(),
2374 })
2375 } else {
2376 serde_json::from_str::<RunStoreLockRecord>(&contents).map_err(io::Error::other)
2377 }
2378}
2379
2380#[cfg(test)]
2381fn write_run_store_lock_owner(
2382 lock_dir: &Path,
2383 owner: &crate::watcher::ProcessIdentity,
2384) -> Result<(), ReviewerError> {
2385 fs::create_dir_all(lock_dir).map_err(ReviewerError::RunIo)?;
2386 let bytes = serde_json::to_vec_pretty(owner).map_err(ReviewerError::RunJson)?;
2387 fs::write(run_store_lock_owner_path(lock_dir), bytes).map_err(ReviewerError::RunIo)?;
2388 fs::write(run_store_lock_heartbeat_path(lock_dir), b"heartbeat")
2389 .map_err(ReviewerError::RunIo)?;
2390 Ok(())
2391}
2392
2393fn run_store_lock_is_stale(path: &Path) -> bool {
2401 let Ok(record) = read_run_store_lock_record(path) else {
2402 return false;
2403 };
2404 matches!(
2405 decide_run_store_lock_liveness(&record.owner),
2406 RunStoreLockLiveness::Dead
2407 ) && lock_is_stale_by_heartbeat(path)
2408}
2409
2410fn new_run_store_lock_token() -> String {
2411 let nanos = SystemTime::now()
2412 .duration_since(UNIX_EPOCH)
2413 .map_or(0, |duration| duration.as_nanos());
2414 let counter = RUN_STORE_LOCK_TOKEN_COUNTER.fetch_add(1, Ordering::Relaxed);
2415 format!("{}-{nanos}-{counter}", std::process::id())
2416}
2417
2418fn write_run_store_lock_record(
2419 lease_path: &Path,
2420 record: &RunStoreLockRecord,
2421) -> Result<(), ReviewerError> {
2422 let bytes = serde_json::to_vec_pretty(record).map_err(ReviewerError::RunJson)?;
2423 let mut file = fs::OpenOptions::new()
2424 .write(true)
2425 .truncate(true)
2426 .open(lease_path)
2427 .map_err(ReviewerError::RunIo)?;
2428 file.write_all(&bytes).map_err(ReviewerError::RunIo)?;
2429 file.write_all(b"\n").map_err(ReviewerError::RunIo)?;
2430 file.sync_all().map_err(ReviewerError::RunIo)?;
2431 Ok(())
2432}
2433
2434fn refresh_run_store_lock_heartbeat(lease_path: &Path) -> io::Result<()> {
2435 let mut file = fs::OpenOptions::new().append(true).open(lease_path)?;
2436 file.write_all(b"\n")?;
2440 file.sync_data()
2441}
2442
2443fn spawn_run_store_lock_heartbeat(
2444 lease_path: PathBuf,
2445 stop: Arc<AtomicBool>,
2446) -> std::thread::JoinHandle<()> {
2447 std::thread::spawn(move || {
2448 while !stop.load(Ordering::Acquire) {
2449 std::thread::sleep(RUN_STORE_LOCK_HEARTBEAT_INTERVAL);
2450 if stop.load(Ordering::Acquire) {
2451 break;
2452 }
2453 if let Err(error) = refresh_run_store_lock_heartbeat(&lease_path) {
2454 tracing::debug!(%error, "run-store lock heartbeat refresh stopped");
2455 break;
2456 }
2457 }
2458 })
2459}
2460
2461struct RunStoreReclaimGate {
2479 #[allow(dead_code)]
2482 file: fs::File,
2483}
2484
2485fn acquire_run_store_reclaim_gate(root: &Path) -> io::Result<Option<RunStoreReclaimGate>> {
2486 let path = root.join(RUN_STORE_LOCK_RECLAIM_FILE);
2487 let file = fs::OpenOptions::new()
2488 .write(true)
2489 .create(true)
2490 .truncate(false)
2491 .open(&path)?;
2492 match file.try_lock() {
2493 Ok(()) => Ok(Some(RunStoreReclaimGate { file })),
2494 Err(fs::TryLockError::WouldBlock) => Ok(None),
2495 Err(fs::TryLockError::Error(error)) => Err(error),
2496 }
2497}
2498
2499fn release_run_store_lock(
2500 path: &Path,
2501 lease_path: &Path,
2502 owner: &crate::watcher::ProcessIdentity,
2503 acquisition_token: &str,
2504) {
2505 if run_store_lock_is_owned_by_current_process(path, owner, acquisition_token) {
2506 let _ = fs::remove_file(path);
2507 }
2508 let _ = fs::remove_file(lease_path);
2510}
2511
2512impl ReviewRunStore {
2513 pub fn new(root: impl Into<PathBuf>) -> Self {
2514 Self { root: root.into() }
2515 }
2516
2517 pub fn runs_dir(&self) -> PathBuf {
2518 self.root.join(REVIEW_RUNS_DIR)
2519 }
2520
2521 pub fn path(&self, id: &str) -> PathBuf {
2522 self.runs_dir().join(format!("{id}.json"))
2523 }
2524
2525 fn lock(&self) -> Result<RunStoreLock, ReviewerError> {
2532 fs::create_dir_all(&self.root).map_err(ReviewerError::RunIo)?;
2533 let path = self.root.join(RUN_STORE_LOCK_DIR);
2534 let owner = crate::watcher::ProcessIdentity::current();
2535 let started = Instant::now();
2536 loop {
2537 if path.exists() {
2541 if run_store_lock_is_stale(&path) {
2542 match acquire_run_store_reclaim_gate(&self.root) {
2547 Ok(Some(_gate)) => {
2548 if run_store_lock_is_stale(&path) {
2553 let remove_result = if path.is_dir() {
2554 fs::remove_dir_all(&path)
2555 } else {
2556 fs::remove_file(&path)
2557 };
2558 if let Err(error) = remove_result
2559 && error.kind() != io::ErrorKind::NotFound
2560 {
2561 return Err(ReviewerError::RunIo(error));
2562 }
2563 }
2564 }
2566 Ok(None) => {
2567 }
2570 Err(error) => return Err(ReviewerError::RunIo(error)),
2571 }
2572 std::thread::sleep(RUN_STORE_LOCK_POLL);
2573 continue;
2574 }
2575 if started.elapsed() >= RUN_STORE_LOCK_WAIT {
2576 if let Ok(record) = read_run_store_lock_record(&path)
2577 && matches!(
2578 decide_run_store_lock_liveness(&record.owner),
2579 RunStoreLockLiveness::Unknown
2580 )
2581 {
2582 tracing::warn!(
2583 "run-store lock owner remained Unknown; refusing automatic reclaim"
2584 );
2585 }
2586 return Err(ReviewerError::RunStoreLockTimeout {
2587 wait_secs: RUN_STORE_LOCK_WAIT.as_secs(),
2588 });
2589 }
2590 std::thread::sleep(RUN_STORE_LOCK_POLL);
2591 continue;
2592 }
2593
2594 let acquisition_token = new_run_store_lock_token();
2595 let lease_path = self
2596 .root
2597 .join(format!("{RUN_STORE_LOCK_LEASE_PREFIX}.{acquisition_token}"));
2598 let record = RunStoreLockRecord {
2599 owner: owner.clone(),
2600 acquisition_token: acquisition_token.clone(),
2601 };
2602 match fs::OpenOptions::new()
2603 .write(true)
2604 .create_new(true)
2605 .open(&lease_path)
2606 {
2607 Ok(_) => {
2608 if let Err(error) = write_run_store_lock_record(&lease_path, &record) {
2609 let _ = fs::remove_file(&lease_path);
2610 return Err(error);
2611 }
2612 match fs::hard_link(&lease_path, &path) {
2613 Ok(()) => {
2614 let heartbeat_stop = Arc::new(AtomicBool::new(false));
2615 let heartbeat = spawn_run_store_lock_heartbeat(
2616 lease_path.clone(),
2617 Arc::clone(&heartbeat_stop),
2618 );
2619 return Ok(RunStoreLock {
2620 path: path.clone(),
2621 lease_path,
2622 owner,
2623 acquisition_token,
2624 heartbeat_stop,
2625 heartbeat: Some(heartbeat),
2626 });
2627 }
2628 Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {
2629 let _ = fs::remove_file(&lease_path);
2630 }
2631 Err(error) => {
2632 let _ = fs::remove_file(&lease_path);
2633 return Err(ReviewerError::RunIo(error));
2634 }
2635 }
2636 }
2637 Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue,
2638 Err(error) => return Err(ReviewerError::RunIo(error)),
2639 }
2640
2641 if started.elapsed() >= RUN_STORE_LOCK_WAIT {
2642 if let Ok(record) = read_run_store_lock_record(&path)
2643 && matches!(
2644 decide_run_store_lock_liveness(&record.owner),
2645 RunStoreLockLiveness::Unknown
2646 )
2647 {
2648 tracing::warn!(
2649 "run-store lock owner remained Unknown; refusing automatic reclaim"
2650 );
2651 }
2652 return Err(ReviewerError::RunStoreLockTimeout {
2653 wait_secs: RUN_STORE_LOCK_WAIT.as_secs(),
2654 });
2655 }
2656 std::thread::sleep(RUN_STORE_LOCK_POLL);
2657 }
2658 }
2659
2660 pub fn create_queued(
2661 &self,
2662 commit_sha: &str,
2663 target: impl Into<String>,
2664 ) -> Result<ReviewRun, ReviewerError> {
2665 let _guard = self.lock()?;
2666 let checkpoint = entire_checkpoint_for_current_repo(commit_sha);
2667 let transaction = StateTransaction::acquire(&self.root)?;
2668 self.create_queued_locked(&transaction, commit_sha, target, checkpoint, None)
2669 }
2670
2671 fn create_queued_locked(
2672 &self,
2673 transaction: &StateTransaction,
2674 commit_sha: &str,
2675 target: impl Into<String>,
2676 checkpoint: Option<EntireCheckpointRef>,
2677 petition_for: Option<String>,
2678 ) -> Result<ReviewRun, ReviewerError> {
2679 let run = ReviewRun::queued_with_provenance(
2680 generate_run_id(commit_sha),
2681 commit_sha,
2682 target,
2683 checkpoint,
2684 petition_for,
2685 );
2686 self.write_locked(transaction, &run)?;
2687 Ok(run)
2688 }
2689
2690 fn ensure_queued(
2691 &self,
2692 run_id: &str,
2693 commit_sha: &str,
2694 target: &str,
2695 petition_for: Option<String>,
2696 ) -> Result<ReviewRun, ReviewerError> {
2697 match self.read(run_id) {
2698 Ok(run) => return Ok(run),
2699 Err(ReviewerError::ReviewRunNotFound { .. }) => {}
2700 Err(error) => return Err(error),
2701 }
2702
2703 let checkpoint = entire_checkpoint_for_current_repo(commit_sha);
2704 let transaction = StateTransaction::acquire(&self.root)?;
2705 self.ensure_queued_locked(
2706 &transaction,
2707 run_id,
2708 commit_sha,
2709 target,
2710 checkpoint,
2711 petition_for,
2712 )
2713 }
2714
2715 fn ensure_queued_locked(
2716 &self,
2717 transaction: &StateTransaction,
2718 run_id: &str,
2719 commit_sha: &str,
2720 target: &str,
2721 checkpoint: Option<EntireCheckpointRef>,
2722 petition_for: Option<String>,
2723 ) -> Result<ReviewRun, ReviewerError> {
2724 let _guard = self.lock()?;
2725 match self.read(run_id) {
2726 Ok(run) => Ok(run),
2727 Err(ReviewerError::ReviewRunNotFound { .. }) => {
2728 let run = ReviewRun::queued_with_provenance(
2729 run_id,
2730 commit_sha,
2731 target,
2732 checkpoint,
2733 petition_for,
2734 );
2735 self.write_locked(transaction, &run)?;
2736 Ok(run)
2737 }
2738 Err(error) => Err(error),
2739 }
2740 }
2741
2742 pub fn read(&self, id: &str) -> Result<ReviewRun, ReviewerError> {
2743 let path = self.path(id);
2744 let contents = fs::read_to_string(&path).map_err(|source| match source.kind() {
2745 io::ErrorKind::NotFound => ReviewerError::ReviewRunNotFound { id: id.to_owned() },
2746 _ => ReviewerError::RunIo(source),
2747 })?;
2748 serde_json::from_str(&contents).map_err(ReviewerError::RunJson)
2749 }
2750
2751 pub fn list(&self) -> Result<Vec<ReviewRun>, ReviewerError> {
2752 let dir = self.runs_dir();
2753 let entries = match fs::read_dir(&dir) {
2754 Ok(entries) => entries,
2755 Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(Vec::new()),
2756 Err(error) => return Err(ReviewerError::RunIo(error)),
2757 };
2758 let mut runs: Vec<ReviewRun> = Vec::new();
2759 for entry in entries {
2760 let entry = entry.map_err(ReviewerError::RunIo)?;
2761 if entry
2762 .path()
2763 .extension()
2764 .is_none_or(|extension| extension != "json")
2765 {
2766 continue;
2767 }
2768 let contents = fs::read_to_string(entry.path()).map_err(ReviewerError::RunIo)?;
2769 runs.push(serde_json::from_str(&contents).map_err(ReviewerError::RunJson)?);
2770 }
2771 runs.sort_by(|left, right| {
2772 right
2773 .updated_at_unix
2774 .cmp(&left.updated_at_unix)
2775 .then_with(|| right.id.cmp(&left.id))
2776 });
2777 Ok(runs)
2778 }
2779
2780 fn list_lenient(&self) -> Result<(Vec<ReviewRun>, usize), ReviewerError> {
2784 let dir = self.runs_dir();
2785 let entries = match fs::read_dir(&dir) {
2786 Ok(entries) => entries,
2787 Err(error)
2790 if error.kind() == io::ErrorKind::NotFound
2791 || error.kind() == io::ErrorKind::NotADirectory =>
2792 {
2793 return Ok((Vec::new(), 0));
2794 }
2795 Err(error) => return Err(ReviewerError::RunIo(error)),
2796 };
2797 let mut runs: Vec<ReviewRun> = Vec::new();
2798 let mut skipped_records = 0usize;
2799 for entry in entries {
2800 let Ok(entry) = entry else {
2801 skipped_records += 1;
2802 continue;
2803 };
2804 let path = entry.path();
2805 if path.extension().is_none_or(|extension| extension != "json") {
2806 continue;
2807 }
2808 let Ok(contents) = fs::read_to_string(&path) else {
2809 skipped_records += 1;
2810 continue;
2811 };
2812 let Ok(run) = serde_json::from_str::<ReviewRun>(&contents) else {
2813 skipped_records += 1;
2814 continue;
2815 };
2816 runs.push(run);
2817 }
2818 runs.sort_by(|left, right| {
2819 right
2820 .updated_at_unix
2821 .cmp(&left.updated_at_unix)
2822 .then_with(|| right.id.cmp(&left.id))
2823 });
2824 Ok((runs, skipped_records))
2825 }
2826
2827 pub fn status_counts(&self) -> Result<ReviewRunStatusCounts, ReviewerError> {
2828 let dir = self.runs_dir();
2829 let entries = match fs::read_dir(&dir) {
2830 Ok(entries) => entries,
2831 Err(error) if error.kind() == io::ErrorKind::NotFound => {
2832 return Ok(ReviewRunStatusCounts::default());
2833 }
2834 Err(error) => return Err(ReviewerError::RunIo(error)),
2835 };
2836 let mut counts = ReviewRunStatusCounts::default();
2837 for entry in entries {
2838 let Ok(entry) = entry else {
2839 counts.skipped_records += 1;
2840 continue;
2841 };
2842 let path = entry.path();
2843 if path.extension().is_none_or(|extension| extension != "json") {
2844 continue;
2845 }
2846 let Ok(contents) = fs::read_to_string(&path) else {
2847 counts.skipped_records += 1;
2848 continue;
2849 };
2850 let Ok(record) = serde_json::from_str::<ReviewRunStatusRecord>(&contents) else {
2851 counts.skipped_records += 1;
2852 continue;
2853 };
2854 counts.add(record.status);
2855 }
2856 Ok(counts)
2857 }
2858
2859 pub fn latest_result(&self) -> Result<ReviewRun, ReviewerError> {
2860 self.list()?
2861 .into_iter()
2862 .find(|run| {
2863 matches!(
2864 run.status,
2865 ReviewRunStatus::Completed
2866 | ReviewRunStatus::Failed
2867 | ReviewRunStatus::Cancelled
2868 )
2869 })
2870 .ok_or(ReviewerError::NoReviewRuns)
2871 }
2872
2873 pub fn mark_running(&self, id: &str, phase: &str) -> Result<ReviewRun, ReviewerError> {
2874 let transaction = StateTransaction::acquire(&self.root)?;
2875 let mut run = self.read(id)?;
2876 run.mark_running(phase);
2877 self.write_locked(&transaction, &run)?;
2878 Ok(run)
2879 }
2880
2881 pub fn mark_claimed(&self, id: &str) -> Result<ReviewRun, ReviewerError> {
2882 let transaction = StateTransaction::acquire(&self.root)?;
2883 let mut run = self.read(id)?;
2884 run.mark_claimed();
2885 self.write_locked(&transaction, &run)?;
2886 Ok(run)
2887 }
2888
2889 pub fn mark_provider_backoff(
2890 &self,
2891 id: &str,
2892 error: impl Into<String>,
2893 backoff_secs: u64,
2894 ) -> Result<ReviewRun, ReviewerError> {
2895 let transaction = StateTransaction::acquire(&self.root)?;
2896 let mut run = self.read(id)?;
2897 run.mark_provider_backoff(error, backoff_secs);
2898 self.write_locked(&transaction, &run)?;
2899 Ok(run)
2900 }
2901
2902 pub fn set_worktree_path(
2903 &self,
2904 id: &str,
2905 worktree_path: Option<String>,
2906 ) -> Result<ReviewRun, ReviewerError> {
2907 let transaction = StateTransaction::acquire(&self.root)?;
2908 let mut run = self.read(id)?;
2909 run.worktree_path = worktree_path;
2910 run.updated_at_unix = unix_now();
2911 self.write_locked(&transaction, &run)?;
2912 Ok(run)
2913 }
2914
2915 pub fn scheduler_status_read_only(
2917 &self,
2918 owner_pid: Option<u32>,
2919 queue_batch_ids: impl IntoIterator<Item = String>,
2920 ) -> Result<SchedulerStatusSnapshot, ReviewerError> {
2921 let (runs, _) = self.list_lenient()?;
2922 let mut claimed = 0usize;
2923 let mut inflight = 0usize;
2924 let mut worker_pids = std::collections::BTreeSet::new();
2925 let mut batches = std::collections::BTreeSet::new();
2926 for batch_id in queue_batch_ids {
2927 if !batch_id.trim().is_empty() {
2928 batches.insert(batch_id);
2929 }
2930 }
2931 for run in &runs {
2932 if run.is_claimed() {
2933 claimed += 1;
2934 }
2935 if run.is_inflight() {
2936 inflight += 1;
2937 }
2938 if run.status == ReviewRunStatus::Running
2939 && let Some(pid) = run.worker_pid
2940 && crate::watcher::pid_is_alive(pid)
2941 {
2942 worker_pids.insert(pid);
2943 }
2944 if let Some(batch) = &run.batch {
2945 batches.insert(batch.batch_id.clone());
2946 }
2947 }
2948 Ok(SchedulerStatusSnapshot {
2949 owner_pid,
2950 claimed,
2951 inflight,
2952 active_workers: worker_pids.len(),
2953 batches: batches.len(),
2954 })
2955 }
2956
2957 pub fn recover_stale_scheduler_claims(&self) -> Result<usize, ReviewerError> {
2959 let transaction = StateTransaction::acquire(&self.root)?;
2960 self.recover_stale_scheduler_claims_locked(&transaction)
2961 }
2962
2963 fn recover_stale_scheduler_claims_locked(
2964 &self,
2965 transaction: &StateTransaction,
2966 ) -> Result<usize, ReviewerError> {
2967 let (runs, _) = self.list_lenient()?;
2968 let mut recovered = 0usize;
2969 for run in runs {
2970 if run.status != ReviewRunStatus::Running {
2971 continue;
2972 }
2973 if run_ledger_phase_applied(&run, REVIEW_LEDGER_PHASE) {
2974 continue;
2975 }
2976 let Some(pid) = run.worker_pid else {
2977 continue;
2978 };
2979 if crate::watcher::pid_is_alive(pid) {
2980 continue;
2981 }
2982 let mut released = run;
2983 released.release_stale_claim(format!(
2984 "scheduler claim released after worker {pid} exited before durable apply"
2985 ));
2986 self.write_locked(transaction, &released)?;
2987 recovered += 1;
2988 }
2989 Ok(recovered)
2990 }
2991
2992 pub fn mark_ledger_applied(
2993 &self,
2994 id: &str,
2995 phase: &str,
2996 ledger_entries: usize,
2997 ) -> Result<ReviewRun, ReviewerError> {
2998 let transaction = StateTransaction::acquire(&self.root)?;
2999 let mut run = self.read(id)?;
3000 run.mark_ledger_applied(phase, ledger_entries);
3001 self.write_locked(&transaction, &run)?;
3002 Ok(run)
3003 }
3004
3005 pub fn persist_pending_review(
3006 &self,
3007 id: &str,
3008 spool: PendingReviewSpool,
3009 ) -> Result<ReviewRun, ReviewerError> {
3010 let transaction = StateTransaction::acquire(&self.root)?;
3011 let mut run = self.read(id)?;
3012 run.pending_review = Some(spool);
3013 run.updated_at_unix = unix_now();
3014 self.write_locked(&transaction, &run)?;
3015 Ok(run)
3016 }
3017
3018 pub fn clear_pending_review(&self, id: &str) -> Result<ReviewRun, ReviewerError> {
3019 let transaction = StateTransaction::acquire(&self.root)?;
3020 let mut run = self.read(id)?;
3021 run.clear_pending_review();
3022 self.write_locked(&transaction, &run)?;
3023 Ok(run)
3024 }
3025
3026 pub fn mark_completed(
3027 &self,
3028 id: &str,
3029 ledger_entries: usize,
3030 ) -> Result<ReviewRun, ReviewerError> {
3031 let transaction = StateTransaction::acquire(&self.root)?;
3032 let mut run = self.read(id)?;
3033 run.mark_completed(ledger_entries);
3034 self.write_locked(&transaction, &run)?;
3035 Ok(run)
3036 }
3037
3038 pub fn mark_failed(
3039 &self,
3040 id: &str,
3041 error: impl Into<String>,
3042 ) -> Result<ReviewRun, ReviewerError> {
3043 let transaction = StateTransaction::acquire(&self.root)?;
3044 self.mark_failed_locked(&transaction, id, error)
3045 }
3046
3047 fn mark_failed_locked(
3048 &self,
3049 transaction: &StateTransaction,
3050 id: &str,
3051 error: impl Into<String>,
3052 ) -> Result<ReviewRun, ReviewerError> {
3053 let mut run = self.read(id)?;
3054 run.mark_failed(error);
3055 self.write_locked(transaction, &run)?;
3056 Ok(run)
3057 }
3058 pub fn set_batch_metadata(
3059 &self,
3060 id: &str,
3061 batch: BatchRunRef,
3062 ) -> Result<ReviewRun, ReviewerError> {
3063 let transaction = StateTransaction::acquire(&self.root)?;
3064 let mut run = self.read(id)?;
3065 run.batch = Some(batch);
3066 self.write_locked(&transaction, &run)?;
3067 Ok(run)
3068 }
3069
3070 pub fn cancel_queued(&self, id: &str) -> Result<ReviewRun, ReviewerError> {
3071 let transaction = StateTransaction::acquire(&self.root)?;
3072 self.cancel_queued_locked(&transaction, id)
3073 }
3074
3075 fn cancel_queued_locked(
3076 &self,
3077 transaction: &StateTransaction,
3078 id: &str,
3079 ) -> Result<ReviewRun, ReviewerError> {
3080 let mut run = self.read(id)?;
3081 if run.status != ReviewRunStatus::Queued {
3082 return Err(ReviewerError::CannotCancelReview {
3083 id: id.to_owned(),
3084 status: run.status,
3085 });
3086 }
3087 run.mark_cancelled();
3088 self.write_locked(transaction, &run)?;
3089 Ok(run)
3090 }
3091
3092 pub fn read_reconciled(&self, id: &str) -> Result<ReviewRun, ReviewerError> {
3099 let transaction = StateTransaction::acquire(&self.root)?;
3100 let mut run = self.read(id)?;
3101 match run.reconcile_liveness(crate::watcher::probe_worker_identity) {
3102 ReconcileOutcome::Reaped => self.write_locked(&transaction, &run)?,
3103 ReconcileOutcome::ProbeUnknown => log_inconclusive_probe(&run),
3104 ReconcileOutcome::Unchanged => {}
3105 }
3106 Ok(run)
3107 }
3108
3109 pub fn list_reconciled(&self) -> Result<Vec<ReviewRun>, ReviewerError> {
3112 let transaction = StateTransaction::acquire(&self.root)?;
3113 let mut runs = self.list()?;
3114 for run in &mut runs {
3115 match run.reconcile_liveness(crate::watcher::probe_worker_identity) {
3116 ReconcileOutcome::Reaped => self.write_locked(&transaction, run)?,
3117 ReconcileOutcome::ProbeUnknown => log_inconclusive_probe(run),
3118 ReconcileOutcome::Unchanged => {}
3119 }
3120 }
3121 Ok(runs)
3122 }
3123
3124 pub fn reconcile_stale_runs(&self) -> Result<usize, ReviewerError> {
3137 let transaction = StateTransaction::acquire(&self.root)?;
3138 let mut reaped = 0;
3139 for mut run in self.list()? {
3140 match run.reconcile_liveness(crate::watcher::probe_worker_identity) {
3141 ReconcileOutcome::Reaped => {
3142 self.write_locked(&transaction, &run)?;
3143 reaped += 1;
3144 }
3145 ReconcileOutcome::ProbeUnknown => log_inconclusive_probe(&run),
3146 ReconcileOutcome::Unchanged => {}
3147 }
3148 }
3149 Ok(reaped)
3150 }
3151
3152 pub fn stale_running_count(&self) -> Result<usize, ReviewerError> {
3158 let mut stale = 0;
3159 for mut run in self.list()? {
3160 if run.reconcile_liveness(crate::watcher::probe_worker_identity)
3161 == ReconcileOutcome::Reaped
3162 {
3163 stale += 1;
3164 }
3165 }
3166 Ok(stale)
3167 }
3168
3169 pub fn cancel(&self, id: &str, force: bool) -> Result<ReviewRun, ReviewerError> {
3181 let transaction = StateTransaction::acquire(&self.root)?;
3182 let mut run = self.read(id)?;
3183 match run.status {
3184 ReviewRunStatus::Queued => run.mark_cancelled(),
3185 ReviewRunStatus::Running => match run.worker_pid {
3186 Some(pid) => {
3187 let identity = crate::watcher::ProcessIdentity {
3188 pid,
3189 start_token: run.worker_start_token.clone().unwrap_or_default(),
3190 };
3191 match crate::watcher::probe_worker_identity(&identity) {
3192 crate::watcher::WorkerLiveness::Alive => {
3193 if !force {
3194 return Err(ReviewerError::ReviewRunStillAlive {
3195 id: id.to_owned(),
3196 pid,
3197 });
3198 }
3199 kill_pid(pid)?;
3200 run.mark_cancelled();
3201 }
3202 crate::watcher::WorkerLiveness::Dead => {
3203 run.mark_failed(stale_worker_reason(pid));
3204 }
3205 crate::watcher::WorkerLiveness::Unknown => {
3206 if !force {
3207 return Err(ReviewerError::ReviewRunLivenessUnknown {
3208 id: id.to_owned(),
3209 });
3210 }
3211 run.mark_failed(
3212 "worker liveness could not be verified; force-cancelled",
3213 );
3214 }
3215 }
3216 }
3217 None => {
3218 if !force {
3219 return Err(ReviewerError::ReviewRunLivenessUnknown { id: id.to_owned() });
3220 }
3221 run.mark_failed("worker liveness could not be verified; force-cancelled");
3222 }
3223 },
3224 terminal => {
3225 return Err(ReviewerError::CannotCancelReview {
3226 id: id.to_owned(),
3227 status: terminal,
3228 });
3229 }
3230 }
3231 self.write_locked(&transaction, &run)?;
3232 Ok(run)
3233 }
3234
3235 #[cfg(test)]
3237 fn write(&self, run: &ReviewRun) -> Result<(), ReviewerError> {
3238 let transaction = StateTransaction::acquire(&self.root)?;
3239 self.write_locked(&transaction, run)
3240 }
3241
3242 fn write_locked(
3243 &self,
3244 _transaction: &StateTransaction,
3245 run: &ReviewRun,
3246 ) -> Result<(), ReviewerError> {
3247 let bytes = serde_json::to_vec_pretty(run).map_err(ReviewerError::RunJson)?;
3248 atomic_replace(&self.path(&run.id), &bytes)?;
3249 Ok(())
3250 }
3251}
3252
3253fn log_inconclusive_probe(run: &ReviewRun) {
3256 tracing::warn!(
3257 run_id = %run.id,
3258 worker_pid = run.worker_pid,
3259 "worker liveness probe inconclusive; leaving the running row untouched (a probe failure is not proof of death)"
3260 );
3261}
3262
3263fn entire_checkpoint_for_current_repo(commit_sha: &str) -> Option<EntireCheckpointRef> {
3264 let repo_root = current_repo_root().unwrap_or_else(|| PathBuf::from("."));
3265 provenance::entire_checkpoint_for_commit(&repo_root, commit_sha)
3266}
3267
3268fn current_repo_root() -> Option<PathBuf> {
3269 let output = Command::new("git")
3270 .args(["rev-parse", "--show-toplevel"])
3271 .output()
3272 .ok()?;
3273 output
3274 .status
3275 .success()
3276 .then(|| PathBuf::from(String::from_utf8_lossy(&output.stdout).trim()))
3277}
3278
3279#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
3280pub struct QueuedReview {
3281 #[serde(default)]
3282 pub run_id: String,
3283 pub commit_sha: String,
3284 pub enqueued_at_unix: u64,
3285 #[serde(default, skip_serializing_if = "Option::is_none")]
3290 pub petition_for: Option<String>,
3291 #[serde(default, skip_serializing_if = "Option::is_none")]
3294 pub batch_id: Option<String>,
3295}
3296
3297#[derive(Clone, Debug)]
3298pub struct ReviewQueue {
3299 root: PathBuf,
3300}
3301
3302#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
3303struct PetitionFailureMarker {
3304 run_id: String,
3305 fix_sha: String,
3306 original_sha: String,
3307 attempts: u32,
3308 error: String,
3309}
3310
3311fn read_petition_failure_markers(root: &Path) -> Result<Vec<PetitionFailureMarker>, ReviewerError> {
3312 let contents = match fs::read_to_string(root.join(PETITION_FAILURES_FILE)) {
3313 Ok(contents) => contents,
3314 Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(Vec::new()),
3315 Err(error) => return Err(ReviewerError::QueueIo(error)),
3316 };
3317 contents
3318 .lines()
3319 .filter(|line| !line.trim().is_empty())
3320 .map(|line| serde_json::from_str(line).map_err(ReviewerError::QueueJson))
3321 .collect()
3322}
3323
3324fn append_petition_failure_marker_locked(
3325 root: &Path,
3326 _transaction: &StateTransaction,
3327 marker: &PetitionFailureMarker,
3328) -> Result<(), ReviewerError> {
3329 let mut markers = read_petition_failure_markers(root)?;
3330 if markers
3331 .iter()
3332 .any(|existing| existing.run_id == marker.run_id)
3333 {
3334 return Ok(());
3335 }
3336 markers.push(marker.clone());
3337 let mut bytes = Vec::new();
3338 for marker in markers {
3339 serde_json::to_writer(&mut bytes, &marker).map_err(ReviewerError::QueueJson)?;
3340 bytes.push(b'\n');
3341 }
3342 atomic_replace(&root.join(PETITION_FAILURES_FILE), &bytes)?;
3343 Ok(())
3344}
3345
3346#[derive(Clone, Debug, Default, Eq, PartialEq)]
3347pub struct ReviewQueueSummary {
3348 pub pending_count: usize,
3349 pub oldest_enqueued_at_unix: Option<u64>,
3350}
3351
3352impl ReviewQueueSummary {
3353 pub fn oldest_age_secs_at(&self, now: u64) -> Option<u64> {
3354 self.oldest_enqueued_at_unix
3355 .map(|oldest| now.saturating_sub(oldest))
3356 }
3357}
3358
3359impl ReviewQueue {
3360 pub fn new(root: impl Into<PathBuf>) -> Self {
3361 Self { root: root.into() }
3362 }
3363
3364 pub fn path(&self) -> PathBuf {
3365 self.root.join(REVIEW_QUEUE_FILE)
3366 }
3367
3368 pub fn enqueue(&self, commit_sha: impl Into<String>) -> Result<QueuedReview, ReviewerError> {
3369 self.enqueue_item(commit_sha.into(), None)
3370 }
3371
3372 pub fn enqueue_petition(
3377 &self,
3378 fix_sha: impl Into<String>,
3379 original_sha: impl Into<String>,
3380 ) -> Result<QueuedReview, ReviewerError> {
3381 self.enqueue_item(fix_sha.into(), Some(original_sha.into()))
3382 }
3383
3384 fn enqueue_item(
3385 &self,
3386 commit_sha: String,
3387 petition_for: Option<String>,
3388 ) -> Result<QueuedReview, ReviewerError> {
3389 let transaction = StateTransaction::acquire(&self.root)?;
3390 self.recover_durable_enqueue_locked(&transaction)?;
3391 self.enqueue_item_locked(&transaction, commit_sha, petition_for)
3392 }
3393
3394 pub(crate) fn enqueue_item_locked(
3395 &self,
3396 transaction: &StateTransaction,
3397 commit_sha: String,
3398 petition_for: Option<String>,
3399 ) -> Result<QueuedReview, ReviewerError> {
3400 let pending = self.read_pending_unlocked()?;
3401 if let Some(existing) = pending
3402 .iter()
3403 .find(|queued| queued.commit_sha == commit_sha && queued.petition_for == petition_for)
3404 {
3405 return Ok(existing.clone());
3406 }
3407
3408 let checkpoint = entire_checkpoint_for_current_repo(&commit_sha);
3409 let target = queue_target_label(petition_for.is_some());
3410 let run_store = ReviewRunStore::new(&self.root);
3411 let run = run_store.create_queued_locked(
3412 transaction,
3413 &commit_sha,
3414 target,
3415 checkpoint,
3416 petition_for.clone(),
3417 )?;
3418 let item = QueuedReview {
3419 run_id: run.id,
3420 commit_sha,
3421 enqueued_at_unix: unix_now(),
3422 petition_for,
3423 ..Default::default()
3424 };
3425 write_enqueue_inflight(&self.root, &item)?;
3426 self.append_queue_item_locked(transaction, &item)?;
3427 remove_enqueue_inflight(&self.root)?;
3428 Ok(item)
3429 }
3430
3431 pub fn pending_read_only(&self) -> Result<Vec<QueuedReview>, ReviewerError> {
3433 self.read_pending_unlocked()
3434 }
3435
3436 pub fn pending(&self) -> Result<Vec<QueuedReview>, ReviewerError> {
3437 let transaction = StateTransaction::acquire(&self.root)?;
3438 self.recover_durable_enqueue_locked(&transaction)?;
3439 let run_store = ReviewRunStore::new(&self.root);
3442 run_store.recover_stale_scheduler_claims_locked(&transaction)?;
3443 self.read_pending_unlocked()
3444 }
3445
3446 fn read_pending_unlocked(&self) -> Result<Vec<QueuedReview>, ReviewerError> {
3447 let contents = match fs::read_to_string(self.path()) {
3448 Ok(contents) => contents,
3449 Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(Vec::new()),
3450 Err(error) => return Err(ReviewerError::QueueIo(error)),
3451 };
3452
3453 contents
3454 .lines()
3455 .filter(|line| !line.trim().is_empty())
3456 .map(|line| serde_json::from_str(line).map_err(ReviewerError::QueueJson))
3457 .collect()
3458 }
3459
3460 fn append_queue_item_locked(
3461 &self,
3462 _transaction: &StateTransaction,
3463 item: &QueuedReview,
3464 ) -> Result<(), ReviewerError> {
3465 let mut bytes = match fs::read(self.path()) {
3466 Ok(bytes) => bytes,
3467 Err(error) if error.kind() == io::ErrorKind::NotFound => Vec::new(),
3468 Err(error) => return Err(ReviewerError::QueueIo(error)),
3469 };
3470 serde_json::to_writer(&mut bytes, item).map_err(ReviewerError::QueueJson)?;
3471 bytes.push(b'\n');
3472 atomic_replace(&self.path(), &bytes)?;
3473 Ok(())
3474 }
3475
3476 fn recover_durable_enqueue(&self) -> Result<(), ReviewerError> {
3477 let transaction = StateTransaction::acquire(&self.root)?;
3478 self.recover_durable_enqueue_locked(&transaction)
3479 }
3480
3481 fn recover_durable_enqueue_locked(
3482 &self,
3483 transaction: &StateTransaction,
3484 ) -> Result<(), ReviewerError> {
3485 let inflight_path = self.root.join(ENQUEUE_INFLIGHT_FILE);
3486 if inflight_path.exists() {
3487 let bytes = fs::read(&inflight_path).map_err(ReviewerError::QueueIo)?;
3488 let item: QueuedReview =
3489 serde_json::from_slice(&bytes).map_err(ReviewerError::QueueJson)?;
3490 let run_store = ReviewRunStore::new(&self.root);
3491 run_store.ensure_queued_locked(
3492 transaction,
3493 &item.run_id,
3494 &item.commit_sha,
3495 queue_target_label(item.petition_for.is_some()),
3496 entire_checkpoint_for_current_repo(&item.commit_sha),
3497 item.petition_for.clone(),
3498 )?;
3499 let pending = self.read_pending_unlocked()?;
3500 if !queue_contains_item(&pending, &item) {
3501 self.append_queue_item_locked(transaction, &item)?;
3502 }
3503 remove_enqueue_inflight(&self.root)?;
3504 }
3505 self.recover_orphan_queued_runs_locked(transaction)?;
3506 self.normalize_legacy_queue_run_ids_locked(transaction)?;
3507 self.remove_terminal_failed_petitions_locked(transaction)?;
3508 self.recover_unqueued_petition_enqueues_locked(transaction)
3509 }
3510
3511 fn recover_orphan_queued_runs_locked(
3512 &self,
3513 transaction: &StateTransaction,
3514 ) -> Result<(), ReviewerError> {
3515 let pending = self.read_pending_unlocked()?;
3516 let referenced: std::collections::BTreeSet<String> = pending
3517 .iter()
3518 .filter(|item| !item.run_id.trim().is_empty())
3519 .map(|item| item.run_id.clone())
3520 .collect();
3521 let run_store = ReviewRunStore::new(&self.root);
3522 let (runs, _skipped) = run_store.list_lenient()?;
3523 for run in runs {
3524 if run.status != ReviewRunStatus::Queued || referenced.contains(&run.id) {
3525 continue;
3526 }
3527 if run.target == "commit" || run.target == "petition" {
3528 let item = QueuedReview {
3529 run_id: run.id.clone(),
3530 commit_sha: run.commit_sha.clone(),
3531 enqueued_at_unix: run.created_at_unix,
3532 petition_for: run.petition_for.clone(),
3533 ..Default::default()
3534 };
3535 if !queue_contains_item(&pending, &item) {
3536 self.append_queue_item_locked(transaction, &item)?;
3537 }
3538 } else {
3539 run_store.cancel_queued_locked(transaction, &run.id)?;
3540 }
3541 }
3542 Ok(())
3543 }
3544
3545 pub fn summary(&self) -> Result<ReviewQueueSummary, ReviewerError> {
3546 let pending = self.pending_read_only()?;
3547 Ok(ReviewQueueSummary {
3548 pending_count: pending.len(),
3549 oldest_enqueued_at_unix: pending.iter().map(|item| item.enqueued_at_unix).min(),
3550 })
3551 }
3552
3553 pub fn remove_sha(&self, sha: &str) -> Result<(), ReviewerError> {
3556 self.recover_durable_enqueue()?;
3557 let transaction = StateTransaction::acquire(&self.root)?;
3558 let remaining: Vec<QueuedReview> = self
3559 .read_pending_unlocked()?
3560 .into_iter()
3561 .filter(|item| item.commit_sha != sha)
3562 .collect();
3563 self.rewrite(&transaction, &remaining)
3564 }
3565
3566 fn rewrite(
3567 &self,
3568 _transaction: &StateTransaction,
3569 items: &[QueuedReview],
3570 ) -> Result<(), ReviewerError> {
3571 let mut bytes = Vec::new();
3572 for item in items {
3573 serde_json::to_writer(&mut bytes, item).map_err(ReviewerError::QueueJson)?;
3574 bytes.push(b'\n');
3575 }
3576 atomic_replace(&self.path(), &bytes)?;
3577 Ok(())
3578 }
3579
3580 pub fn remove_item(&self, sha: &str, petition_for: Option<&str>) -> Result<(), ReviewerError> {
3584 self.recover_durable_enqueue()?;
3585 let transaction = StateTransaction::acquire(&self.root)?;
3586 self.remove_item_locked(&transaction, sha, petition_for)
3587 }
3588
3589 fn remove_item_locked(
3590 &self,
3591 transaction: &StateTransaction,
3592 sha: &str,
3593 petition_for: Option<&str>,
3594 ) -> Result<(), ReviewerError> {
3595 let remaining: Vec<QueuedReview> = self
3596 .read_pending_unlocked()?
3597 .into_iter()
3598 .filter(|item| {
3599 !(item.commit_sha == sha && item.petition_for.as_deref() == petition_for)
3600 })
3601 .collect();
3602 self.rewrite(transaction, &remaining)
3603 }
3604
3605 pub fn remove_run_id(&self, run_id: &str) -> Result<(), ReviewerError> {
3606 self.recover_durable_enqueue()?;
3607 let transaction = StateTransaction::acquire(&self.root)?;
3608 let remaining: Vec<QueuedReview> = self
3609 .read_pending_unlocked()?
3610 .into_iter()
3611 .filter(|item| item.run_id != run_id)
3612 .collect();
3613 self.rewrite(&transaction, &remaining)
3614 }
3615
3616 pub(crate) fn attach_run_id_for_item(
3617 &self,
3618 sha: &str,
3619 petition_for: Option<&str>,
3620 run_id: &str,
3621 ) -> Result<(), ReviewerError> {
3622 let transaction = StateTransaction::acquire(&self.root)?;
3623 let pending = self.read_pending_unlocked()?;
3624 let mut changed = false;
3625 let updated: Vec<QueuedReview> = pending
3626 .iter()
3627 .map(|item| {
3628 if item.commit_sha == sha
3629 && item.petition_for.as_deref() == petition_for
3630 && item.run_id.trim().is_empty()
3631 {
3632 changed = true;
3633 QueuedReview {
3634 run_id: run_id.to_owned(),
3635 ..item.clone()
3636 }
3637 } else {
3638 item.clone()
3639 }
3640 })
3641 .collect();
3642 if changed {
3643 self.rewrite(&transaction, &updated)?;
3644 }
3645 Ok(())
3646 }
3647 pub(crate) fn attach_batch_id_for_item(
3648 &self,
3649 sha: &str,
3650 petition_for: Option<&str>,
3651 batch_id: &str,
3652 ) -> Result<(), ReviewerError> {
3653 let transaction = StateTransaction::acquire(&self.root)?;
3654 let pending = self.read_pending_unlocked()?;
3655 let mut changed = false;
3656 let updated: Vec<QueuedReview> = pending
3657 .iter()
3658 .map(|item| {
3659 if item.commit_sha == sha
3660 && item.petition_for.as_deref() == petition_for
3661 && item.batch_id.is_none()
3662 {
3663 changed = true;
3664 QueuedReview {
3665 batch_id: Some(batch_id.to_owned()),
3666 ..item.clone()
3667 }
3668 } else {
3669 item.clone()
3670 }
3671 })
3672 .collect();
3673 if changed {
3674 self.rewrite(&transaction, &updated)?;
3675 }
3676 Ok(())
3677 }
3678}
3679
3680fn unqueued_petition_enqueues(
3681 store: &LedgerStore,
3682 pending: &[QueuedReview],
3683 failures: &[PetitionFailureMarker],
3684) -> Result<Vec<TransitionProvenance>, ReviewerError> {
3685 let history = store.read_history().map_err(ReviewerError::Ledger)?;
3686 let mut latest_enqueue = std::collections::BTreeMap::<String, TransitionProvenance>::new();
3687 for entry in history {
3688 let Some(provenance) = entry.transition_provenance.as_ref() else {
3689 continue;
3690 };
3691 match provenance.kind {
3692 TransitionProvenanceKind::PetitionEnqueued => {
3693 latest_enqueue.insert(provenance.original_sha.clone(), provenance.clone());
3694 }
3695 TransitionProvenanceKind::PetitionReview => {
3696 latest_enqueue.remove(&provenance.original_sha);
3697 }
3698 }
3699 }
3700 Ok(latest_enqueue
3701 .into_values()
3702 .filter(|provenance| {
3703 !failures.iter().any(|failure| {
3704 failure.fix_sha == provenance.fix_sha
3705 && failure.original_sha == provenance.original_sha
3706 && failure.attempts == provenance.attempts
3707 })
3708 })
3709 .filter(|provenance| {
3710 !queue_contains_item(
3711 pending,
3712 &QueuedReview {
3713 run_id: String::new(),
3714 commit_sha: provenance.fix_sha.clone(),
3715 enqueued_at_unix: 0,
3716 petition_for: Some(provenance.original_sha.clone()),
3717 ..Default::default()
3718 },
3719 )
3720 })
3721 .collect())
3722}
3723
3724fn resolve_run_id_for_item(
3725 run_store: &ReviewRunStore,
3726 item: &QueuedReview,
3727) -> Result<(String, bool), ReviewerError> {
3728 if !item.run_id.trim().is_empty() {
3729 return Ok((item.run_id.clone(), false));
3730 }
3731 let (runs, _) = run_store.list_lenient()?;
3732 let mut matches: Vec<&ReviewRun> = runs
3733 .iter()
3734 .filter(|run| {
3735 run.status == ReviewRunStatus::Queued
3736 && run.commit_sha == item.commit_sha
3737 && run.petition_for == item.petition_for
3738 })
3739 .collect();
3740 matches.sort_by_key(|run| run.created_at_unix);
3741 if let Some(run) = matches.first() {
3742 return Ok((run.id.clone(), false));
3743 }
3744 Ok((generate_run_id(&item.commit_sha), true))
3745}
3746
3747pub(crate) fn commit_petition_resolve(
3751 state_dir: &Path,
3752 store: &LedgerStore,
3753 rejection_sha: &str,
3754 fix_sha: &str,
3755) -> Result<(QueuedReview, u32), ReviewerError> {
3756 let queue = ReviewQueue::new(state_dir);
3757 let transaction = StateTransaction::acquire(state_dir)?;
3758 queue.recover_durable_enqueue_locked(&transaction)?;
3759
3760 let original = store.show(rejection_sha).map_err(ReviewerError::Ledger)?;
3761 if !original.is_unresolved_rejection() {
3762 return Err(ReviewerError::PetitionRejectionNotOpen {
3763 sha: rejection_sha.to_owned(),
3764 });
3765 }
3766
3767 let pending = queue.read_pending_unlocked()?;
3768 if pending
3769 .iter()
3770 .any(|item| item.petition_for.as_deref() == Some(rejection_sha))
3771 {
3772 return Err(ReviewerError::PetitionInFlight {
3773 rejection_sha: rejection_sha.to_owned(),
3774 });
3775 }
3776
3777 let prior = store
3778 .petition_attempts_for(rejection_sha)
3779 .map_err(ReviewerError::Ledger)?;
3780 if prior >= crate::resolve::MAX_PETITION_ATTEMPTS {
3781 return Err(ReviewerError::PetitionExhausted {
3782 rejection_sha: rejection_sha.to_owned(),
3783 });
3784 }
3785 let next_attempts = prior.saturating_add(1);
3786 let reason = format!(
3787 "petition review enqueued: fix={fix_sha}, attempts={next_attempts}/{}",
3788 crate::resolve::MAX_PETITION_ATTEMPTS
3789 );
3790 store
3791 .append_petition_transition_locked(
3792 &transaction,
3793 rejection_sha,
3794 Disposition::Open,
3795 ResolutionKind::Resolved,
3796 &reason,
3797 next_attempts,
3798 Some(TransitionProvenance {
3799 kind: TransitionProvenanceKind::PetitionEnqueued,
3800 fix_sha: fix_sha.to_owned(),
3801 original_sha: rejection_sha.to_owned(),
3802 attempts: next_attempts,
3803 }),
3804 )
3805 .map_err(ReviewerError::Ledger)?;
3806 let item = queue.enqueue_item_locked(
3807 &transaction,
3808 fix_sha.to_owned(),
3809 Some(rejection_sha.to_owned()),
3810 )?;
3811 Ok((item, next_attempts))
3812}
3813
3814impl ReviewQueue {
3815 fn normalize_legacy_queue_run_ids_locked(
3816 &self,
3817 transaction: &StateTransaction,
3818 ) -> Result<(), ReviewerError> {
3819 let pending = self.read_pending_unlocked()?;
3820 let run_store = ReviewRunStore::new(&self.root);
3821 let (runs, _) = run_store.list_lenient()?;
3822 let mut changed = false;
3823 let normalized: Vec<QueuedReview> = pending
3824 .iter()
3825 .map(|item| {
3826 if !item.run_id.trim().is_empty() {
3827 return item.clone();
3828 }
3829 let mut matches: Vec<&ReviewRun> = runs
3830 .iter()
3831 .filter(|run| {
3832 run.status == ReviewRunStatus::Queued
3833 && run.commit_sha == item.commit_sha
3834 && run.petition_for == item.petition_for
3835 })
3836 .collect();
3837 matches.sort_by_key(|run| run.created_at_unix);
3838 if let Some(run) = matches.first() {
3839 changed = true;
3840 QueuedReview {
3841 run_id: run.id.clone(),
3842 ..item.clone()
3843 }
3844 } else {
3845 item.clone()
3846 }
3847 })
3848 .collect();
3849 if changed {
3850 self.rewrite(transaction, &normalized)?;
3851 }
3852 Ok(())
3853 }
3854
3855 fn remove_terminal_failed_petitions_locked(
3856 &self,
3857 transaction: &StateTransaction,
3858 ) -> Result<(), ReviewerError> {
3859 let failures = read_petition_failure_markers(&self.root)?;
3860 if failures.is_empty() {
3861 return Ok(());
3862 }
3863 let pending = self.read_pending_unlocked()?;
3864 let remaining: Vec<QueuedReview> = pending
3865 .iter()
3866 .filter(|item| {
3867 !failures.iter().any(|failure| {
3868 failure.run_id == item.run_id
3869 && failure.fix_sha == item.commit_sha
3870 && item.petition_for.as_deref() == Some(failure.original_sha.as_str())
3871 })
3872 })
3873 .cloned()
3874 .collect();
3875 if remaining.len() != pending.len() {
3876 self.rewrite(transaction, &remaining)?;
3877 }
3878 Ok(())
3879 }
3880
3881 fn recover_unqueued_petition_enqueues_locked(
3882 &self,
3883 transaction: &StateTransaction,
3884 ) -> Result<(), ReviewerError> {
3885 let store = LedgerStore::new(&self.root);
3886 let pending = self.read_pending_unlocked()?;
3887 let failures = read_petition_failure_markers(&self.root)?;
3888 for provenance in unqueued_petition_enqueues(&store, &pending, &failures)? {
3889 self.enqueue_item_locked(
3890 transaction,
3891 provenance.fix_sha.clone(),
3892 Some(provenance.original_sha.clone()),
3893 )?;
3894 }
3895 Ok(())
3896 }
3897}
3898
3899fn queue_target_label(is_petition: bool) -> &'static str {
3900 if is_petition { "petition" } else { "commit" }
3901}
3902
3903fn queue_contains_item(pending: &[QueuedReview], item: &QueuedReview) -> bool {
3904 pending.iter().any(|queued| {
3905 queued.commit_sha == item.commit_sha
3906 && queued.petition_for == item.petition_for
3907 && (queued.run_id.trim().is_empty()
3908 || item.run_id.trim().is_empty()
3909 || queued.run_id == item.run_id)
3910 })
3911}
3912
3913fn write_enqueue_inflight(root: &Path, item: &QueuedReview) -> Result<(), ReviewerError> {
3914 let bytes = serde_json::to_vec(item).map_err(ReviewerError::QueueJson)?;
3915 atomic_replace(&root.join(ENQUEUE_INFLIGHT_FILE), &bytes)?;
3916 Ok(())
3917}
3918
3919fn remove_enqueue_inflight(root: &Path) -> Result<(), ReviewerError> {
3920 match fs::remove_file(root.join(ENQUEUE_INFLIGHT_FILE)) {
3921 Ok(()) => {}
3922 Err(error) if error.kind() == io::ErrorKind::NotFound => {}
3923 Err(error) => return Err(ReviewerError::QueueIo(error)),
3924 }
3925 Ok(())
3926}
3927
3928fn run_ledger_phase_applied(run: &ReviewRun, phase: &str) -> bool {
3929 run.ledger_applied_phase.as_deref() == Some(phase)
3930}
3931
3932fn finish_replayed_queue_item(
3933 queue: &ReviewQueue,
3934 run_store: &ReviewRunStore,
3935 run_id: &str,
3936 sha: &str,
3937 petition_for: Option<&str>,
3938 ledger_entries: usize,
3939 report: &mut DrainReport,
3940) -> Result<(), ReviewerError> {
3941 let run = run_store.read(run_id)?;
3942 if run.status != ReviewRunStatus::Completed {
3943 run_store.mark_completed(run_id, ledger_entries)?;
3944 }
3945 queue.remove_item(sha, petition_for)?;
3946 report.reviewed.push(sha.to_owned());
3947 report.ledger_entries += ledger_entries;
3948 Ok(())
3949}
3950
3951pub trait MaterialLoader {
3954 fn load(&self, sha: &str) -> Result<(Claim, String), ReviewerError>;
3955}
3956
3957#[derive(Clone, Debug, Default)]
3958pub struct GitMaterialLoader {
3959 pub evidence_patterns: Vec<String>,
3962}
3963
3964impl GitMaterialLoader {
3965 pub fn from_config(config: &config::TruthMirrorConfig) -> Self {
3966 Self::with_patterns(config.gates.to_policy().evidence_patterns)
3967 }
3968
3969 pub fn with_patterns(evidence_patterns: Vec<String>) -> Self {
3970 Self { evidence_patterns }
3971 }
3972}
3973
3974impl MaterialLoader for GitMaterialLoader {
3975 fn load(&self, sha: &str) -> Result<(Claim, String), ReviewerError> {
3976 let message = git_output(["show", "--format=%B", "--no-patch", sha])?;
3977 let diff = git_output(["show", "--format=", "--patch", sha])?;
3978 let claim = if self.evidence_patterns.is_empty() {
3979 Claim::parse(&message)?
3980 } else {
3981 Claim::parse_with(&message, &self.evidence_patterns)?
3982 };
3983 Ok((claim, diff))
3984 }
3985}
3986
3987#[derive(Clone, Debug, Default, Eq, PartialEq)]
3988pub struct DrainReport {
3989 pub reviewed: Vec<String>,
3990 pub failed: Vec<String>,
3992 pub ledger_entries: usize,
3993}
3994
3995#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
3996struct StrictPetitionBatchKey<'a> {
3997 arbiter_harness: &'static str,
3998 arbiter_model: &'a str,
3999 arbiter_effort: &'static str,
4000}
4001
4002#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
4003struct PetitionBatchKey<'a> {
4004 repository: &'a Path,
4005 fix_sha: &'a str,
4006 watched_agent: &'static str,
4007 watched_model: &'a str,
4008 reviewer_harness: &'static str,
4009 reviewer_model: &'a str,
4010 reviewer_effort: &'static str,
4011 allow_same_model: bool,
4012 strict: Option<StrictPetitionBatchKey<'a>>,
4013 materialized_context: &'a str,
4014}
4015
4016impl<'a> PetitionBatchKey<'a> {
4017 fn new(
4018 repository: &'a Path,
4019 fix_sha: &'a str,
4020 selection: &'a ReviewSelection,
4021 materialized_context: &'a str,
4022 ) -> Self {
4023 Self {
4024 repository,
4025 fix_sha,
4026 watched_agent: surface::agent_slug(selection.watched_agent),
4027 watched_model: &selection.watched_model,
4028 reviewer_harness: harness_slug(selection.reviewer_harness),
4029 reviewer_model: &selection.reviewer_model,
4030 reviewer_effort: selection.reviewer_effort.as_str(),
4031 allow_same_model: selection.allow_same_model,
4032 strict: selection
4033 .strict
4034 .as_ref()
4035 .map(|strict| StrictPetitionBatchKey {
4036 arbiter_harness: harness_slug(strict.arbiter_harness),
4037 arbiter_model: &strict.arbiter_model,
4038 arbiter_effort: strict.arbiter_effort.as_str(),
4039 }),
4040 materialized_context,
4041 }
4042 }
4043}
4044
4045#[derive(Clone, Debug, Eq, PartialEq)]
4046enum PlannedDrainJob {
4047 Single(QueuedReview),
4048 PetitionBatch(Vec<QueuedReview>),
4049}
4050
4051fn plan_batched_drain(
4052 pending: &[QueuedReview],
4053 repository: &Path,
4054 selection: &ReviewSelection,
4055 materialized_context: &str,
4056 configured_max_batch_size: usize,
4057) -> Vec<PlannedDrainJob> {
4058 let max_batch_size = configured_max_batch_size.clamp(1, config::MAX_PETITION_BATCH_SIZE);
4059 let mut positioned = Vec::<(usize, usize, PlannedDrainJob)>::new();
4060 let mut groups =
4061 std::collections::BTreeMap::<PetitionBatchKey<'_>, (usize, Vec<QueuedReview>)>::new();
4062
4063 for (index, item) in pending.iter().enumerate() {
4064 if item.petition_for.is_none() {
4065 positioned.push((index, 0, PlannedDrainJob::Single(item.clone())));
4066 continue;
4067 }
4068 let key = PetitionBatchKey::new(
4069 repository,
4070 &item.commit_sha,
4071 selection,
4072 materialized_context,
4073 );
4074 let group = groups.entry(key).or_insert_with(|| (index, Vec::new()));
4075 group.1.push(item.clone());
4076 }
4077
4078 for (_, (first_index, mut members)) in groups {
4079 members.sort_by(|left, right| {
4080 left.petition_for
4081 .cmp(&right.petition_for)
4082 .then_with(|| left.run_id.cmp(&right.run_id))
4083 });
4084 for (chunk_index, chunk) in members.chunks(max_batch_size).enumerate() {
4085 positioned.push((
4086 first_index,
4087 chunk_index,
4088 PlannedDrainJob::PetitionBatch(chunk.to_vec()),
4089 ));
4090 }
4091 }
4092 positioned.sort_by_key(|(first_index, chunk_index, _)| (*first_index, *chunk_index));
4093 positioned.into_iter().map(|(_, _, job)| job).collect()
4094}
4095
4096fn complete_review_execution<R: ProcessRunner>(
4102 job: &ReviewJob,
4103 runner: &R,
4104 run_store: &ReviewRunStore,
4105 run_id: &str,
4106 existing_spool: Option<PendingReviewSpool>,
4107 timeout: Option<Duration>,
4108) -> Result<ReviewExecution, ReviewerError> {
4109 complete_review_execution_in_dir(
4110 job,
4111 runner,
4112 run_store,
4113 run_id,
4114 existing_spool,
4115 timeout,
4116 None,
4117 )
4118}
4119
4120fn complete_review_execution_in_dir<R: ProcessRunner>(
4121 job: &ReviewJob,
4122 runner: &R,
4123 run_store: &ReviewRunStore,
4124 run_id: &str,
4125 existing_spool: Option<PendingReviewSpool>,
4126 timeout: Option<Duration>,
4127 current_dir: Option<&Path>,
4128) -> Result<ReviewExecution, ReviewerError> {
4129 if let Some(spool) = existing_spool {
4130 if let Some(first_stdout) = spool.strict_pass_stdout.as_ref()
4131 && spool.entries.len() == 1
4132 {
4133 let strict_entry =
4134 draft_review_strict_pass_in_dir(job, runner, first_stdout, timeout, current_dir)?;
4135 let mut entries = spool.entries;
4136 entries.push(strict_entry);
4137 run_store.persist_pending_review(
4138 run_id,
4139 PendingReviewSpool {
4140 entries: entries.clone(),
4141 petition_transition_pending: spool.petition_transition_pending,
4142 strict_pass_stdout: None,
4143 },
4144 )?;
4145 return Ok(ReviewExecution { entries });
4146 }
4147 if !spool.entries.is_empty() {
4148 return Ok(ReviewExecution {
4149 entries: spool.entries,
4150 });
4151 }
4152 }
4153
4154 let (first_entry, first_stdout) =
4155 draft_review_first_pass_in_dir(job, runner, timeout, current_dir)?;
4156 let first_verdict = ParsedVerdict::parse(&first_stdout)?;
4157 if strict_pass_needed(job, &first_verdict) {
4158 run_store.persist_pending_review(
4159 run_id,
4160 PendingReviewSpool {
4161 entries: vec![first_entry.clone()],
4162 petition_transition_pending: job.petition.is_some(),
4163 strict_pass_stdout: Some(first_stdout.clone()),
4164 },
4165 )?;
4166 let strict_entry =
4167 draft_review_strict_pass_in_dir(job, runner, &first_stdout, timeout, current_dir)?;
4168 let entries = vec![first_entry, strict_entry];
4169 run_store.persist_pending_review(
4170 run_id,
4171 PendingReviewSpool {
4172 entries: entries.clone(),
4173 petition_transition_pending: job.petition.is_some(),
4174 strict_pass_stdout: None,
4175 },
4176 )?;
4177 Ok(ReviewExecution { entries })
4178 } else {
4179 let entries = vec![first_entry];
4180 run_store.persist_pending_review(
4181 run_id,
4182 PendingReviewSpool {
4183 entries: entries.clone(),
4184 petition_transition_pending: job.petition.is_some(),
4185 strict_pass_stdout: None,
4186 },
4187 )?;
4188 Ok(ReviewExecution { entries })
4189 }
4190}
4191
4192#[derive(Clone, Debug)]
4193struct PreparedPetitionMember {
4194 item: QueuedReview,
4195 run_id: String,
4196 run: ReviewRun,
4197 petition: PetitionContext,
4198 batch_id: Option<String>,
4199}
4200
4201#[derive(Clone, Debug)]
4202struct ReadyPetitionMember {
4203 member: PreparedPetitionMember,
4204 execution: ReviewExecution,
4205}
4206fn batch_record_path(root: &Path, batch_id: &str) -> PathBuf {
4207 root.join(BATCHES_DIR).join(format!("{batch_id}.json"))
4208}
4209
4210fn read_batch_review_record(
4211 root: &Path,
4212 batch_id: &str,
4213) -> Result<Option<BatchReviewRecord>, ReviewerError> {
4214 let path = batch_record_path(root, batch_id);
4215 match fs::read_to_string(&path) {
4216 Ok(contents) => serde_json::from_str(&contents)
4217 .map_err(ReviewerError::RunJson)
4218 .map(Some),
4219 Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None),
4220 Err(error) => Err(ReviewerError::QueueIo(error)),
4221 }
4222}
4223
4224fn write_batch_review_record(root: &Path, record: &BatchReviewRecord) -> Result<(), ReviewerError> {
4225 let path = batch_record_path(root, &record.batch_id);
4226 let bytes = serde_json::to_vec_pretty(record).map_err(ReviewerError::RunJson)?;
4227 atomic_replace(&path, &bytes)?;
4228 Ok(())
4229}
4230fn build_batch_review_record(
4231 batch_id: &str,
4232 fix_sha: &str,
4233 phase: &str,
4234 members: &[&PreparedPetitionMember],
4235 plan: &ReviewPlan,
4236) -> BatchReviewRecord {
4237 let timestamp = unix_now();
4238 BatchReviewRecord {
4239 batch_id: batch_id.to_owned(),
4240 fix_sha: fix_sha.to_owned(),
4241 phase: phase.to_owned(),
4242 member_run_ids: members.iter().map(|member| member.run_id.clone()).collect(),
4243 member_original_shas: members
4244 .iter()
4245 .map(|member| member.petition.original_sha.clone())
4246 .collect(),
4247 reviewer_harness: harness_slug(plan.reviewer_harness).to_owned(),
4248 reviewer_model: plan.reviewer_model.clone(),
4249 raw_stdout: None,
4250 created_at_unix: timestamp,
4251 updated_at_unix: timestamp,
4252 }
4253}
4254
4255fn update_batch_review_record_phase_and_output(
4256 record: &mut BatchReviewRecord,
4257 phase: &str,
4258 raw_stdout: Option<String>,
4259) {
4260 record.phase = phase.to_owned();
4261 record.raw_stdout = raw_stdout;
4262 record.updated_at_unix = unix_now();
4263}
4264fn claim_petition_batch(
4265 queue: &ReviewQueue,
4266 run_store: &ReviewRunStore,
4267 fix_sha: &str,
4268 batch_id: &str,
4269 members: &mut [PreparedPetitionMember],
4270) -> Result<(), ReviewerError> {
4271 let member_count = members.len();
4272 for (index, member) in members.iter_mut().enumerate() {
4273 member.batch_id = Some(batch_id.to_owned());
4274 let batch_ref = BatchRunRef {
4275 batch_id: batch_id.to_owned(),
4276 kind: BatchKind::Petition,
4277 member_index: index as u32,
4278 member_count: member_count as u32,
4279 fix_sha: fix_sha.to_owned(),
4280 };
4281 member.run.batch = Some(batch_ref.clone());
4282 run_store.set_batch_metadata(&member.run_id, batch_ref)?;
4283 queue.attach_batch_id_for_item(fix_sha, member.item.petition_for.as_deref(), batch_id)?;
4284 }
4285 Ok(())
4286}
4287
4288fn petition_batch_id(phase: &str, first_run_id: &str) -> String {
4289 format!("petition-batch-{phase}-{first_run_id}")
4290}
4291
4292#[allow(clippy::too_many_arguments)]
4293fn run_petition_batch_pass<R: ProcessRunner>(
4294 state_dir: &Path,
4295 request: ReviewRequest,
4296 prompt: &str,
4297 batch_id: &str,
4298 fix_sha: &str,
4299 requested_shas: &[String],
4300 runner: &R,
4301 phase: &str,
4302 members: &[&PreparedPetitionMember],
4303 timeout: Option<Duration>,
4304 current_dir: Option<&Path>,
4305) -> Result<(ReviewPlan, ParsedPetitionBatch), ReviewerError> {
4306 let plan = ReviewPlan::build(request)?;
4307
4308 let mut record = match read_batch_review_record(state_dir, batch_id)? {
4309 Some(existing) => existing,
4310 None => {
4311 let record = build_batch_review_record(batch_id, fix_sha, "claimed", members, &plan);
4312 write_batch_review_record(state_dir, &record)?;
4313 record
4314 }
4315 };
4316
4317 if let Some(raw_stdout) = record.raw_stdout.as_deref() {
4318 let parsed = ParsedPetitionBatch::parse(raw_stdout, batch_id, fix_sha, requested_shas)?;
4319 for unknown in &parsed.unknown_item_ids {
4320 tracing::warn!(
4321 batch_id,
4322 original_rejection_sha = unknown,
4323 "ignored unknown or malformed petition-batch result"
4324 );
4325 }
4326 return Ok((plan, parsed));
4327 }
4328
4329 let output = plan.run_with_dir(prompt, runner, timeout, current_dir)?;
4330 ensure_process_success(&output)?;
4331 update_batch_review_record_phase_and_output(&mut record, phase, Some(output.stdout.clone()));
4332 write_batch_review_record(state_dir, &record)?;
4333 let parsed = ParsedPetitionBatch::parse(&output.stdout, batch_id, fix_sha, requested_shas)?;
4334 for unknown in &parsed.unknown_item_ids {
4335 tracing::warn!(
4336 batch_id,
4337 original_rejection_sha = unknown,
4338 "ignored unknown or malformed petition-batch result"
4339 );
4340 }
4341 Ok((plan, parsed))
4342}
4343
4344fn fail_petition_batch_member(
4345 queue: &ReviewQueue,
4346 run_store: &ReviewRunStore,
4347 member: &PreparedPetitionMember,
4348 error: impl Into<String>,
4349 remove_from_queue: bool,
4350 report: &mut DrainReport,
4351) -> Result<(), ReviewerError> {
4352 let error = error.into();
4353 if remove_from_queue {
4354 let transaction = StateTransaction::acquire(&queue.root)?;
4355 run_store.mark_failed_locked(&transaction, &member.run_id, &error)?;
4356 append_petition_failure_marker_locked(
4357 &queue.root,
4358 &transaction,
4359 &PetitionFailureMarker {
4360 run_id: member.run_id.clone(),
4361 fix_sha: member.item.commit_sha.clone(),
4362 original_sha: member.petition.original_sha.clone(),
4363 attempts: member.petition.attempts_so_far,
4364 error,
4365 },
4366 )?;
4367 queue.remove_item_locked(
4368 &transaction,
4369 &member.item.commit_sha,
4370 member.item.petition_for.as_deref(),
4371 )?;
4372 report.failed.push(member.item.commit_sha.clone());
4373 } else {
4374 run_store.mark_failed(&member.run_id, error)?;
4375 }
4376 Ok(())
4377}
4378
4379fn finish_petition_batch_member(
4380 queue: &ReviewQueue,
4381 run_store: &ReviewRunStore,
4382 ready: ReadyPetitionMember,
4383 store: &LedgerStore,
4384 config: &config::TruthMirrorConfig,
4385 report: &mut DrainReport,
4386) -> Result<(), ReviewerError> {
4387 let ReadyPetitionMember { member, execution } = ready;
4388 apply_petition_batch_execution(&member.petition, &execution, store)?;
4389 run_store.mark_ledger_applied(&member.run_id, REVIEW_LEDGER_PHASE, execution.entries.len())?;
4390 run_store.clear_pending_review(&member.run_id)?;
4391 record_memory_skill_outcome(
4392 &queue.root,
4393 config,
4394 &member.run_id,
4395 &member.item.commit_sha,
4396 "watch-drain-batch",
4397 &execution.entries,
4398 );
4399 run_store.mark_completed(&member.run_id, execution.entries.len())?;
4400 queue.remove_item(&member.item.commit_sha, member.item.petition_for.as_deref())?;
4401 report.ledger_entries += execution.entries.len();
4402 report.reviewed.push(member.item.commit_sha);
4403 Ok(())
4404}
4405
4406fn finish_ready_petition_members(
4407 queue: &ReviewQueue,
4408 run_store: &ReviewRunStore,
4409 mut ready: Vec<ReadyPetitionMember>,
4410 store: &LedgerStore,
4411 config: &config::TruthMirrorConfig,
4412 report: &mut DrainReport,
4413) -> Result<(), ReviewerError> {
4414 ready.sort_by(|left, right| {
4415 left.member
4416 .petition
4417 .original_sha
4418 .cmp(&right.member.petition.original_sha)
4419 });
4420 for member in ready {
4421 finish_petition_batch_member(queue, run_store, member, store, config, report)?;
4422 }
4423 Ok(())
4424}
4425
4426#[allow(clippy::too_many_arguments)]
4427fn drain_petition_batch<R: ProcessRunner, L: MaterialLoader>(
4428 queue: &ReviewQueue,
4429 run_store: &ReviewRunStore,
4430 items: Vec<QueuedReview>,
4431 loader: &L,
4432 selection: &ReviewSelection,
4433 context: &str,
4434 runner: &R,
4435 store: &LedgerStore,
4436 config: &config::TruthMirrorConfig,
4437 current_dir: Option<&Path>,
4438) -> Result<DrainReport, ReviewerError> {
4439 let timeout = config.reviewer.timeout();
4440 let mut report = DrainReport::default();
4441 let Some(fix_sha) = items.first().map(|item| item.commit_sha.clone()) else {
4442 return Ok(report);
4443 };
4444 debug_assert!(
4445 items
4446 .iter()
4447 .all(|item| { item.commit_sha == fix_sha && item.petition_for.is_some() })
4448 );
4449
4450 let mut active = Vec::new();
4451 for item in items {
4452 let petition_for = item.petition_for.clone();
4453 let (run_id, _) = resolve_run_id_for_item(run_store, &item)?;
4454 if item.run_id.trim().is_empty() {
4455 queue.attach_run_id_for_item(&fix_sha, petition_for.as_deref(), &run_id)?;
4456 }
4457 let run = match run_store.ensure_queued(&run_id, &fix_sha, "petition", petition_for.clone())
4458 {
4459 Ok(run) => run,
4460 Err(ReviewerError::RunJson(_)) => {
4461 queue.remove_item(&fix_sha, petition_for.as_deref())?;
4462 report.failed.push(fix_sha.clone());
4463 continue;
4464 }
4465 Err(error) => return Err(error),
4466 };
4467 if run.status == ReviewRunStatus::Cancelled {
4468 queue.remove_item(&fix_sha, petition_for.as_deref())?;
4469 continue;
4470 }
4471 if run.status == ReviewRunStatus::Completed
4472 || run_ledger_phase_applied(&run, REVIEW_LEDGER_PHASE)
4473 {
4474 finish_replayed_queue_item(
4475 queue,
4476 run_store,
4477 &run_id,
4478 &fix_sha,
4479 petition_for.as_deref(),
4480 run.ledger_entries,
4481 &mut report,
4482 )?;
4483 continue;
4484 }
4485 active.push((item, run_id, run));
4486 }
4487 if active.is_empty() {
4488 return Ok(report);
4489 }
4490
4491 let (claim, diff) = match loader.load(&fix_sha) {
4492 Ok(material) => material,
4493 Err(error) => {
4494 let message = error.to_string();
4495 for (item, run_id, _) in active {
4496 run_store.mark_failed(&run_id, &message)?;
4497 queue.remove_item(&fix_sha, item.petition_for.as_deref())?;
4498 report.failed.push(fix_sha.clone());
4499 }
4500 return Ok(report);
4501 }
4502 };
4503
4504 let mut prepared = Vec::new();
4505 for (item, run_id, run) in active {
4506 let original_sha = item
4507 .petition_for
4508 .as_deref()
4509 .expect("petition batch member missing petition identity");
4510 match petition_context_from_ledger(original_sha, &fix_sha, store) {
4511 Ok(petition) => prepared.push(PreparedPetitionMember {
4512 item,
4513 run_id,
4514 run,
4515 petition,
4516 batch_id: None,
4517 }),
4518 Err(error) => {
4519 run_store.mark_failed(&run_id, error.to_string())?;
4520 queue.remove_item(&fix_sha, Some(original_sha))?;
4521 report.failed.push(fix_sha.clone());
4522 }
4523 }
4524 }
4525 if prepared.is_empty() {
4526 return Ok(report);
4527 }
4528
4529 let mut fresh = Vec::new();
4530 let mut strict_pending = Vec::<(PreparedPetitionMember, LedgerEntry)>::new();
4531 let mut ready = Vec::new();
4532 for member in prepared {
4533 match member.run.pending_review.clone() {
4534 Some(spool) if !spool.entries.is_empty() => {
4535 if spool.strict_pass_stdout.is_some() && spool.entries.len() == 1 {
4536 if selection.strict.is_some() {
4537 strict_pending.push((member, spool.entries[0].clone()));
4538 } else {
4539 fail_petition_batch_member(
4540 queue,
4541 run_store,
4542 &member,
4543 "pending strict petition batch has no strict reviewer selection",
4544 true,
4545 &mut report,
4546 )?;
4547 }
4548 } else {
4549 ready.push(ReadyPetitionMember {
4550 member,
4551 execution: ReviewExecution {
4552 entries: spool.entries,
4553 },
4554 });
4555 }
4556 }
4557 _ => fresh.push(member),
4558 }
4559 }
4560
4561 if !fresh.is_empty() {
4562 for member in &fresh {
4563 if member.run.status != ReviewRunStatus::Running
4564 || member.run.phase != REVIEW_LEDGER_PHASE
4565 {
4566 run_store.mark_running(&member.run_id, REVIEW_LEDGER_PHASE)?;
4567 }
4568 }
4569 let batch_id = petition_batch_id("first", &fresh[0].run_id);
4570 claim_petition_batch(queue, run_store, &fix_sha, &batch_id, &mut fresh)?;
4571 let petitions: Vec<&PetitionContext> =
4572 fresh.iter().map(|member| &member.petition).collect();
4573 let requested_shas: Vec<String> = petitions
4574 .iter()
4575 .map(|petition| petition.original_sha.clone())
4576 .collect();
4577 let prompt = petition_batch_prompt(&batch_id, &petitions, &claim, &diff, context);
4578 let request = selection.request_for(String::new());
4579 match run_petition_batch_pass(
4580 queue.root.as_path(),
4581 request,
4582 &prompt,
4583 &batch_id,
4584 &fix_sha,
4585 &requested_shas,
4586 runner,
4587 "first-pass-running",
4588 &fresh.iter().collect::<Vec<_>>(),
4589 timeout,
4590 current_dir,
4591 ) {
4592 Ok((plan, mut parsed)) => {
4593 for member in fresh {
4594 let original_sha = member.petition.original_sha.clone();
4595 match parsed
4596 .members
4597 .remove(&original_sha)
4598 .expect("batch parser omitted a requested member classification")
4599 {
4600 ParsedPetitionBatchMember::Valid(verdict) => {
4601 let first_entry = petition_entry_from_batch_verdict(
4602 &fix_sha,
4603 &claim,
4604 &member.petition,
4605 &plan,
4606 &verdict,
4607 &member.run_id,
4608 member.batch_id.as_deref().unwrap_or(&batch_id),
4609 );
4610 if selection.strict.is_some() && verdict.verdict == Verdict::Pass {
4611 run_store.persist_pending_review(
4612 &member.run_id,
4613 PendingReviewSpool {
4614 entries: vec![first_entry.clone()],
4615 petition_transition_pending: true,
4616 strict_pass_stdout: Some(verdict.raw),
4617 },
4618 )?;
4619 strict_pending.push((member, first_entry));
4620 } else {
4621 let execution = ReviewExecution {
4622 entries: vec![first_entry],
4623 };
4624 run_store.persist_pending_review(
4625 &member.run_id,
4626 PendingReviewSpool {
4627 entries: execution.entries.clone(),
4628 petition_transition_pending: true,
4629 strict_pass_stdout: None,
4630 },
4631 )?;
4632 ready.push(ReadyPetitionMember { member, execution });
4633 }
4634 }
4635 ParsedPetitionBatchMember::Invalid(message) => {
4636 fail_petition_batch_member(
4637 queue,
4638 run_store,
4639 &member,
4640 message,
4641 true,
4642 &mut report,
4643 )?;
4644 }
4645 }
4646 }
4647 }
4648 Err(
4649 error @ (ReviewerError::VerdictJson { .. }
4650 | ReviewerError::BatchVerdictSchema { .. }),
4651 ) => {
4652 let message = error.to_string();
4653 for member in fresh {
4654 fail_petition_batch_member(
4655 queue,
4656 run_store,
4657 &member,
4658 &message,
4659 true,
4660 &mut report,
4661 )?;
4662 }
4663 }
4664 Err(error) if is_retryable_provider_failure(&error) => {
4665 let message = error.to_string();
4666 for member in &fresh {
4667 run_store.mark_provider_backoff(
4669 &member.run_id,
4670 &message,
4671 DEFAULT_PROVIDER_BACKOFF_SECS,
4672 )?;
4673 }
4674 return Ok(report);
4675 }
4676 Err(error) => {
4677 let message = error.to_string();
4678 for member in &fresh {
4679 fail_petition_batch_member(
4680 queue,
4681 run_store,
4682 member,
4683 &message,
4684 false,
4685 &mut report,
4686 )?;
4687 }
4688 return Err(error);
4689 }
4690 }
4691 }
4692
4693 if strict_pending.is_empty() {
4694 finish_ready_petition_members(queue, run_store, ready, store, config, &mut report)?;
4695 return Ok(report);
4696 }
4697 strict_pending.sort_by(|left, right| {
4698 left.0
4699 .petition
4700 .original_sha
4701 .cmp(&right.0.petition.original_sha)
4702 });
4703 for (member, _) in &strict_pending {
4704 if member.run.status != ReviewRunStatus::Running || member.run.phase != REVIEW_LEDGER_PHASE
4705 {
4706 run_store.mark_running(&member.run_id, REVIEW_LEDGER_PHASE)?;
4707 }
4708 }
4709
4710 let strict = selection
4711 .strict
4712 .as_ref()
4713 .expect("strict petition members without strict selection");
4714 let reviewer_request = selection.request_for(String::new());
4715 validate_strict_arbiter(&reviewer_request, strict)?;
4716 let strict_request = ReviewRequest::new(
4717 selection.watched_agent,
4718 selection.watched_model.clone(),
4719 strict.arbiter_harness,
4720 strict.arbiter_model.clone(),
4721 false,
4722 String::new(),
4723 )
4724 .with_effort(strict.arbiter_effort);
4725 let batch_id = petition_batch_id("strict", &strict_pending[0].0.run_id);
4726 {
4727 let count = strict_pending.len();
4728 for (index, (member, _)) in strict_pending.iter_mut().enumerate() {
4729 member.batch_id = Some(batch_id.clone());
4730 let batch_ref = BatchRunRef {
4731 batch_id: batch_id.clone(),
4732 kind: BatchKind::Petition,
4733 member_index: index as u32,
4734 member_count: count as u32,
4735 fix_sha: fix_sha.clone(),
4736 };
4737 member.run.batch = Some(batch_ref.clone());
4738 run_store.set_batch_metadata(&member.run_id, batch_ref)?;
4739 queue.attach_batch_id_for_item(
4740 &fix_sha,
4741 member.item.petition_for.as_deref(),
4742 &batch_id,
4743 )?;
4744 }
4745 }
4746 let petitions: Vec<&PetitionContext> = strict_pending
4747 .iter()
4748 .map(|(member, _)| &member.petition)
4749 .collect();
4750 let first_entries: Vec<&LedgerEntry> = strict_pending.iter().map(|(_, entry)| entry).collect();
4751 let requested_shas: Vec<String> = petitions
4752 .iter()
4753 .map(|petition| petition.original_sha.clone())
4754 .collect();
4755 let prompt = strict_petition_batch_prompt(
4756 &batch_id,
4757 &petitions,
4758 &claim,
4759 &diff,
4760 context,
4761 &first_entries,
4762 );
4763 let mut strict_ready = Vec::new();
4764 match run_petition_batch_pass(
4765 queue.root.as_path(),
4766 strict_request,
4767 &prompt,
4768 &batch_id,
4769 &fix_sha,
4770 &requested_shas,
4771 runner,
4772 "strict-running",
4773 &strict_pending
4774 .iter()
4775 .map(|(member, _)| member)
4776 .collect::<Vec<_>>(),
4777 timeout,
4778 current_dir,
4779 ) {
4780 Ok((plan, mut parsed)) => {
4781 for (member, first_entry) in strict_pending {
4782 let original_sha = member.petition.original_sha.clone();
4783 match parsed
4784 .members
4785 .remove(&original_sha)
4786 .expect("strict batch parser omitted a requested member classification")
4787 {
4788 ParsedPetitionBatchMember::Valid(verdict) => {
4789 let strict_entry = petition_entry_from_batch_verdict(
4790 &fix_sha,
4791 &claim,
4792 &member.petition,
4793 &plan,
4794 &verdict,
4795 &member.run_id,
4796 member.batch_id.as_deref().unwrap_or(&batch_id),
4797 );
4798 let execution = ReviewExecution {
4799 entries: vec![first_entry, strict_entry],
4800 };
4801 run_store.persist_pending_review(
4802 &member.run_id,
4803 PendingReviewSpool {
4804 entries: execution.entries.clone(),
4805 petition_transition_pending: true,
4806 strict_pass_stdout: None,
4807 },
4808 )?;
4809 strict_ready.push(ReadyPetitionMember { member, execution });
4810 }
4811 ParsedPetitionBatchMember::Invalid(message) => {
4812 fail_petition_batch_member(
4813 queue,
4814 run_store,
4815 &member,
4816 message,
4817 true,
4818 &mut report,
4819 )?;
4820 }
4821 }
4822 }
4823 }
4824 Err(
4825 error @ (ReviewerError::VerdictJson { .. } | ReviewerError::BatchVerdictSchema { .. }),
4826 ) => {
4827 let message = error.to_string();
4828 for (member, _) in strict_pending {
4829 fail_petition_batch_member(queue, run_store, &member, &message, true, &mut report)?;
4830 }
4831 }
4832 Err(error) if is_retryable_provider_failure(&error) => {
4833 let message = error.to_string();
4834 for (member, _) in &strict_pending {
4835 run_store.mark_provider_backoff(
4837 &member.run_id,
4838 &message,
4839 DEFAULT_PROVIDER_BACKOFF_SECS,
4840 )?;
4841 }
4842 finish_ready_petition_members(queue, run_store, ready, store, config, &mut report)?;
4843 return Ok(report);
4844 }
4845 Err(error) => {
4846 let message = error.to_string();
4847 for (member, _) in &strict_pending {
4848 fail_petition_batch_member(queue, run_store, member, &message, false, &mut report)?;
4849 }
4850 finish_ready_petition_members(queue, run_store, ready, store, config, &mut report)?;
4851 return Err(error);
4852 }
4853 }
4854 ready.extend(strict_ready);
4855 finish_ready_petition_members(queue, run_store, ready, store, config, &mut report)?;
4856 Ok(report)
4857}
4858
4859fn dedupe_pending_for_batch(
4860 pending: &[QueuedReview],
4861 run_store: &ReviewRunStore,
4862) -> Result<Vec<QueuedReview>, ReviewerError> {
4863 let mut seen = std::collections::BTreeSet::new();
4864 let mut order = Vec::new();
4865 for item in pending {
4866 let identity = (item.commit_sha.clone(), item.petition_for.clone());
4867 if seen.insert(identity) {
4868 order.push(item.clone());
4869 } else if !item.run_id.trim().is_empty() {
4870 match run_store.read(&item.run_id) {
4871 Ok(run) if run.status == ReviewRunStatus::Queued => {
4872 run_store.cancel_queued(&item.run_id)?;
4873 }
4874 _ => {}
4875 }
4876 }
4877 }
4878 Ok(order)
4879}
4880
4881fn planned_job_commit_sha(job: &PlannedDrainJob) -> &str {
4882 match job {
4883 PlannedDrainJob::Single(item) => &item.commit_sha,
4884 PlannedDrainJob::PetitionBatch(items) => {
4885 &items
4886 .first()
4887 .expect("planned petition batch is non-empty")
4888 .commit_sha
4889 }
4890 }
4891}
4892
4893fn planned_job_worktree_name(job: &PlannedDrainJob, index: usize) -> String {
4894 let identity = match job {
4895 PlannedDrainJob::Single(item) if !item.run_id.trim().is_empty() => item.run_id.clone(),
4896 PlannedDrainJob::Single(item) => format!("commit-{}", item.commit_sha),
4897 PlannedDrainJob::PetitionBatch(items) => {
4898 let first = items.first().expect("planned petition batch is non-empty");
4899 let petition = first.petition_for.as_deref().unwrap_or("batch");
4900 format!("batch-{}-{}", first.commit_sha, petition)
4901 }
4902 };
4903 format!("{index}-{}", identity.replace(['/', '\\'], "_"))
4904}
4905
4906#[allow(clippy::too_many_arguments)]
4907fn drain_planned_job_in_dir<R: ProcessRunner, L: MaterialLoader>(
4908 queue: &ReviewQueue,
4909 job: PlannedDrainJob,
4910 loader: &L,
4911 selection: &ReviewSelection,
4912 context: &str,
4913 runner: &R,
4914 store: &LedgerStore,
4915 config: &config::TruthMirrorConfig,
4916 current_dir: Option<&Path>,
4917) -> Result<DrainReport, ReviewerError> {
4918 match job {
4919 PlannedDrainJob::Single(item) => drain_pending_serial_in_dir(
4920 queue,
4921 vec![item],
4922 loader,
4923 selection,
4924 context,
4925 runner,
4926 store,
4927 config,
4928 current_dir,
4929 ),
4930 PlannedDrainJob::PetitionBatch(items) => {
4931 let run_store = ReviewRunStore::new(&queue.root);
4932 drain_petition_batch(
4933 queue,
4934 &run_store,
4935 items,
4936 loader,
4937 selection,
4938 context,
4939 runner,
4940 store,
4941 config,
4942 current_dir,
4943 )
4944 }
4945 }
4946}
4947
4948#[allow(clippy::too_many_arguments)]
4949fn drain_pending_concurrent<R, L>(
4950 queue: &ReviewQueue,
4951 pending: Vec<QueuedReview>,
4952 loader: &L,
4953 selection: &ReviewSelection,
4954 context: &str,
4955 runner: &R,
4956 store: &LedgerStore,
4957 config: &config::TruthMirrorConfig,
4958) -> Result<DrainReport, ReviewerError>
4959where
4960 R: ProcessRunner + Sync,
4961 L: MaterialLoader + Sync,
4962{
4963 let repository = PathBuf::from(git_output(["rev-parse", "--show-toplevel"])?.trim());
4964 let run_store = ReviewRunStore::new(&queue.root);
4965 let pending = filter_pending_ready_for_attempt(pending, &run_store, unix_now())?;
4966 let planned = if config.reviewer.batch_petitions {
4967 let pending = dedupe_pending_for_batch(&pending, &run_store)?;
4968 plan_batched_drain(
4969 &pending,
4970 &queue.root,
4971 selection,
4972 context,
4973 config.reviewer.max_petition_batch_size,
4974 )
4975 } else {
4976 pending.into_iter().map(PlannedDrainJob::Single).collect()
4977 };
4978 let workers = config
4979 .reviewer
4980 .max_concurrent_runs
4981 .clamp(1, config::MAX_CONCURRENT_REVIEW_RUNS);
4982 let mut report = DrainReport::default();
4983
4984 for (wave_start, wave) in planned.chunks(workers).enumerate() {
4985 let mut claimed_wave = Vec::with_capacity(wave.len());
4988 for job in wave.iter().cloned() {
4989 claimed_wave.push(claim_planned_job(queue, &run_store, job)?);
4990 }
4991
4992 let (sender, receiver) = std::sync::mpsc::channel();
4993 std::thread::scope(|scope| {
4994 for (offset, job) in claimed_wave.into_iter().enumerate() {
4995 let index = wave_start * workers + offset;
4996 let worktree = queue
4997 .root
4998 .join("worktrees")
4999 .join(planned_job_worktree_name(&job, index));
5000 let commit_sha = planned_job_commit_sha(&job).to_owned();
5001 let cleanup_repository = repository.clone();
5002 let sender = sender.clone();
5003 scope.spawn(move || {
5004 let worktree = match create_review_worktree(
5008 &cleanup_repository,
5009 &worktree,
5010 &commit_sha,
5011 ) {
5012 Ok(path) => path,
5013 Err(error) => {
5014 let _ = sender.send((index, Err(error)));
5015 return;
5016 }
5017 };
5018 let worker_store = ReviewRunStore::new(&queue.root);
5019 let _ = record_job_worktree_paths(&worker_store, &job, &worktree);
5020 let result = drain_planned_job_in_dir(
5024 queue,
5025 job.clone(),
5026 loader,
5027 selection,
5028 context,
5029 runner,
5030 store,
5031 config,
5032 Some(worktree.as_path()),
5033 );
5034 clear_job_worktree_paths(&worker_store, &job);
5035 if let Err(error) = remove_review_worktree(&cleanup_repository, &worktree)
5036 && result.is_ok()
5037 {
5038 eprintln!(
5039 "truth-mirror watch: worktree cleanup failed after durable apply: {error}"
5040 );
5041 }
5042 let _ = sender.send((index, result));
5043 });
5044 }
5045 });
5046 drop(sender);
5047
5048 let mut outcomes = receiver.into_iter().collect::<Vec<_>>();
5049 outcomes.sort_by_key(|(index, _)| *index);
5050 let mut first_error = None;
5051 for (_, result) in outcomes {
5052 match result {
5053 Ok(completed) => {
5054 report.reviewed.extend(completed.reviewed);
5055 report.failed.extend(completed.failed);
5056 report.ledger_entries += completed.ledger_entries;
5057 }
5058 Err(error) if first_error.is_none() => first_error = Some(error),
5059 Err(_) => {}
5060 }
5061 }
5062 if let Some(error) = first_error {
5063 maintain_terminal_worktrees(&queue.root, &repository);
5064 return Err(error);
5065 }
5066 }
5067 maintain_terminal_worktrees(&queue.root, &repository);
5069 Ok(report)
5070}
5071
5072fn is_retryable_provider_failure(error: &ReviewerError) -> bool {
5073 match error {
5074 ReviewerError::ReviewerProcessFailed { status, stderr } => {
5075 if *status == Some(429) {
5076 return true;
5077 }
5078 let lowered = stderr.to_ascii_lowercase();
5079 lowered.contains("rate limit")
5080 || lowered.contains("rate_limit")
5081 || lowered.contains("ratelimit")
5082 || lowered.contains("too many requests")
5083 || lowered.contains("resource_exhausted")
5084 || lowered.contains("overloaded")
5085 || lowered.contains("temporarily unavailable")
5086 || lowered.contains("try again later")
5087 || lowered.contains("retry later")
5088 || lowered.contains(" 429")
5089 || lowered.contains("\"429\"")
5090 }
5091 _ => false,
5092 }
5093}
5094
5095fn filter_pending_ready_for_attempt(
5096 pending: Vec<QueuedReview>,
5097 run_store: &ReviewRunStore,
5098 now: u64,
5099) -> Result<Vec<QueuedReview>, ReviewerError> {
5100 let mut ready = Vec::with_capacity(pending.len());
5101 for item in pending {
5102 if item.run_id.trim().is_empty() {
5103 ready.push(item);
5104 continue;
5105 }
5106 match run_store.read(&item.run_id) {
5107 Ok(run) if run.backoff_blocks_attempt(now) => {
5108 tracing::info!(
5109 run_id = %item.run_id,
5110 next_attempt_at_unix = ?run.next_attempt_at_unix,
5111 "skipping queued review until provider backoff elapses"
5112 );
5113 }
5114 Ok(_) | Err(ReviewerError::ReviewRunNotFound { .. }) => ready.push(item),
5115 Err(error) => return Err(error),
5116 }
5117 }
5118 Ok(ready)
5119}
5120
5121fn claim_planned_job(
5122 queue: &ReviewQueue,
5123 run_store: &ReviewRunStore,
5124 job: PlannedDrainJob,
5125) -> Result<PlannedDrainJob, ReviewerError> {
5126 match job {
5127 PlannedDrainJob::Single(mut item) => {
5128 item = claim_queue_item(queue, run_store, item)?;
5129 Ok(PlannedDrainJob::Single(item))
5130 }
5131 PlannedDrainJob::PetitionBatch(items) => {
5132 let mut claimed = Vec::with_capacity(items.len());
5133 for item in items {
5134 claimed.push(claim_queue_item(queue, run_store, item)?);
5135 }
5136 Ok(PlannedDrainJob::PetitionBatch(claimed))
5137 }
5138 }
5139}
5140
5141fn claim_queue_item(
5142 queue: &ReviewQueue,
5143 run_store: &ReviewRunStore,
5144 mut item: QueuedReview,
5145) -> Result<QueuedReview, ReviewerError> {
5146 let sha = item.commit_sha.clone();
5147 let petition_for = item.petition_for.clone();
5148 let (run_id, _) = resolve_run_id_for_item(run_store, &item)?;
5149 if item.run_id.trim().is_empty() && !run_id.trim().is_empty() {
5150 queue.attach_run_id_for_item(&sha, petition_for.as_deref(), &run_id)?;
5151 item.run_id = run_id.clone();
5152 } else if item.run_id.trim().is_empty() {
5153 item.run_id = run_id.clone();
5154 }
5155 let target = if petition_for.is_some() {
5156 "petition"
5157 } else {
5158 "commit"
5159 };
5160 run_store.ensure_queued(&item.run_id, &sha, target, petition_for)?;
5161 let run = run_store.read(&item.run_id)?;
5162 if run.status == ReviewRunStatus::Completed
5163 || run.status == ReviewRunStatus::Cancelled
5164 || run_ledger_phase_applied(&run, REVIEW_LEDGER_PHASE)
5165 {
5166 return Ok(item);
5167 }
5168 if run.status == ReviewRunStatus::Running && run.phase != CLAIMED_PHASE {
5169 return Ok(item);
5170 }
5171 run_store.mark_claimed(&item.run_id)?;
5172 Ok(item)
5173}
5174
5175fn record_job_worktree_paths(
5176 run_store: &ReviewRunStore,
5177 job: &PlannedDrainJob,
5178 worktree: &Path,
5179) -> Result<(), ReviewerError> {
5180 let items = match job {
5181 PlannedDrainJob::Single(item) => std::slice::from_ref(item),
5182 PlannedDrainJob::PetitionBatch(items) => items.as_slice(),
5183 };
5184 let path = Some(worktree.display().to_string());
5185 for item in items {
5186 if item.run_id.trim().is_empty() {
5187 continue;
5188 }
5189 let _ = run_store.set_worktree_path(&item.run_id, path.clone());
5190 }
5191 Ok(())
5192}
5193
5194fn clear_job_worktree_paths(run_store: &ReviewRunStore, job: &PlannedDrainJob) {
5195 let items = match job {
5196 PlannedDrainJob::Single(item) => std::slice::from_ref(item),
5197 PlannedDrainJob::PetitionBatch(items) => items.as_slice(),
5198 };
5199 for item in items {
5200 if item.run_id.trim().is_empty() {
5201 continue;
5202 }
5203 let _ = run_store.set_worktree_path(&item.run_id, None);
5204 }
5205}
5206
5207fn maintain_terminal_worktrees(state_dir: &Path, repository: &Path) {
5210 let worktrees_root = state_dir.join("worktrees");
5211 let entries = match fs::read_dir(&worktrees_root) {
5212 Ok(entries) => entries,
5213 Err(error) if error.kind() == io::ErrorKind::NotFound => return,
5214 Err(error) => {
5215 eprintln!("truth-mirror: orphan worktree maintenance skipped (non-fatal): {error}");
5216 return;
5217 }
5218 };
5219 let run_store = ReviewRunStore::new(state_dir);
5220 let active = match run_store.list_lenient() {
5221 Ok((runs, _)) => runs
5222 .into_iter()
5223 .filter(|run| run.status == ReviewRunStatus::Running)
5224 .filter_map(|run| run.worktree_path)
5225 .collect::<std::collections::BTreeSet<_>>(),
5226 Err(error) => {
5227 eprintln!("truth-mirror: orphan worktree maintenance skipped (non-fatal): {error}");
5228 return;
5229 }
5230 };
5231 for entry in entries.flatten() {
5232 let path = entry.path();
5233 if !path.is_dir() {
5234 continue;
5235 }
5236 let key = path.display().to_string();
5237 if active.contains(&key) {
5238 continue;
5239 }
5240 match remove_review_worktree(repository, &path) {
5241 Ok(()) => {
5242 eprintln!(
5243 "truth-mirror: removed orphan review worktree (non-fatal maintenance): {key}"
5244 );
5245 }
5246 Err(error) => {
5247 eprintln!(
5248 "truth-mirror: orphan worktree cleanup failed (non-fatal): {key}: {error}"
5249 );
5250 }
5251 }
5252 }
5253}
5254
5255fn create_review_worktree(
5256 repository: &Path,
5257 worktree: &Path,
5258 commit_sha: &str,
5259) -> Result<PathBuf, ReviewerError> {
5260 if worktree.exists() {
5261 remove_review_worktree(repository, worktree)?;
5262 }
5263 if let Some(parent) = worktree.parent() {
5264 fs::create_dir_all(parent).map_err(ReviewerError::RunIo)?;
5265 }
5266 let output = Command::new("git")
5267 .current_dir(repository)
5268 .args(["worktree", "add", "--detach"])
5269 .arg(worktree)
5270 .arg(commit_sha)
5271 .output()
5272 .map_err(ReviewerError::GitSpawn)?;
5273 if !output.status.success() {
5274 return Err(ReviewerError::GitFailed {
5275 args: vec!["worktree".to_owned(), "add".to_owned()],
5276 stderr: String::from_utf8_lossy(&output.stderr).into_owned(),
5277 });
5278 }
5279 Ok(worktree.to_path_buf())
5280}
5281
5282fn remove_review_worktree(repository: &Path, worktree: &Path) -> Result<(), ReviewerError> {
5283 if !worktree.exists() {
5284 return Ok(());
5285 }
5286 let output = Command::new("git")
5287 .current_dir(repository)
5288 .args(["worktree", "remove", "--force"])
5289 .arg(worktree)
5290 .output()
5291 .map_err(ReviewerError::GitSpawn)?;
5292 if !output.status.success() {
5293 return Err(ReviewerError::GitFailed {
5294 args: vec!["worktree".to_owned(), "remove".to_owned()],
5295 stderr: String::from_utf8_lossy(&output.stderr).into_owned(),
5296 });
5297 }
5298 Ok(())
5299}
5300
5301pub fn drain_once<R: ProcessRunner, L: MaterialLoader>(
5302 queue: &ReviewQueue,
5303 loader: &L,
5304 selection: &ReviewSelection,
5305 context: &str,
5306 runner: &R,
5307 store: &LedgerStore,
5308 config: &config::TruthMirrorConfig,
5309) -> Result<DrainReport, ReviewerError> {
5310 let pending = queue.pending()?;
5311 if !config.reviewer.batch_petitions {
5312 return drain_pending_serial(
5313 queue, pending, loader, selection, context, runner, store, config,
5314 );
5315 }
5316 let run_store = ReviewRunStore::new(&queue.root);
5317 let pending = dedupe_pending_for_batch(&pending, &run_store)?;
5318 let planned = plan_batched_drain(
5319 &pending,
5320 &queue.root,
5321 selection,
5322 context,
5323 config.reviewer.max_petition_batch_size,
5324 );
5325 let mut report = DrainReport::default();
5326 for job in planned {
5327 let completed = match job {
5328 PlannedDrainJob::Single(item) => drain_pending_serial(
5329 queue,
5330 vec![item],
5331 loader,
5332 selection,
5333 context,
5334 runner,
5335 store,
5336 config,
5337 )?,
5338 PlannedDrainJob::PetitionBatch(items) => drain_petition_batch(
5339 queue, &run_store, items, loader, selection, context, runner, store, config, None,
5340 )?,
5341 };
5342 report.reviewed.extend(completed.reviewed);
5343 report.failed.extend(completed.failed);
5344 report.ledger_entries += completed.ledger_entries;
5345 }
5346 Ok(report)
5347}
5348
5349pub fn drain_once_parallel<R, L>(
5350 queue: &ReviewQueue,
5351 loader: &L,
5352 selection: &ReviewSelection,
5353 context: &str,
5354 runner: &R,
5355 store: &LedgerStore,
5356 config: &config::TruthMirrorConfig,
5357) -> Result<DrainReport, ReviewerError>
5358where
5359 R: ProcessRunner + Sync,
5360 L: MaterialLoader + Sync,
5361{
5362 drain_pending_concurrent(
5363 queue,
5364 queue.pending()?,
5365 loader,
5366 selection,
5367 context,
5368 runner,
5369 store,
5370 config,
5371 )
5372}
5373
5374#[allow(clippy::too_many_arguments)]
5375fn drain_pending_serial<R: ProcessRunner, L: MaterialLoader>(
5376 queue: &ReviewQueue,
5377 pending: Vec<QueuedReview>,
5378 loader: &L,
5379 selection: &ReviewSelection,
5380 context: &str,
5381 runner: &R,
5382 store: &LedgerStore,
5383 config: &config::TruthMirrorConfig,
5384) -> Result<DrainReport, ReviewerError> {
5385 drain_pending_serial_in_dir(
5386 queue, pending, loader, selection, context, runner, store, config, None,
5387 )
5388}
5389
5390#[allow(clippy::too_many_arguments)]
5391fn drain_pending_serial_in_dir<R: ProcessRunner, L: MaterialLoader>(
5392 queue: &ReviewQueue,
5393 pending: Vec<QueuedReview>,
5394 loader: &L,
5395 selection: &ReviewSelection,
5396 context: &str,
5397 runner: &R,
5398 store: &LedgerStore,
5399 config: &config::TruthMirrorConfig,
5400 current_dir: Option<&Path>,
5401) -> Result<DrainReport, ReviewerError> {
5402 let run_store = ReviewRunStore::new(&queue.root);
5403 let timeout = config.reviewer.timeout();
5406 let pending = filter_pending_ready_for_attempt(pending, &run_store, unix_now())?;
5407 let mut seen = std::collections::BTreeSet::new();
5412 let mut order = Vec::new();
5413 for item in &pending {
5414 let identity = (item.commit_sha.clone(), item.petition_for.clone());
5415 if seen.insert(identity) {
5416 order.push(item.clone());
5417 } else if !item.run_id.trim().is_empty() {
5418 match run_store.read(&item.run_id) {
5419 Ok(run) if run.status == ReviewRunStatus::Queued => {
5420 run_store.cancel_queued(&item.run_id)?;
5421 }
5422 _ => {}
5423 }
5424 }
5425 }
5426
5427 let mut report = DrainReport::default();
5428 for item in order {
5429 let sha = item.commit_sha.clone();
5430 let petition_for = item.petition_for.clone();
5431 let (run_id, created_new_run) = resolve_run_id_for_item(&run_store, &item)?;
5432 if item.run_id.trim().is_empty() && !run_id.trim().is_empty() {
5433 queue.attach_run_id_for_item(&sha, petition_for.as_deref(), &run_id)?;
5434 }
5435 let target = if petition_for.is_some() {
5436 "petition"
5437 } else {
5438 "commit"
5439 };
5440 let run = match run_store.ensure_queued(&run_id, &sha, target, petition_for.clone()) {
5441 Ok(run) => run,
5442 Err(ReviewerError::RunJson(_)) => {
5443 queue.remove_item(&sha, petition_for.as_deref())?;
5444 report.failed.push(sha);
5445 continue;
5446 }
5447 Err(error) => return Err(error),
5448 };
5449 let cancel_new_run = || {
5450 if created_new_run {
5451 let _ = run_store.cancel_queued(&run_id);
5452 }
5453 };
5454 if run.status == ReviewRunStatus::Cancelled {
5455 cancel_new_run();
5456 queue.remove_item(&sha, item.petition_for.as_deref())?;
5457 continue;
5458 }
5459 if run.status == ReviewRunStatus::Completed {
5460 finish_replayed_queue_item(
5461 queue,
5462 &run_store,
5463 &run_id,
5464 &sha,
5465 item.petition_for.as_deref(),
5466 run.ledger_entries,
5467 &mut report,
5468 )?;
5469 continue;
5470 }
5471 if run_ledger_phase_applied(&run, REVIEW_LEDGER_PHASE) {
5472 finish_replayed_queue_item(
5473 queue,
5474 &run_store,
5475 &run_id,
5476 &sha,
5477 item.petition_for.as_deref(),
5478 run.ledger_entries,
5479 &mut report,
5480 )?;
5481 continue;
5482 }
5483 let (claim, diff) = match loader.load(&sha) {
5484 Ok(material) => material,
5485 Err(error) => {
5486 cancel_new_run();
5487 run_store.mark_failed(&run_id, error.to_string())?;
5488 queue.remove_item(&sha, item.petition_for.as_deref())?;
5489 report.failed.push(sha);
5490 continue;
5491 }
5492 };
5493 let petition = match &item.petition_for {
5497 Some(original_sha) => {
5498 if !petition_target_is_actionable(store, original_sha)? {
5499 let notice = format!(
5504 "petition target {original_sha} has no open rejection (already resolved or unknown); skipping queued petition for {sha}"
5505 );
5506 eprintln!("truth-mirror watch: {notice}");
5507 run_store.mark_failed(&run_id, format!("skipped: {notice}"))?;
5508 queue.remove_item(&sha, item.petition_for.as_deref())?;
5509 report.failed.push(sha);
5510 continue;
5511 }
5512 Some(petition_context_from_ledger(original_sha, &sha, store)?)
5513 }
5514 None => None,
5515 };
5516 let prompt = first_pass_prompt(&claim, &diff, context);
5517 let job = materialize_petition_prompt(ReviewJob {
5518 commit_sha: sha.clone(),
5519 claim,
5520 diff,
5521 context: context.to_owned(),
5522 request: selection.request_for(prompt),
5523 strict: selection.strict.clone(),
5524 petition,
5525 });
5526 let execution = {
5527 if run.status != ReviewRunStatus::Running && run.phase != CLAIMED_PHASE {
5529 run_store.mark_claimed(&run_id)?;
5530 }
5531 if run.status != ReviewRunStatus::Running || run.phase != REVIEW_LEDGER_PHASE {
5532 run_store.mark_running(&run_id, REVIEW_LEDGER_PHASE)?;
5533 }
5534 match complete_review_execution_in_dir(
5535 &job,
5536 runner,
5537 &run_store,
5538 &run_id,
5539 run.pending_review,
5540 timeout,
5541 current_dir,
5542 ) {
5543 Ok(execution) => execution,
5544 Err(
5545 error @ (ReviewerError::VerdictJson { .. }
5546 | ReviewerError::VerdictSchema { .. }),
5547 ) => {
5548 run_store.mark_failed(&run_id, error.to_string())?;
5549 queue.remove_item(&sha, item.petition_for.as_deref())?;
5550 report.failed.push(sha);
5551 continue;
5552 }
5553 Err(
5559 error @ (ReviewerError::ReviewerTimeout { .. }
5560 | ReviewerError::ReviewerOutputWedged { .. }
5561 | ReviewerError::KillReviewer { .. }),
5562 ) => {
5563 run_store.mark_failed(&run_id, error.to_string())?;
5564 queue.remove_item(&sha, item.petition_for.as_deref())?;
5565 report.failed.push(sha);
5566 continue;
5567 }
5568 Err(error) if is_retryable_provider_failure(&error) => {
5569 run_store.mark_provider_backoff(
5571 &run_id,
5572 error.to_string(),
5573 DEFAULT_PROVIDER_BACKOFF_SECS,
5574 )?;
5575 continue;
5576 }
5577 Err(error) => {
5578 let _ = run_store.mark_failed(&run_id, error.to_string());
5579 return Err(error);
5580 }
5581 }
5582 };
5583 apply_review_execution(&job, &execution, store)?;
5588 run_store.mark_ledger_applied(&run_id, REVIEW_LEDGER_PHASE, execution.entries.len())?;
5589 run_store.clear_pending_review(&run_id)?;
5590 record_memory_skill_outcome(
5591 &queue.root,
5592 config,
5593 &run_id,
5594 &sha,
5595 "watch-drain",
5596 &execution.entries,
5597 );
5598 report.ledger_entries += execution.entries.len();
5599 run_store.mark_completed(&run_id, execution.entries.len())?;
5600 queue.remove_item(&sha, item.petition_for.as_deref())?;
5603 report.reviewed.push(sha);
5604 }
5605
5606 Ok(report)
5607}
5608
5609fn record_memory_skill_outcome(
5610 state_dir: &Path,
5611 config: &config::TruthMirrorConfig,
5612 run_id: &str,
5613 commit_sha: &str,
5614 phase: &str,
5615 entries: &[LedgerEntry],
5616) {
5617 if !is_full_git_sha(commit_sha) {
5618 tracing::debug!(
5619 run_id = %run_id,
5620 commit_sha = %commit_sha,
5621 phase = %phase,
5622 "memory-skill extraction skipped for non-commit review target"
5623 );
5624 return;
5625 }
5626 match crate::memory_skill::evaluate_review_completion(state_dir, config, run_id, entries) {
5627 Ok(report) => {
5628 let outcome = if !report.staged.is_empty() {
5629 "staged"
5630 } else if !report.advised.is_empty() {
5631 "advised"
5632 } else {
5633 "skipped"
5634 };
5635 tracing::info!(
5636 run_id = %run_id,
5637 commit_sha = %commit_sha,
5638 phase = %phase,
5639 memory_skill_outcome = outcome,
5640 skipped_reason = report.skipped_reason.as_deref().unwrap_or(""),
5641 occurrence_count = report.occurrence_count,
5642 nearest_cluster_size = report.nearest_cluster_size,
5643 staged_count = report.staged.len(),
5644 advised_count = report.advised.len(),
5645 "memory-skill evaluation completed"
5646 );
5647 }
5648 Err(crate::memory_skill::MemorySkillError::ScanRejected { reason }) => {
5649 tracing::info!(
5650 run_id = %run_id,
5651 commit_sha = %commit_sha,
5652 phase = %phase,
5653 memory_skill_outcome = "scan_rejected",
5654 skipped_reason = %reason,
5655 reason = %reason,
5656 "memory-skill extraction skipped by scan gate"
5657 );
5658 }
5659 Err(error) => {
5660 let error_kind = memory_skill_error_kind(&error);
5661 tracing::warn!(
5662 run_id = %run_id,
5663 commit_sha = %commit_sha,
5664 phase = %phase,
5665 memory_skill_outcome = "failed",
5666 memory_skill_error_kind = error_kind,
5667 error = %error,
5668 "memory-skill extraction failed after review completion"
5669 );
5670 }
5671 }
5672}
5673
5674fn memory_skill_error_kind(error: &crate::memory_skill::MemorySkillError) -> &'static str {
5675 match error {
5676 crate::memory_skill::MemorySkillError::Io(_) => "io",
5677 crate::memory_skill::MemorySkillError::Json(_) => "json",
5678 crate::memory_skill::MemorySkillError::Ledger(_) => "ledger",
5679 crate::memory_skill::MemorySkillError::CandidateNotFound { .. } => "candidate_not_found",
5680 crate::memory_skill::MemorySkillError::AdvisoryNotFound { .. } => "advisory_not_found",
5681 crate::memory_skill::MemorySkillError::EmptyRejectReason => "empty_reject_reason",
5682 crate::memory_skill::MemorySkillError::EmptySupersedeReason => "empty_supersede_reason",
5683 crate::memory_skill::MemorySkillError::SelfSupersede { .. } => "self_supersede",
5684 crate::memory_skill::MemorySkillError::InvalidSupersedeReplacement { .. } => {
5685 "invalid_supersede_replacement"
5686 }
5687 crate::memory_skill::MemorySkillError::InvalidTransition { .. } => "invalid_transition",
5688 crate::memory_skill::MemorySkillError::TransitionLocked { .. } => "transition_locked",
5689 crate::memory_skill::MemorySkillError::UnsafeGlobalWrite { .. } => "unsafe_global_write",
5690 crate::memory_skill::MemorySkillError::UnsafeApprovedPath { .. } => "unsafe_approved_path",
5691 crate::memory_skill::MemorySkillError::UnsafeStatePath { .. } => "unsafe_state_path",
5692 crate::memory_skill::MemorySkillError::ScanRejected { .. } => "scan_rejected",
5693 crate::memory_skill::MemorySkillError::RenderedSkillTooLarge { .. } => {
5694 "rendered_skill_too_large"
5695 }
5696 }
5697}
5698
5699fn review_context(config: &config::TruthMirrorConfig) -> String {
5702 let repo_root = match git_output(["rev-parse", "--show-toplevel"]) {
5703 Ok(root) => PathBuf::from(root.trim()),
5704 Err(_) => return String::new(),
5705 };
5706 let provider = crate::context::trajectory_provider(&repo_root, &config.history);
5707 crate::context::build_review_context(
5708 &repo_root,
5709 &config.ground_truth,
5710 &config.history,
5711 Some(provider.as_ref()),
5712 )
5713 .unwrap_or_default()
5714}
5715
5716pub fn run_watch_command(
5717 args: cli::WatchArgs,
5718 state_dir: &Path,
5719 config: &config::TruthMirrorConfig,
5720) -> Result<ExitCode> {
5721 if args.wait_for_lock && !args.once && !args.until_empty {
5722 anyhow::bail!("--wait-for-lock is only supported with --once or --until-empty");
5723 }
5724 let selection = ReviewSelection::resolve(
5725 args.watched_agent,
5726 args.watched_model,
5727 args.reviewer_harness,
5728 args.reviewer_model,
5729 args.reviewer_effort,
5730 args.allow_same_model,
5731 config,
5732 )?;
5733 let queue = ReviewQueue::new(state_dir);
5734 let store = LedgerStore::new(state_dir);
5735 let loader = GitMaterialLoader::from_config(config);
5736 let runner = StdProcessRunner;
5737
5738 let interval = std::time::Duration::from_secs(args.poll_secs.max(1));
5739
5740 match claim_watcher_lock(
5750 state_dir,
5751 args.wait_for_lock,
5752 interval,
5753 args.expect_lock_handoff,
5754 ) {
5755 Ok(WatchLockClaim::Owned) => {}
5756 Ok(WatchLockClaim::Refused(reason)) => {
5757 eprintln!("truth-mirror watch: {reason}");
5758 return Ok(ExitCode::FAILURE);
5759 }
5760 Err(error) => {
5761 eprintln!(
5762 "truth-mirror watch: failed to acquire the watcher lock ({error}); refusing to process the queue without lock ownership"
5763 );
5764 return Ok(ExitCode::FAILURE);
5765 }
5766 }
5767 let watcher_lock_token = crate::watcher::lock_acquisition_token(state_dir).unwrap_or_default();
5768
5769 let _ = ReviewRunStore::new(state_dir).reconcile_stale_runs();
5773
5774 if args.once {
5775 let context = review_context(config);
5776 let result = if config.reviewer.max_concurrent_runs > 1 {
5777 drain_once_parallel(
5778 &queue, &loader, &selection, &context, &runner, &store, config,
5779 )
5780 } else {
5781 drain_once(
5782 &queue, &loader, &selection, &context, &runner, &store, config,
5783 )
5784 };
5785 let _ = crate::watcher::release_lock_if_owned(state_dir, &watcher_lock_token);
5789 let report = result?;
5790 println!(
5791 "truth-mirror watch: reviewed {} commit(s), skipped {} failed item(s), wrote {} ledger entrie(s)",
5792 report.reviewed.len(),
5793 report.failed.len(),
5794 report.ledger_entries
5795 );
5796 return Ok(ExitCode::SUCCESS);
5797 }
5798
5799 if args.until_empty {
5800 return run_until_empty(
5801 &queue,
5802 &loader,
5803 &selection,
5804 &runner,
5805 &store,
5806 config,
5807 state_dir,
5808 args.grace,
5809 interval,
5810 &watcher_lock_token,
5811 );
5812 }
5813
5814 let outcome = loop {
5815 let context = review_context(config);
5817 let report = match if config.reviewer.max_concurrent_runs > 1 {
5818 drain_once_parallel(
5819 &queue, &loader, &selection, &context, &runner, &store, config,
5820 )
5821 } else {
5822 drain_once(
5823 &queue, &loader, &selection, &context, &runner, &store, config,
5824 )
5825 } {
5826 Ok(report) => report,
5827 Err(error) => break Err(error),
5828 };
5829 if !report.reviewed.is_empty() {
5830 println!(
5831 "truth-mirror watch: reviewed {} commit(s)",
5832 report.reviewed.len()
5833 );
5834 }
5835 if !report.failed.is_empty() {
5836 eprintln!(
5837 "truth-mirror watch: skipped {} failed queue item(s)",
5838 report.failed.len()
5839 );
5840 }
5841 std::thread::sleep(interval);
5842 };
5843 let _ = crate::watcher::release_lock_if_owned(state_dir, &watcher_lock_token);
5846 outcome?;
5847 Ok(ExitCode::SUCCESS)
5848}
5849
5850fn watcher_lock_details(state_dir: &Path) -> String {
5851 crate::watcher::lock_description(state_dir).unwrap_or_else(|error| {
5852 format!(
5853 "state=unavailable path={} error={error}",
5854 state_dir.join(crate::watcher::WATCHER_LOCK_FILE).display()
5855 )
5856 })
5857}
5858
5859#[derive(Clone, Debug, Eq, PartialEq)]
5861enum WatchLockClaim {
5862 Owned,
5865 Refused(String),
5868}
5869
5870fn claim_watcher_lock(
5888 state_dir: &Path,
5889 wait_for_lock: bool,
5890 interval: std::time::Duration,
5891 expect_handoff: bool,
5892) -> Result<WatchLockClaim, crate::watcher::WatcherError> {
5893 let mut announced_wait = false;
5894 let handoff_deadline = expect_handoff.then(|| Instant::now() + HANDOFF_GRACE);
5895 let mut announced_handoff_wait = false;
5896 loop {
5897 match crate::watcher::try_acquire_lock(state_dir)? {
5898 crate::watcher::LockClaim::Acquired => return Ok(WatchLockClaim::Owned),
5899 crate::watcher::LockClaim::HeldByLiveWatcher => {
5900 if crate::watcher::lock_owned_by_current_process(state_dir) {
5901 return Ok(WatchLockClaim::Owned);
5903 }
5904 if let Some(deadline) = handoff_deadline
5905 && Instant::now() < deadline
5906 {
5907 if !announced_handoff_wait {
5908 tracing::info!("watch: waiting for the ensure-watcher lock handoff");
5909 announced_handoff_wait = true;
5910 }
5911 std::thread::sleep(HANDOFF_POLL_INTERVAL);
5912 continue;
5913 }
5914 if !wait_for_lock {
5915 let details = watcher_lock_details(state_dir);
5916 return Ok(WatchLockClaim::Refused(format!(
5917 "watcher lock is already held; {details}; refusing to touch the queue \
5918 or process it without lock ownership (pass --wait-for-lock to wait \
5919 for the slot)"
5920 )));
5921 }
5922 if !announced_wait {
5923 let details = watcher_lock_details(state_dir);
5924 eprintln!(
5925 "truth-mirror watch: another live watcher holds the lock ({details}); \
5926 waiting for it (--wait-for-lock)"
5927 );
5928 announced_wait = true;
5929 }
5930 std::thread::sleep(interval);
5931 }
5932 }
5933 }
5934}
5935
5936const HANDOFF_GRACE: Duration = Duration::from_secs(crate::watcher::LOCK_SETTLING_SECS + 2);
5942
5943const HANDOFF_POLL_INTERVAL: Duration = Duration::from_millis(100);
5946
5947#[derive(Clone, Copy, Debug, Eq, PartialEq)]
5955pub enum UntilEmptyDecision {
5956 KeepWatching,
5959 Exit,
5961}
5962
5963pub fn until_empty_decision(empty_for_secs: Option<u64>, grace_secs: u64) -> UntilEmptyDecision {
5964 match empty_for_secs {
5965 Some(elapsed) if elapsed >= grace_secs => UntilEmptyDecision::Exit,
5966 _ => UntilEmptyDecision::KeepWatching,
5967 }
5968}
5969
5970#[derive(Clone, Debug, Eq, PartialEq)]
5971enum QueueFingerprint {
5972 Missing,
5973 Present {
5974 len: u64,
5975 modified: Option<SystemTime>,
5976 },
5977}
5978#[derive(Clone, Debug, Default)]
5979struct QueueEmptyCache {
5980 fingerprint: Option<QueueFingerprint>,
5981 pending_count: Option<usize>,
5982}
5983
5984fn queue_fingerprint(queue: &ReviewQueue) -> Result<QueueFingerprint, ReviewerError> {
5985 match fs::metadata(queue.path()) {
5986 Ok(metadata) => Ok(QueueFingerprint::Present {
5987 len: metadata.len(),
5988 modified: metadata.modified().ok(),
5989 }),
5990 Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(QueueFingerprint::Missing),
5991 Err(error) => Err(ReviewerError::QueueIo(error)),
5992 }
5993}
5994fn queue_has_pending_cached(
5995 queue: &ReviewQueue,
5996 cache: &mut QueueEmptyCache,
5997) -> Result<bool, ReviewerError> {
5998 let fingerprint = queue_fingerprint(queue)?;
5999 if matches!(
6000 fingerprint,
6001 QueueFingerprint::Missing | QueueFingerprint::Present { len: 0, .. }
6002 ) {
6003 cache.fingerprint = Some(fingerprint);
6004 cache.pending_count = Some(0);
6005 return Ok(false);
6006 }
6007
6008 if let (true, Some(pending_count)) = (
6009 cache.fingerprint.as_ref() == Some(&fingerprint),
6010 cache.pending_count,
6011 ) {
6012 return Ok(pending_count > 0);
6013 }
6014
6015 let pending_count = queue.summary()?.pending_count;
6016 cache.fingerprint = Some(fingerprint);
6017 cache.pending_count = Some(pending_count);
6018 Ok(pending_count > 0)
6019}
6020
6021#[allow(clippy::too_many_arguments)]
6029fn run_until_empty<R: ProcessRunner + Sync, L: MaterialLoader + Sync>(
6030 queue: &ReviewQueue,
6031 loader: &L,
6032 selection: &ReviewSelection,
6033 runner: &R,
6034 store: &LedgerStore,
6035 config: &config::TruthMirrorConfig,
6036 state_dir: &Path,
6037 grace_secs: u64,
6038 interval: std::time::Duration,
6039 watcher_lock_token: &str,
6040) -> Result<ExitCode> {
6041 let outcome = (|| -> Result<()> {
6042 let mut empty_since: Option<u64> = None;
6043 let mut queue_empty_cache = QueueEmptyCache::default();
6044 loop {
6045 let context = review_context(config);
6046 let report = if config.reviewer.max_concurrent_runs > 1 {
6047 drain_once_parallel(queue, loader, selection, &context, runner, store, config)?
6048 } else {
6049 drain_once(queue, loader, selection, &context, runner, store, config)?
6050 };
6051 if !report.reviewed.is_empty() {
6052 println!(
6053 "truth-mirror watch: reviewed {} commit(s)",
6054 report.reviewed.len()
6055 );
6056 }
6057 if !report.failed.is_empty() {
6058 eprintln!(
6059 "truth-mirror watch: skipped {} failed queue item(s)",
6060 report.failed.len()
6061 );
6062 }
6063
6064 let now = unix_now();
6065 if !queue_has_pending_cached(queue, &mut queue_empty_cache)? {
6066 let started = *empty_since.get_or_insert(now);
6067 let empty_for = now.saturating_sub(started);
6068 if until_empty_decision(Some(empty_for), grace_secs) == UntilEmptyDecision::Exit {
6069 return Ok(());
6070 }
6071 } else {
6072 empty_since = None;
6074 }
6075
6076 std::thread::sleep(interval);
6077 }
6078 })();
6079
6080 let _ = crate::watcher::release_lock_if_owned(state_dir, watcher_lock_token);
6083 outcome?;
6084 Ok(ExitCode::SUCCESS)
6085}
6086
6087pub fn run_ensure_watcher_command(
6093 _args: cli::EnsureWatcherArgs,
6094 state_dir: &Path,
6095) -> Result<ExitCode> {
6096 match crate::watcher::ensure_watcher(state_dir)? {
6097 crate::watcher::LockClaim::Acquired => {
6098 println!("truth-mirror: watcher started");
6099 }
6100 crate::watcher::LockClaim::HeldByLiveWatcher => {
6101 tracing::debug!("ensure-watcher: live watcher already owns the state dir");
6102 }
6103 }
6104 Ok(ExitCode::SUCCESS)
6105}
6106
6107#[derive(Clone, Copy, Debug, Eq, PartialEq)]
6108pub struct StrictGoalPolicy {
6109 pub stop_after_lies: u32,
6110 pub stop_after_fuckups: u32,
6111}
6112
6113#[derive(Clone, Copy, Debug, Eq, PartialEq)]
6114pub struct StrictGoalCounters {
6115 pub lies_exposed: u32,
6116 pub fuckups_registered: u32,
6117}
6118
6119#[derive(Clone, Copy, Debug, Eq, PartialEq)]
6120pub enum StrictGoalDecision {
6121 Continue,
6122 Stop { reason: StrictGoalStopReason },
6123}
6124
6125#[derive(Clone, Copy, Debug, Eq, PartialEq)]
6126pub enum StrictGoalStopReason {
6127 LiesExposed,
6128 FuckupsRegistered,
6129}
6130
6131impl StrictGoalPolicy {
6132 pub fn decide(&self, counters: StrictGoalCounters) -> StrictGoalDecision {
6133 if self.stop_after_lies > 0 && counters.lies_exposed >= self.stop_after_lies {
6134 return StrictGoalDecision::Stop {
6135 reason: StrictGoalStopReason::LiesExposed,
6136 };
6137 }
6138
6139 if self.stop_after_fuckups > 0 && counters.fuckups_registered >= self.stop_after_fuckups {
6140 return StrictGoalDecision::Stop {
6141 reason: StrictGoalStopReason::FuckupsRegistered,
6142 };
6143 }
6144
6145 StrictGoalDecision::Continue
6146 }
6147}
6148
6149#[derive(Clone, Debug, Eq, PartialEq)]
6150pub struct StrictGoalOutcome {
6151 pub passes: u32,
6152 pub counters: StrictGoalCounters,
6153 pub stop_reason: Option<StrictGoalStopReason>,
6156 pub entries: Vec<LedgerEntry>,
6157}
6158
6159impl StrictGoalOutcome {
6160 pub fn stop_reason_suffix(&self) -> &'static str {
6161 match self.stop_reason {
6162 Some(StrictGoalStopReason::LiesExposed) => " (stopped: lies exposed)",
6163 Some(StrictGoalStopReason::FuckupsRegistered) => " (stopped: fuckups registered)",
6164 None => " (stopped: max passes)",
6165 }
6166 }
6167}
6168
6169#[allow(clippy::too_many_arguments)]
6174pub fn run_strict_goal_loop<R: ProcessRunner>(
6175 commit_sha: &str,
6176 claim: &Claim,
6177 diff: &str,
6178 context: &str,
6179 selection: &ReviewSelection,
6180 policy: StrictGoalPolicy,
6181 max_passes: u32,
6182 runner: &R,
6183 store: &LedgerStore,
6184 timeout: Option<Duration>,
6185) -> Result<StrictGoalOutcome, ReviewerError> {
6186 let ceiling = max_passes.max(1);
6187 let mut outcome = StrictGoalOutcome {
6188 passes: 0,
6189 counters: StrictGoalCounters {
6190 lies_exposed: 0,
6191 fuckups_registered: 0,
6192 },
6193 stop_reason: None,
6194 entries: Vec::new(),
6195 };
6196
6197 while outcome.passes < ceiling {
6198 let prompt = strict_goal_prompt(claim, diff, context, outcome.passes + 1, &outcome.entries);
6199 let request = selection.request_for(prompt);
6200 let plan = ReviewPlan::build(request.clone())?;
6201 let output = plan.run_with(&request.prompt, runner, timeout)?;
6202 ensure_process_success(&output)?;
6203 let verdict = ParsedVerdict::parse(&output.stdout)?;
6204
6205 let job = ReviewJob {
6206 commit_sha: commit_sha.to_owned(),
6207 claim: claim.clone(),
6208 diff: diff.to_owned(),
6209 context: context.to_owned(),
6210 request,
6211 strict: None,
6212 petition: None,
6213 };
6214 let entry = entry_from_verdict(&job, &plan, &verdict);
6215 store.append_entry(&entry)?;
6216 outcome.entries.push(entry);
6217
6218 outcome.passes += 1;
6219 if verdict.verdict == Verdict::Reject {
6220 outcome.counters.lies_exposed += 1;
6221 }
6222 outcome.counters.fuckups_registered = outcome
6223 .counters
6224 .fuckups_registered
6225 .saturating_add(u32::try_from(verdict.findings.len()).unwrap_or(u32::MAX));
6226
6227 if let StrictGoalDecision::Stop { reason } = policy.decide(outcome.counters) {
6228 outcome.stop_reason = Some(reason);
6229 break;
6230 }
6231 }
6232
6233 Ok(outcome)
6234}
6235
6236fn strict_goal_prompt(
6237 claim: &Claim,
6238 diff: &str,
6239 context: &str,
6240 pass: u32,
6241 prior: &[LedgerEntry],
6242) -> String {
6243 let prior_findings: Vec<String> = prior
6244 .iter()
6245 .flat_map(|entry| entry.findings.clone())
6246 .collect();
6247 let prior_block = if prior_findings.is_empty() {
6248 "(none)".to_owned()
6249 } else {
6250 prior_findings.join("\n")
6251 };
6252 format!(
6253 "{ADVERSARIAL_PREAMBLE}\n\nStrict-goal loop, pass {pass}. Keep hunting for any lie the claim hides; do not repeat prior findings verbatim.{}\n\nCLAIM:\n{}\n\nPRIOR FINDINGS:\n{prior_block}\n\nDIFF:\n{}",
6254 context_block(context),
6255 claim.to_line(),
6256 diff
6257 )
6258}
6259
6260pub fn run_review_command(
6261 args: cli::ReviewArgs,
6262 state_dir: &Path,
6263 config: &config::TruthMirrorConfig,
6264) -> Result<ExitCode> {
6265 if let Some(command) = args.command {
6266 return run_review_run_command(command, state_dir);
6267 }
6268
6269 let material = ReviewMaterial::load(
6270 &args,
6271 state_dir,
6272 &config.gates.to_policy().evidence_patterns,
6273 )?;
6274
6275 let mut selection = ReviewSelection::resolve(
6276 args.watched_agent,
6277 args.watched_model,
6278 args.reviewer_harness,
6279 args.reviewer_model,
6280 args.reviewer_effort,
6281 args.allow_same_model,
6282 config,
6283 )?;
6284
6285 if args.strict_two_pass {
6286 selection.strict = Some(ReviewSelection::resolve_arbiter(
6287 selection.watched_agent,
6288 args.arbiter_harness,
6289 args.arbiter_model,
6290 args.arbiter_effort,
6291 config,
6292 )?);
6293 }
6294 let store = LedgerStore::new(state_dir);
6295 let run_store = ReviewRunStore::new(state_dir);
6296 let context = review_context(config);
6297 let run = run_store.create_queued(&material.commit_sha, material.target_label.clone())?;
6298 run_store.mark_running(&run.id, "reviewing")?;
6299
6300 if args.strict_goal {
6301 let policy = config
6302 .strict
6303 .goal_policy(args.stop_after_lies, args.stop_after_fuckups);
6304 let max_passes = args.max_passes.unwrap_or(config.strict.max_passes);
6305 let outcome = match run_strict_goal_loop(
6306 &material.commit_sha,
6307 &material.claim,
6308 &material.diff,
6309 &context,
6310 &selection,
6311 policy,
6312 max_passes,
6313 &StdProcessRunner,
6314 &store,
6315 config.reviewer.timeout(),
6316 ) {
6317 Ok(outcome) => outcome,
6318 Err(error) => {
6319 let _ = run_store.mark_failed(&run.id, error.to_string());
6320 return Err(error.into());
6321 }
6322 };
6323 record_memory_skill_outcome(
6324 state_dir,
6325 config,
6326 &run.id,
6327 &material.commit_sha,
6328 "strict-goal",
6329 &outcome.entries,
6330 );
6331 run_store.mark_completed(&run.id, outcome.entries.len())?;
6332 println!(
6333 "truth-mirror strict-goal: run {}, {} pass(es), {} lie(s), {} fuckup(s){}",
6334 run.id,
6335 outcome.passes,
6336 outcome.counters.lies_exposed,
6337 outcome.counters.fuckups_registered,
6338 outcome.stop_reason_suffix(),
6339 );
6340 return Ok(ExitCode::SUCCESS);
6341 }
6342
6343 let prompt = first_pass_prompt(&material.claim, &material.diff, &context);
6344 let commit_sha = material.commit_sha.clone();
6345 let job = ReviewJob {
6346 commit_sha: material.commit_sha,
6347 claim: material.claim,
6348 diff: material.diff,
6349 context,
6350 request: selection.request_for(prompt),
6351 strict: selection.strict.clone(),
6352 petition: None,
6353 };
6354
6355 let strict_two_pass = selection.strict.is_some();
6356 let job = materialize_petition_prompt(job);
6357 let timeout = config.reviewer.timeout();
6358 let execution = if strict_two_pass {
6359 run_store.mark_running(&run.id, REVIEW_LEDGER_PHASE)?;
6360 let run = run_store.read(&run.id)?;
6361 match complete_review_execution(
6362 &job,
6363 &StdProcessRunner,
6364 &run_store,
6365 &run.id,
6366 run.pending_review,
6367 timeout,
6368 ) {
6369 Ok(execution) => execution,
6370 Err(error) => {
6371 let _ = run_store.mark_failed(&run.id, error.to_string());
6372 return Err(error.into());
6373 }
6374 }
6375 } else {
6376 match execute_review_job(job.clone(), &StdProcessRunner, &store, timeout) {
6377 Ok(execution) => execution,
6378 Err(error) => {
6379 let _ = run_store.mark_failed(&run.id, error.to_string());
6380 return Err(error.into());
6381 }
6382 }
6383 };
6384 if strict_two_pass {
6385 apply_review_execution(&job, &execution, &store)?;
6386 run_store.mark_ledger_applied(&run.id, REVIEW_LEDGER_PHASE, execution.entries.len())?;
6387 run_store.clear_pending_review(&run.id)?;
6388 }
6389 record_memory_skill_outcome(
6390 state_dir,
6391 config,
6392 &run.id,
6393 &commit_sha,
6394 if strict_two_pass {
6395 "manual-review-strict"
6396 } else {
6397 "manual-review"
6398 },
6399 &execution.entries,
6400 );
6401 run_store.mark_completed(&run.id, execution.entries.len())?;
6402 println!(
6403 "truth-mirror review: run {}, wrote {} ledger entrie(s)",
6404 run.id,
6405 execution.entries.len()
6406 );
6407 Ok(ExitCode::SUCCESS)
6408}
6409
6410fn run_review_run_command(command: cli::ReviewCommand, state_dir: &Path) -> Result<ExitCode> {
6411 let runs = ReviewRunStore::new(state_dir);
6412 match command {
6413 cli::ReviewCommand::Status { run_id } => {
6414 if let Some(run_id) = run_id {
6415 print_run(&runs.read_reconciled(&run_id)?);
6416 } else {
6417 let all = runs.list_reconciled()?;
6418 if all.is_empty() {
6419 println!("No review runs.");
6420 } else {
6421 for run in all {
6422 print_run_summary(&run);
6423 }
6424 }
6425 }
6426 }
6427 cli::ReviewCommand::Result { run_id } => {
6428 let run = match run_id {
6429 Some(run_id) => runs.read(&run_id)?,
6430 None => runs.latest_result()?,
6431 };
6432 print_run(&run);
6433 print_run_ledger_entries(state_dir, &run)?;
6434 }
6435 cli::ReviewCommand::Cancel { run_id, force } => {
6436 let run = runs.cancel(&run_id, force)?;
6437 ReviewQueue::new(state_dir).remove_run_id(&run_id)?;
6438 match run.status {
6439 ReviewRunStatus::Failed => println!(
6440 "reaped stale review run {} ({}): {}",
6441 run.id,
6442 run.commit_sha,
6443 run.error.as_deref().unwrap_or("worker was not alive"),
6444 ),
6445 _ => println!("cancelled review run {} ({})", run.id, run.commit_sha),
6446 }
6447 }
6448 }
6449 Ok(ExitCode::SUCCESS)
6450}
6451
6452fn print_run_summary(run: &ReviewRun) {
6453 println!(
6454 "{} {} {} {} entries={} updated={}",
6455 run.id, run.status, run.commit_sha, run.phase, run.ledger_entries, run.updated_at_unix
6456 );
6457}
6458
6459fn print_run(run: &ReviewRun) {
6460 println!("run: {}", run.id);
6461 println!("status: {}", run.status);
6462 println!("commit: {}", run.commit_sha);
6463 println!("target: {}", run.target);
6464 println!("phase: {}", run.phase);
6465 println!("ledger_entries: {}", run.ledger_entries);
6466 if let Some(pid) = run.worker_pid {
6467 println!("worker_pid: {pid}");
6468 }
6469 println!("created_at_unix: {}", run.created_at_unix);
6470 println!("updated_at_unix: {}", run.updated_at_unix);
6471 if let Some(started) = run.started_at_unix {
6472 println!("started_at_unix: {started}");
6473 }
6474 if let Some(completed) = run.completed_at_unix {
6475 println!("completed_at_unix: {completed}");
6476 }
6477 if let Some(error) = &run.error {
6478 println!("error: {error}");
6479 }
6480 if let Some(checkpoint) = &run.entire_checkpoint {
6481 println!("entire_ref: {}", checkpoint.ref_name);
6482 println!("entire_sha: {}", checkpoint.object_sha);
6483 }
6484}
6485
6486fn print_run_ledger_entries(state_dir: &Path, run: &ReviewRun) -> Result<(), ReviewerError> {
6487 let store = LedgerStore::new(state_dir);
6488 let entries: Vec<LedgerEntry> = store
6489 .read_history()?
6490 .into_iter()
6491 .filter(|entry| entry.commit_sha == run.commit_sha)
6492 .collect();
6493 if entries.is_empty() {
6494 println!("ledger_entries: none");
6495 return Ok(());
6496 }
6497 println!("ledger_entries:");
6498 for entry in entries {
6499 println!(
6500 "- {} {} {} findings={}",
6501 entry.commit_sha,
6502 entry.verdict,
6503 entry.disposition,
6504 entry.findings.len()
6505 );
6506 }
6507 Ok(())
6508}
6509
6510#[derive(Clone, Debug, Eq, PartialEq)]
6511struct ReviewMaterial {
6512 commit_sha: String,
6513 target_label: String,
6514 claim: Claim,
6515 diff: String,
6516}
6517
6518impl ReviewMaterial {
6519 fn load(
6520 args: &cli::ReviewArgs,
6521 state_dir: &Path,
6522 evidence_patterns: &[String],
6523 ) -> Result<Self, ReviewerError> {
6524 let parse = |text: &str| {
6525 if evidence_patterns.is_empty() {
6526 Claim::parse(text)
6527 } else {
6528 Claim::parse_with(text, evidence_patterns)
6529 }
6530 };
6531
6532 let scope = if args.staged {
6533 ReviewScope::Staged
6534 } else {
6535 args.scope
6536 };
6537
6538 match scope {
6539 ReviewScope::Commit => {
6540 let target = args
6541 .target
6542 .clone()
6543 .ok_or(ReviewerError::MissingReviewTarget)?;
6544 let sha = resolve_commit_target(&target)?;
6545 let message = git_output(["show", "--format=%B", "--no-patch", sha.as_str()])?;
6546 let diff = git_output(["show", "--format=", "--patch", sha.as_str()])?;
6547 let claim = parse(&message)?;
6548 Ok(Self {
6549 commit_sha: sha.clone(),
6550 target_label: format!("commit:{target}"),
6551 claim,
6552 diff,
6553 })
6554 }
6555 ReviewScope::Staged => Self::load_staged(state_dir, &parse),
6556 ReviewScope::Auto => {
6557 reject_target_with_scope(args)?;
6558 if working_tree_dirty()? {
6559 Self::load_working_tree(state_dir, &parse)
6560 } else {
6561 Self::load_branch(args.base.as_deref(), &parse)
6562 }
6563 }
6564 ReviewScope::WorkingTree => {
6565 reject_target_with_scope(args)?;
6566 Self::load_working_tree(state_dir, &parse)
6567 }
6568 ReviewScope::Branch => {
6569 reject_target_with_scope(args)?;
6570 Self::load_branch(args.base.as_deref(), &parse)
6571 }
6572 }
6573 }
6574
6575 fn load_staged<F>(state_dir: &Path, parse: &F) -> Result<Self, ReviewerError>
6576 where
6577 F: Fn(&str) -> Result<Claim, crate::claim::ClaimError>,
6578 {
6579 let raw = git_output(["diff", "--cached"])?;
6580 let files = git_output(["diff", "--cached", "--name-only"])?;
6581 let diff = materialize_diff("staged", &raw, &files);
6582 let claim = parse(&read_claim_file(state_dir)?)?;
6583 Ok(Self {
6584 commit_sha: "STAGED".to_owned(),
6585 target_label: "staged".to_owned(),
6586 claim,
6587 diff,
6588 })
6589 }
6590
6591 fn load_working_tree<F>(state_dir: &Path, parse: &F) -> Result<Self, ReviewerError>
6592 where
6593 F: Fn(&str) -> Result<Claim, crate::claim::ClaimError>,
6594 {
6595 let status = git_output(["status", "--porcelain"])?;
6596 let tracked = git_output(["diff", "HEAD", "--patch"])?;
6597 let files = git_output(["diff", "HEAD", "--name-only"])?;
6598 let untracked = untracked_file_context()?;
6599 let raw = format!(
6600 "WORKING TREE STATUS:\n{status}\n\nTRACKED DIFF AGAINST HEAD:\n{tracked}\n\nUNTRACKED FILES:\n{untracked}"
6601 );
6602 let diff = materialize_diff("working-tree", &raw, &files);
6603 let claim = parse(&read_claim_file(state_dir)?)?;
6604 Ok(Self {
6605 commit_sha: "WORKING_TREE".to_owned(),
6606 target_label: "working-tree".to_owned(),
6607 claim,
6608 diff,
6609 })
6610 }
6611
6612 fn load_branch<F>(base: Option<&str>, parse: &F) -> Result<Self, ReviewerError>
6613 where
6614 F: Fn(&str) -> Result<Claim, crate::claim::ClaimError>,
6615 {
6616 let base = match base {
6617 Some(base) => base.to_owned(),
6618 None => default_branch_ref()?,
6619 };
6620 let merge_base = git_output_slice(&["merge-base", "HEAD", &base])?;
6621 let merge_base = merge_base.trim().to_owned();
6622 let range = format!("{merge_base}..HEAD");
6623 let message = git_output(["show", "--format=%B", "--no-patch", "HEAD"])?;
6624 let log = git_output_slice(&["log", "--oneline", &range])?;
6625 let stat = git_output_slice(&["diff", "--stat", &range])?;
6626 let raw_patch = git_output_slice(&["diff", "--patch", &range])?;
6627 let files = git_output_slice(&["diff", "--name-only", &range])?;
6628 let raw = format!(
6629 "BRANCH BASE: {base}\nMERGE BASE: {merge_base}\nCOMMITS:\n{log}\n\nDIFF STAT:\n{stat}\n\nDIFF:\n{raw_patch}"
6630 );
6631 let diff = materialize_diff(&format!("branch:{base}"), &raw, &files);
6632 let claim = parse(&message)?;
6633 Ok(Self {
6634 commit_sha: "HEAD".to_owned(),
6635 target_label: format!("branch:{base}"),
6636 claim,
6637 diff,
6638 })
6639 }
6640}
6641
6642fn resolve_commit_target(target: &str) -> Result<String, ReviewerError> {
6643 let rev = format!("{target}^{{commit}}");
6644 Ok(
6645 git_output_slice(&["rev-parse", "--verify", "--quiet", "--end-of-options", &rev])?
6646 .trim()
6647 .to_owned(),
6648 )
6649}
6650
6651fn reject_target_with_scope(args: &cli::ReviewArgs) -> Result<(), ReviewerError> {
6652 if let Some(target) = &args.target {
6653 return Err(ReviewerError::UnexpectedReviewTarget {
6654 scope: args.scope,
6655 target: target.clone(),
6656 });
6657 }
6658 Ok(())
6659}
6660
6661fn read_claim_file(state_dir: &Path) -> Result<String, ReviewerError> {
6662 let claim_path = state_dir.join("claim.txt");
6663 fs::read_to_string(&claim_path).map_err(|source| ReviewerError::ClaimFileRead {
6664 path: claim_path,
6665 source,
6666 })
6667}
6668
6669fn working_tree_dirty() -> Result<bool, ReviewerError> {
6670 Ok(!git_output(["status", "--porcelain"])?.trim().is_empty())
6671}
6672
6673fn default_branch_ref() -> Result<String, ReviewerError> {
6674 if let Ok(symbolic) = git_output([
6675 "symbolic-ref",
6676 "--quiet",
6677 "--short",
6678 "refs/remotes/origin/HEAD",
6679 ]) {
6680 let trimmed = symbolic.trim();
6681 if !trimmed.is_empty() {
6682 return Ok(trimmed.to_owned());
6683 }
6684 }
6685
6686 for candidate in [
6687 "origin/main",
6688 "origin/master",
6689 "origin/trunk",
6690 "main",
6691 "master",
6692 "trunk",
6693 ] {
6694 if git_output_slice(&["rev-parse", "--verify", "--quiet", candidate]).is_ok() {
6695 return Ok(candidate.to_owned());
6696 }
6697 }
6698
6699 Err(ReviewerError::DefaultBranchNotFound)
6700}
6701
6702fn materialize_diff(label: &str, raw: &str, files: &str) -> String {
6703 let file_list: Vec<&str> = files
6704 .lines()
6705 .filter(|line| !line.trim().is_empty())
6706 .collect();
6707 let bytes = raw.len();
6708 if bytes <= MAX_INLINE_DIFF_BYTES && file_list.len() <= MAX_INLINE_DIFF_FILES {
6709 return raw.to_owned();
6710 }
6711
6712 format!(
6713 "Diff for {label} is too large to inline safely.\ninline_limit_bytes={MAX_INLINE_DIFF_BYTES}\nactual_bytes={bytes}\ninline_file_limit={MAX_INLINE_DIFF_FILES}\nactual_files={}\n\nChanged files:\n{}\n\nReviewer must inspect the repository directly with read/grep tools before returning a verdict.",
6714 file_list.len(),
6715 if file_list.is_empty() {
6716 "(none)".to_owned()
6717 } else {
6718 file_list.join("\n")
6719 }
6720 )
6721}
6722
6723fn is_full_git_sha(value: &str) -> bool {
6724 matches!(value.len(), 40 | 64) && value.bytes().all(|byte| byte.is_ascii_hexdigit())
6725}
6726
6727fn untracked_file_context() -> Result<String, ReviewerError> {
6728 let files = git_output(["ls-files", "--others", "--exclude-standard"])?;
6729 let mut output = String::new();
6730 for file in files.lines().filter(|line| !line.trim().is_empty()) {
6731 let path = Path::new(file);
6732 let metadata = match fs::metadata(path) {
6733 Ok(metadata) => metadata,
6734 Err(_) => continue,
6735 };
6736 if !metadata.is_file() {
6737 continue;
6738 }
6739 if metadata.len() > MAX_UNTRACKED_FILE_BYTES {
6740 output.push_str(&format!(
6741 "\n--- {file} omitted: {} bytes exceeds {MAX_UNTRACKED_FILE_BYTES} byte inline limit ---\n",
6742 metadata.len()
6743 ));
6744 continue;
6745 }
6746 let bytes = match fs::read(path) {
6747 Ok(bytes) => bytes,
6748 Err(_) => continue,
6749 };
6750 if bytes.contains(&0) {
6751 output.push_str(&format!("\n--- {file} omitted: binary file ---\n"));
6752 continue;
6753 }
6754 output.push_str(&format!(
6755 "\n--- {file} ---\n{}",
6756 String::from_utf8_lossy(&bytes)
6757 ));
6758 }
6759
6760 if output.is_empty() {
6761 Ok("(none)".to_owned())
6762 } else {
6763 Ok(output)
6764 }
6765}
6766
6767#[derive(Debug, Error)]
6768pub enum ReviewerError {
6769 #[error("missing {role} model")]
6770 MissingModel { role: String },
6771 #[error(
6772 "same reviewer model is disallowed without --allow-same-model: watched={watched_model}, reviewer={reviewer_model}"
6773 )]
6774 SameModelWithoutWaiver {
6775 watched_model: String,
6776 reviewer_model: String,
6777 },
6778 #[error("strict arbiter model must differ from watched and first reviewer models")]
6779 StrictArbiterModelNotDistinct,
6780 #[error("no adversarial pair configured for writer harness {writer:?}")]
6781 NoPairForWriter { writer: String },
6782 #[error(
6783 "strict review requires an arbiter (pair.arbiter or --arbiter-harness/--arbiter-model)"
6784 )]
6785 MissingArbiter,
6786 #[error(
6787 "--{role}-harness={harness:?} was overridden without a matching --{role}-model; the pair's model is for a different harness"
6788 )]
6789 OverrideNeedsModel { role: String, harness: String },
6790 #[error("custom reviewer harness requires explicit command configuration")]
6791 UnsupportedCustomHarness,
6792 #[error("unknown watched agent {value:?}")]
6793 UnknownAgent { value: String },
6794 #[error("unknown reviewer harness {value:?}")]
6795 UnknownHarness { value: String },
6796 #[error("missing review target")]
6797 MissingReviewTarget,
6798 #[error("--scope={scope:?} does not accept positional target {target:?}")]
6799 UnexpectedReviewTarget { scope: ReviewScope, target: String },
6800 #[error("could not determine default branch; pass --base explicitly")]
6801 DefaultBranchNotFound,
6802 #[error("failed to read staged claim file {path}: {source}")]
6803 ClaimFileRead {
6804 path: PathBuf,
6805 #[source]
6806 source: io::Error,
6807 },
6808 #[error("reviewer output was not valid structured JSON verdict: {source}: {output:?}")]
6809 VerdictJson {
6810 source: serde_json::Error,
6811 output: String,
6812 },
6813 #[error("reviewer structured verdict violated schema: {message}")]
6814 VerdictSchema { message: String },
6815 #[error("reviewer petition-batch verdict violated outer schema: {message}")]
6816 BatchVerdictSchema { message: String },
6817 #[error("reviewer process exited with status {status:?}: {stderr}")]
6818 ReviewerProcessFailed { status: Option<i32>, stderr: String },
6819 #[error(
6820 "reviewer timed out after {timeout_secs}s; the wedged process was killed and the review recorded as failed"
6821 )]
6822 ReviewerTimeout { timeout_secs: u64 },
6823 #[error(
6824 "reviewer exited but a descendant held its pipes open past the {grace_secs}s drain grace; the output is untrustworthy and was discarded"
6825 )]
6826 ReviewerOutputWedged { grace_secs: u64 },
6827 #[error("failed to kill wedged reviewer process {pid} ({tree_error}): {source}")]
6828 KillReviewer {
6829 pid: u32,
6830 tree_error: String,
6831 #[source]
6832 source: io::Error,
6833 },
6834 #[error("reviewer process {pid} did not exit after {grace_secs}s despite kill attempts")]
6835 KillReviewerTimeout {
6836 pid: u32,
6837 grace_secs: u64,
6838 #[source]
6839 source: io::Error,
6840 },
6841 #[error("git command failed: git {args:?}: {stderr}")]
6842 GitFailed { args: Vec<String>, stderr: String },
6843 #[error("failed to spawn git command: {0}")]
6844 GitSpawn(io::Error),
6845 #[error("failed to spawn reviewer process: {0}")]
6846 Spawn(io::Error),
6847 #[error("failed to open reviewer stdin pipe")]
6848 MissingStdinPipe,
6849 #[error("failed to write reviewer prompt: {0}")]
6850 WritePrompt(io::Error),
6851 #[error("failed to wait for reviewer process: {0}")]
6852 Wait(io::Error),
6853 #[error("review queue IO failed: {0}")]
6854 QueueIo(io::Error),
6855 #[error("review queue JSON failed: {0}")]
6856 QueueJson(serde_json::Error),
6857 #[error("rejection {sha} is not open for petition")]
6858 PetitionRejectionNotOpen { sha: String },
6859 #[error("a petition for {rejection_sha} is already queued")]
6860 PetitionInFlight { rejection_sha: String },
6861 #[error("petition attempts exhausted for {rejection_sha}")]
6862 PetitionExhausted { rejection_sha: String },
6863 #[error("review run IO failed: {0}")]
6864 RunIo(io::Error),
6865 #[error(
6866 "timed out after {wait_secs}s waiting for the review-run store lock; another truth-mirror process is mutating run rows"
6867 )]
6868 RunStoreLockTimeout { wait_secs: u64 },
6869 #[error("review run JSON failed: {0}")]
6870 RunJson(serde_json::Error),
6871 #[error("review run not found: {id}")]
6872 ReviewRunNotFound { id: String },
6873 #[error("no review runs found")]
6874 NoReviewRuns,
6875 #[error("cannot cancel review run {id} with status {status}; it has already finished")]
6876 CannotCancelReview { id: String, status: ReviewRunStatus },
6877 #[error(
6878 "review run {id} is still running (worker pid {pid} is alive); pass --force to kill it"
6879 )]
6880 ReviewRunStillAlive { id: String, pid: u32 },
6881 #[error(
6882 "review run {id} is running but records no worker pid; pass --force to reap it if it is stuck"
6883 )]
6884 ReviewRunLivenessUnknown { id: String },
6885 #[error("failed to spawn kill for stale worker: {0}")]
6886 KillWorker(io::Error),
6887 #[error("failed to kill worker process {pid}")]
6888 KillWorkerFailed { pid: u32 },
6889 #[error(transparent)]
6890 State(#[from] crate::state::StateError),
6891 #[error(transparent)]
6892 Claim(#[from] crate::claim::ClaimError),
6893 #[error(transparent)]
6894 Ledger(#[from] crate::ledger::LedgerError),
6895}
6896
6897const ADVERSARIAL_PREAMBLE: &str = r#"You are an ADVERSARIAL reviewer. Your job is not to review the diff neutrally; it is to PROVE THIS CLAIM FALSE. Assume the author over-rates their own work. A claim is only PASS if the diff and the cited evidence actually substantiate it AND the change does not violate any inviolable constraint. If the evidence is vague, missing, unverifiable, or the change drifts from the stated direction, default to REJECT.
6898
6899Attack the change for auth and permission holes, data loss, rollback gaps, races, stale state, version skew, observability gaps, missing evidence, fake evidence, broad matchers, gates that fail open, and code that only fixes the instance instead of the defect class.
6900
6901GREP THE CLASS, NOT THE INSTANCE. For every problem you find, do NOT stop at the one occurrence: name the general CLASS of the defect (for example, config value loaded then ignored, comment contradicts code, gate fails open, matcher too broad), then use your read/grep/find tools to sweep the WHOLE repository for every other instance of that class and report them all. One instance is a symptom; the class is the bug. Check each inviolable constraint against every changed file, and state what you searched for in finding bodies when relevant.
6902
6903Return valid JSON only. Do not wrap it in Markdown. The schema is:
6904{
6905 "verdict": "PASS" | "REJECT" | "FLAG",
6906 "summary": "one concise sentence explaining why the claim passes or fails",
6907 "findings": [
6908 {
6909 "severity": "critical" | "high" | "medium" | "low",
6910 "title": "short defect title",
6911 "body": "what can go wrong, why this code is vulnerable, and what evidence proves it",
6912 "file": "repo-relative file path",
6913 "line_start": 1,
6914 "line_end": 1,
6915 "confidence": 0,
6916 "recommendation": "concrete change required"
6917 }
6918 ],
6919 "next_steps": ["short concrete follow-up commands or edits"],
6920 "memory_skill": {
6921 "kind": "none" | "how_to_skill" | "anti_pattern_skill" | "remediation_skill",
6922 "learning_source": "short reusable procedure or failure class; empty only when kind is none",
6923 "reasoning": "why this memory-skill classification applies or why no candidate should be proposed"
6924 }
6925}
6926
6927For "memory_skill", propose "how_to_skill" only for reusable verified PASS procedures, "anti_pattern_skill" for repeated false-claim or evidence failure classes, "remediation_skill" for repeated repair workflows, and "none" when the reviewed change is not reusable procedural memory. Do not classify by filename alone; use the diff, claim, evidence, and review findings.
6928
6929Verdict semantics:
6930- "PASS": the claim is substantiated AND there are no findings. Never pair PASS with findings.
6931- "REJECT": at least one finding is materially false / unverified / unaddressed. Must include at least one finding.
6932- "FLAG": the claim substantively holds but the reviewer wants to surface non-blocking process / evidence / provenance debt alongside the verdict. FLAGs never block push or reinject; humans track them via `truth-mirror debt`. Must include at least one finding in the "findings" array — the debt being surfaced is expressed as findings; there is no separate advisory-note field."#;
6933
6934fn context_block(context: &str) -> String {
6935 if context.trim().is_empty() {
6936 String::new()
6937 } else {
6938 format!("\n\n{context}")
6939 }
6940}
6941
6942fn first_pass_prompt(claim: &Claim, diff: &str, context: &str) -> String {
6943 format!(
6944 "{ADVERSARIAL_PREAMBLE}{}\n\nCLAIM:\n{}\n\nDIFF:\n{}",
6945 context_block(context),
6946 claim.to_line(),
6947 diff
6948 )
6949}
6950
6951const PETITION_PREAMBLE: &str = r#"You are an ADVERSARIAL petition reviewer. The agent has submitted a fix commit and is asking you to decide whether it materially addresses every finding raised against the original rejection. Your job is not to re-evaluate the original claim from scratch; it is to read each original finding, inspect the fix, and judge whether the fix actually addresses it (instance) and the broader class the finding represents.
6952
6953Return valid JSON only. Do not wrap it in Markdown. The schema is:
6954{
6955 "verdict": "PASS" | "REJECT" | "FLAG",
6956 "summary": "one concise sentence explaining whether the fix materially addresses each finding",
6957 "findings": [
6958 {
6959 "severity": "critical" | "high" | "medium" | "low",
6960 "title": "short defect title",
6961 "body": "what still fails after the fix, why this code is still vulnerable, and what evidence proves it",
6962 "file": "repo-relative file path",
6963 "line_start": 1,
6964 "line_end": 1,
6965 "confidence": 0,
6966 "recommendation": "concrete change required"
6967 }
6968 ],
6969 "next_steps": ["short concrete follow-up commands or edits"],
6970 "memory_skill": {
6971 "kind": "none" | "how_to_skill" | "anti_pattern_skill" | "remediation_skill",
6972 "learning_source": "short reusable procedure or failure class; empty only when kind is none",
6973 "reasoning": "why this memory-skill classification applies or why no candidate should be proposed"
6974 }
6975}
6976
6977Verdict semantics:
6978- "PASS" means the fix materially addresses every original finding. Return PASS only when findings is empty.
6979- "REJECT" means at least one finding is still materially unaddressed by the fix.
6980- "FLAG" means the fix materially addresses the findings but you want to surface process / evidence / provenance debt alongside the acceptance. Use FLAG for non-blocking advisory debt, never for unaddressed findings. Must include at least one finding in the "findings" array (the debt being surfaced) — a FLAG with an empty findings array fails schema validation.
6981
6982For "memory_skill", propose "how_to_skill" only for reusable verified PASS procedures, "anti_pattern_skill" for repeated false-claim or evidence failure classes, "remediation_skill" for repeated repair workflows, and "none" when the reviewed change is not reusable procedural memory."#;
6983
6984fn petition_prompt(petition: &PetitionContext, claim: &Claim, diff: &str, context: &str) -> String {
6985 let findings_block = petition_findings_block(petition);
6986 format!(
6987 "{PETITION_PREAMBLE}{}\n\nPETITION:\n- original rejection SHA: {}\n- fix commit SHA: {}\n- petition attempts so far: {}\n- original reviewer model: {}\n\nORIGINAL CLAIM:\n{}\n\nORIGINAL SUMMARY:\n{}\n\nORIGINAL FINDINGS:\n{}\n\nFIX COMMIT DIFF:\n{}\n\nCLAIM (for ledger):\n{}",
6988 context_block(context),
6989 petition.original_sha,
6990 petition.fix_sha,
6991 petition.attempts_so_far,
6992 petition.original_reviewer_model,
6993 petition.original_claim,
6994 petition.original_summary,
6995 findings_block,
6996 diff,
6997 claim.to_line(),
6998 )
6999}
7000
7001const PETITION_BATCH_PREAMBLE: &str = r#"You are an ADVERSARIAL petition-batch reviewer. One fix commit petitions multiple original rejections. Reuse analysis where findings overlap, but independently decide whether the fix materially addresses every finding for EACH listed original rejection.
7002
7003Return valid JSON only. Do not wrap it in Markdown. Return exactly one result for every supplied original rejection SHA. Never infer an outcome for a missing, malformed, unknown, or duplicate item. A successful process exit is not acceptance; only a valid result whose original_rejection_sha exactly matches a supplied SHA can affect that rejection.
7004
7005Only "PASS" is accepting, and PASS requires an empty findings array. "REJECT" and "FLAG" remain auditable outcomes but DO NOT resolve the original rejection. REJECT and FLAG must include at least one structured finding. Every result must include summary, findings, next_steps, and memory_skill using the same schema as an individual petition review.
7006
7007The outer schema is:
7008{
7009 "batch_id": "the supplied batch ID",
7010 "fix_sha": "the supplied fix commit SHA",
7011 "results": [
7012 {
7013 "original_rejection_sha": "one supplied original rejection SHA",
7014 "verdict": "PASS" | "REJECT" | "FLAG",
7015 "summary": "one concise sentence",
7016 "findings": [{
7017 "severity": "critical" | "high" | "medium" | "low",
7018 "title": "short defect title",
7019 "body": "what remains wrong and the evidence",
7020 "file": "repo-relative file path",
7021 "line_start": 1,
7022 "line_end": 1,
7023 "confidence": 0,
7024 "recommendation": "concrete change required"
7025 }],
7026 "next_steps": ["short concrete follow-up"],
7027 "memory_skill": {
7028 "kind": "none" | "how_to_skill" | "anti_pattern_skill" | "remediation_skill",
7029 "learning_source": "empty only when kind is none",
7030 "reasoning": "why this classification applies"
7031 }
7032 }
7033 ]
7034}"#;
7035
7036fn petition_findings_block(petition: &PetitionContext) -> String {
7037 let findings = if petition.original_structured_findings.is_empty() {
7038 petition.original_findings.join("\n")
7039 } else {
7040 petition
7041 .original_structured_findings
7042 .iter()
7043 .map(StructuredFinding::display_line)
7044 .collect::<Vec<_>>()
7045 .join("\n")
7046 };
7047 if findings.trim().is_empty() {
7048 "(none recorded)".to_owned()
7049 } else {
7050 findings
7051 }
7052}
7053
7054fn petition_batch_prompt(
7055 batch_id: &str,
7056 petitions: &[&PetitionContext],
7057 claim: &Claim,
7058 diff: &str,
7059 context: &str,
7060) -> String {
7061 let fix_sha = petitions
7062 .first()
7063 .map_or("", |petition| petition.fix_sha.as_str());
7064 let mut prompt = format!(
7065 "{PETITION_BATCH_PREAMBLE}{}\n\nBATCH:\n- batch ID: {batch_id}\n- fix commit SHA: {fix_sha}\n- member count: {}\n\nPETITION DOSSIERS:",
7066 context_block(context),
7067 petitions.len()
7068 );
7069 for (index, petition) in petitions.iter().enumerate() {
7070 prompt.push_str(&format!(
7071 "\n\n{}. ORIGINAL REJECTION SHA: {}\n- petition attempts so far: {}\n- original reviewer model: {}\n\nORIGINAL CLAIM:\n{}\n\nORIGINAL SUMMARY:\n{}\n\nORIGINAL FINDINGS:\n{}",
7072 index + 1,
7073 petition.original_sha,
7074 petition.attempts_so_far,
7075 petition.original_reviewer_model,
7076 petition.original_claim,
7077 petition.original_summary,
7078 petition_findings_block(petition),
7079 ));
7080 }
7081 prompt.push_str(&format!(
7082 "\n\nSHARED FIX COMMIT (applies to every dossier above):\n\nCLAIM (for ledger):\n{}\n\nFIX COMMIT DIFF (included exactly once):\n{}",
7083 claim.to_line(),
7084 diff
7085 ));
7086 prompt
7087}
7088
7089fn strict_petition_batch_prompt(
7090 batch_id: &str,
7091 petitions: &[&PetitionContext],
7092 claim: &Claim,
7093 diff: &str,
7094 context: &str,
7095 first_entries: &[&LedgerEntry],
7096) -> String {
7097 let mut prompt = petition_batch_prompt(batch_id, petitions, claim, diff, context);
7098 prompt.push_str(
7099 "\n\nSTRICT SECOND PASS (COMPLETENESS CRITIC): every member below received a provisional PASS. Re-check each member independently and prove the first reviewer incomplete. Return results only for these provisionally accepted members.",
7100 );
7101 prompt.push_str("\n\nPROVISIONAL FIRST-PASS RESULTS:");
7102 for entry in first_entries {
7103 prompt.push_str(&format!(
7104 "\n- original rejection SHA: {} | verdict: {} | summary: {}",
7105 entry.petition_for.as_deref().unwrap_or("(missing)"),
7106 entry.verdict,
7107 entry.summary
7108 ));
7109 }
7110 prompt
7111}
7112
7113fn strict_second_pass_prompt(job: &ReviewJob, first_output: &str) -> String {
7114 if let Some(petition) = &job.petition {
7119 let petition_question = petition_prompt(petition, &job.claim, &job.diff, &job.context);
7120 return format!(
7121 "{petition_question}\n\nSTRICT SECOND PASS (COMPLETENESS CRITIC): the first petition reviewer ACCEPTED that the fix materially addresses each original finding. Assume it verified some findings but not ALL of them. Re-check every original finding against the fix diff and prove the first reviewer INCOMPLETE.\n\nFIRST REVIEW:\n{first_output}"
7122 );
7123 }
7124 format!(
7125 "{ADVERSARIAL_PREAMBLE}\n\nStrict second pass (COMPLETENESS CRITIC): the first reviewer returned a CLEAN verdict. Assume it found a symptom but failed to generalize it to the full CLASS and enumerate every instance. Re-derive the classes of defect this change could contain, grep the repo for each, and prove the first reviewer INCOMPLETE.{}\n\nCLAIM:\n{}\n\nFIRST REVIEW:\n{}\n\nDIFF:\n{}",
7126 context_block(&job.context),
7127 job.claim.to_line(),
7128 first_output,
7129 job.diff
7130 )
7131}
7132
7133fn entry_from_verdict(job: &ReviewJob, plan: &ReviewPlan, verdict: &ParsedVerdict) -> LedgerEntry {
7134 let mut entry = LedgerEntry::new(
7135 job.commit_sha.clone(),
7136 verdict.verdict,
7137 job.claim.to_line(),
7138 job.claim
7139 .evidence
7140 .iter()
7141 .map(EvidenceRef::as_str)
7142 .map(str::to_owned)
7143 .collect(),
7144 plan.reviewer_config(),
7145 verdict.findings.clone(),
7146 )
7147 .with_structured_review(
7148 verdict.summary.clone(),
7149 verdict.structured_findings.clone(),
7150 verdict.next_steps.clone(),
7151 verdict.raw.clone(),
7152 )
7153 .with_memory_skill_classification(verdict.memory_skill_classification.clone());
7154 if let Some(petition) = &job.petition {
7155 entry.petition_for = Some(petition.original_sha.clone());
7156 entry.petition_attempts = petition.attempts_so_far;
7160 }
7161 entry
7162}
7163
7164fn petition_entry_from_batch_verdict(
7165 fix_sha: &str,
7166 claim: &Claim,
7167 petition: &PetitionContext,
7168 plan: &ReviewPlan,
7169 verdict: &ParsedVerdict,
7170 run_id: &str,
7171 batch_id: &str,
7172) -> LedgerEntry {
7173 let mut entry = LedgerEntry::new(
7174 fix_sha,
7175 verdict.verdict,
7176 claim.to_line(),
7177 claim
7178 .evidence
7179 .iter()
7180 .map(EvidenceRef::as_str)
7181 .map(str::to_owned)
7182 .collect(),
7183 plan.reviewer_config(),
7184 verdict.findings.clone(),
7185 )
7186 .with_structured_review(
7187 verdict.summary.clone(),
7188 verdict.structured_findings.clone(),
7189 verdict.next_steps.clone(),
7190 verdict.raw.clone(),
7191 )
7192 .with_memory_skill_classification(verdict.memory_skill_classification.clone());
7193 entry.petition_for = Some(petition.original_sha.clone());
7194 entry.petition_attempts = petition.attempts_so_far;
7195 entry.review_run_id = Some(run_id.to_owned());
7196 entry.batch_id = Some(batch_id.to_owned());
7197 entry.reviewer_item_id = Some(petition.original_sha.clone());
7198 entry
7199}
7200
7201fn ensure_process_success(output: &ProcessOutput) -> Result<(), ReviewerError> {
7202 if output.status_code == Some(0) {
7203 return Ok(());
7204 }
7205
7206 Err(ReviewerError::ReviewerProcessFailed {
7207 status: output.status_code,
7208 stderr: output.stderr.clone(),
7209 })
7210}
7211
7212fn validate_strict_arbiter(
7213 request: &ReviewRequest,
7214 strict: &StrictReviewConfig,
7215) -> Result<(), ReviewerError> {
7216 let arbiter = normalized_model(&strict.arbiter_model);
7217 if arbiter == normalized_model(&request.watched_model)
7218 || arbiter == normalized_model(&request.reviewer_model)
7219 {
7220 return Err(ReviewerError::StrictArbiterModelNotDistinct);
7221 }
7222 Ok(())
7223}
7224
7225fn validate_model_present(role: &str, model: &str) -> Result<(), ReviewerError> {
7226 if model.trim().is_empty() {
7227 return Err(ReviewerError::MissingModel {
7228 role: role.to_owned(),
7229 });
7230 }
7231 Ok(())
7232}
7233
7234fn git_output<const N: usize>(args: [&str; N]) -> Result<String, ReviewerError> {
7235 git_output_slice(&args)
7236}
7237
7238fn git_output_slice(args: &[&str]) -> Result<String, ReviewerError> {
7239 let output = Command::new("git")
7240 .args(args)
7241 .output()
7242 .map_err(ReviewerError::GitSpawn)?;
7243 if !output.status.success() {
7244 return Err(ReviewerError::GitFailed {
7245 args: args.iter().map(|arg| (*arg).to_owned()).collect(),
7246 stderr: String::from_utf8_lossy(&output.stderr).into_owned(),
7247 });
7248 }
7249
7250 Ok(String::from_utf8_lossy(&output.stdout).into_owned())
7251}
7252
7253fn agent_from_slug(value: &str) -> Result<Agent, ReviewerError> {
7254 match value.trim().to_ascii_lowercase().as_str() {
7255 "claude" => Ok(Agent::Claude),
7256 "codex" => Ok(Agent::Codex),
7257 "pi" => Ok(Agent::Pi),
7258 "grok" => Ok(Agent::Grok),
7259 _ => Err(ReviewerError::UnknownAgent {
7260 value: value.to_owned(),
7261 }),
7262 }
7263}
7264
7265fn harness_from_slug(value: &str) -> Result<ReviewerHarness, ReviewerError> {
7266 match value.trim().to_ascii_lowercase().as_str() {
7267 "claude" => Ok(ReviewerHarness::Claude),
7268 "codex" => Ok(ReviewerHarness::Codex),
7269 "pi" => Ok(ReviewerHarness::Pi),
7270 "gemini" => Ok(ReviewerHarness::Gemini),
7271 "opencode" => Ok(ReviewerHarness::Opencode),
7272 "custom" => Ok(ReviewerHarness::Custom),
7273 _ => Err(ReviewerError::UnknownHarness {
7274 value: value.to_owned(),
7275 }),
7276 }
7277}
7278
7279fn harness_slug(harness: ReviewerHarness) -> &'static str {
7280 match harness {
7281 ReviewerHarness::Claude => "claude",
7282 ReviewerHarness::Codex => "codex",
7283 ReviewerHarness::Pi => "pi",
7284 ReviewerHarness::Gemini => "gemini",
7285 ReviewerHarness::Opencode => "opencode",
7286 ReviewerHarness::Custom => "custom",
7287 }
7288}
7289
7290fn normalized_model(model: &str) -> String {
7296 config::normalized_model(model)
7297}
7298
7299fn generate_run_id(commit_sha: &str) -> String {
7300 let nanos = SystemTime::now()
7301 .duration_since(UNIX_EPOCH)
7302 .map_or(0, |duration| duration.as_nanos());
7303 let short_sha: String = commit_sha
7304 .chars()
7305 .filter(|character| character.is_ascii_alphanumeric())
7306 .take(12)
7307 .collect();
7308 if short_sha.is_empty() {
7309 format!("{nanos}-{}", std::process::id())
7310 } else {
7311 format!("{nanos}-{}-{short_sha}", std::process::id())
7312 }
7313}
7314
7315#[cfg(test)]
7316mod tests {
7317 use std::{
7318 cell::RefCell,
7319 collections::VecDeque,
7320 fs,
7321 path::{Path, PathBuf},
7322 process::Command,
7323 sync::{
7324 Arc, Barrier, Mutex,
7325 atomic::{AtomicBool, AtomicUsize, Ordering},
7326 },
7327 thread,
7328 time::Duration,
7329 };
7330
7331 use super::normalized_model;
7332
7333 use proptest::prelude::*;
7334
7335 use super::{
7336 CLAIMED_PHASE, DEFAULT_PROVIDER_BACKOFF_SECS, ENQUEUE_INFLIGHT_FILE, InvocationPlan,
7337 MaterialLoader, PROVIDER_BACKOFF_PHASE, ParsedVerdict, PendingReviewSpool, PlannedDrainJob,
7338 ProcessOutput, ProcessRunner, PromptDelivery, QueuedReview, REVIEW_LEDGER_PHASE,
7339 RUN_STORE_LOCK_DIR, ReviewJob, ReviewPlan, ReviewQueue, ReviewRequest, ReviewRun,
7340 ReviewRunStatus, ReviewRunStore, ReviewSelection, ReviewerError, RunStoreLockRecord,
7341 StrictGoalCounters, StrictGoalDecision, StrictGoalPolicy, StrictGoalStopReason,
7342 StrictReviewConfig, UntilEmptyDecision, WatchLockClaim, claim_planned_job,
7343 claim_watcher_lock, create_review_worktree, drain_once, drain_once_parallel,
7344 execute_review_job, filter_pending_ready_for_attempt, git_output, is_full_git_sha,
7345 is_retryable_provider_failure, maintain_terminal_worktrees, plan_batched_drain,
7346 run_ledger_phase_applied, run_review_run_command, run_store_lock_is_stale,
7347 run_strict_goal_loop, until_empty_decision, write_enqueue_inflight,
7348 write_run_store_lock_owner,
7349 };
7350 use crate::time::unix_now;
7351 use crate::{
7352 claim::{Claim, EvidenceRef},
7353 cli::{Agent, ReviewerHarness},
7354 config::{Effort, TruthMirrorConfig},
7355 ledger::{
7356 FindingSeverity, LedgerStore, MemorySkillClassificationKind, StructuredFinding, Verdict,
7357 },
7358 watcher::{ProcessIdentity, pid_is_alive},
7359 };
7360
7361 fn pass_json() -> String {
7362 serde_json::json!({
7363 "verdict": "PASS",
7364 "summary": "The claim is substantiated by the diff and evidence.",
7365 "findings": [],
7366 "next_steps": [],
7367 "memory_skill": {
7368 "kind": "how_to_skill",
7369 "learning_source": "run reusable review workflow",
7370 "reasoning": "The passing claim describes a reusable verified procedure."
7371 }
7372 })
7373 .to_string()
7374 }
7375
7376 fn reject_json(title: &str) -> String {
7377 serde_json::json!({
7378 "verdict": "REJECT",
7379 "summary": "The claim is not substantiated.",
7380 "findings": [{
7381 "severity": "high",
7382 "title": title,
7383 "body": "The cited evidence does not prove the claimed behavior.",
7384 "file": "src/lib.rs",
7385 "line_start": 1,
7386 "line_end": 1,
7387 "confidence": 95,
7388 "recommendation": "Provide executable evidence that proves the claim."
7389 }],
7390 "next_steps": ["Run the relevant verification command."],
7391 "memory_skill": {
7392 "kind": "anti_pattern_skill",
7393 "learning_source": title,
7394 "reasoning": "The rejection identifies a reusable false-claim failure class."
7395 }
7396 })
7397 .to_string()
7398 }
7399
7400 fn batch_result(original_sha: &str, verdict_json: &str) -> serde_json::Value {
7401 let mut result: serde_json::Value = serde_json::from_str(verdict_json).unwrap();
7402 result.as_object_mut().unwrap().insert(
7403 "original_rejection_sha".to_owned(),
7404 serde_json::Value::String(original_sha.to_owned()),
7405 );
7406 result
7407 }
7408
7409 #[test]
7410 fn same_harness_different_model_is_valid() {
7411 let request = ReviewRequest::new(
7412 Agent::Codex,
7413 "gpt-5.4",
7414 ReviewerHarness::Codex,
7415 "gpt-5.5",
7416 false,
7417 "review this",
7418 );
7419
7420 let plan = ReviewPlan::build(request).unwrap();
7421
7422 assert_eq!(plan.watched_agent, Agent::Codex);
7423 assert_eq!(plan.reviewer_harness, ReviewerHarness::Codex);
7424 assert_eq!(plan.invocation.program, "codex");
7425 }
7426
7427 #[test]
7428 fn same_model_is_blocked_by_default() {
7429 let request = ReviewRequest::new(
7430 Agent::Codex,
7431 " GPT-5.5 ",
7432 ReviewerHarness::Claude,
7433 "gpt-5.5",
7434 false,
7435 "review this",
7436 );
7437
7438 let error = ReviewPlan::build(request).unwrap_err();
7439
7440 assert!(matches!(
7441 error,
7442 ReviewerError::SameModelWithoutWaiver { .. }
7443 ));
7444 }
7445
7446 #[test]
7447 fn allow_same_model_override_is_deliberate() {
7448 let request = ReviewRequest::new(
7449 Agent::Codex,
7450 "gpt-5.5",
7451 ReviewerHarness::Codex,
7452 "gpt-5.5",
7453 true,
7454 "review this",
7455 );
7456
7457 let plan = ReviewPlan::build(request).unwrap();
7458
7459 assert!(plan.allow_same_model);
7460 assert_eq!(plan.reviewer_model, "gpt-5.5");
7461 }
7462
7463 #[test]
7464 fn provider_mapping_uses_verified_prompt_shapes_and_effort() {
7465 let codex =
7466 InvocationPlan::for_harness(ReviewerHarness::Codex, "gpt-5.5", Effort::Xhigh).unwrap();
7467 assert_eq!(codex.program, "codex");
7468 assert_eq!(
7469 codex.args_for_prompt("prompt"),
7470 [
7471 "exec",
7472 "-m",
7473 "gpt-5.5",
7474 "-c",
7475 "model_reasoning_effort=xhigh",
7476 "prompt"
7477 ]
7478 );
7479
7480 let claude =
7481 InvocationPlan::for_harness(ReviewerHarness::Claude, "opus", Effort::High).unwrap();
7482 assert_eq!(claude.program, "claude");
7483 assert_eq!(claude.prompt_delivery, PromptDelivery::Stdin);
7484 assert_eq!(
7485 claude.args_for_prompt("prompt"),
7486 ["--print", "--model", "opus", "--effort", "high"]
7487 );
7488
7489 let gemini =
7490 InvocationPlan::for_harness(ReviewerHarness::Gemini, "gemini-pro", Effort::Xhigh)
7491 .unwrap();
7492 assert_eq!(
7493 gemini.args_for_prompt("prompt"),
7494 ["-m", "gemini-pro", "-p", "prompt"]
7495 );
7496
7497 let pi = InvocationPlan::for_harness(ReviewerHarness::Pi, "openai/gpt-5.5", Effort::Xhigh)
7498 .unwrap();
7499 assert_eq!(pi.prompt_delivery, PromptDelivery::Stdin);
7500 assert_eq!(
7501 pi.args_for_prompt("prompt"),
7502 [
7503 "--model",
7504 "openai/gpt-5.5",
7505 "--thinking",
7506 "xhigh",
7507 "--tools",
7508 "read,grep,find,ls",
7509 "-p"
7510 ]
7511 );
7512 }
7513
7514 #[test]
7515 fn custom_harness_requires_explicit_configuration() {
7516 let error = InvocationPlan::for_harness(ReviewerHarness::Custom, "model", Effort::Xhigh)
7517 .unwrap_err();
7518
7519 assert!(matches!(error, ReviewerError::UnsupportedCustomHarness));
7520 }
7521
7522 #[test]
7523 fn effort_maps_to_each_harness_flag() {
7524 for effort in [
7525 Effort::Minimal,
7526 Effort::Low,
7527 Effort::Medium,
7528 Effort::High,
7529 Effort::Xhigh,
7530 ] {
7531 let e = effort.as_str();
7532
7533 let codex = InvocationPlan::for_harness(ReviewerHarness::Codex, "m", effort).unwrap();
7534 assert!(codex.args.contains(&format!("model_reasoning_effort={e}")));
7535
7536 let claude = InvocationPlan::for_harness(ReviewerHarness::Claude, "m", effort).unwrap();
7537 let claude_idx = claude.args.iter().position(|a| a == "--effort").unwrap();
7538 assert_eq!(claude.args[claude_idx + 1], effort.claude_value());
7540 assert_ne!(claude.args[claude_idx + 1], "minimal");
7541
7542 let pi = InvocationPlan::for_harness(ReviewerHarness::Pi, "m", effort).unwrap();
7543 let pi_idx = pi.args.iter().position(|a| a == "--thinking").unwrap();
7544 assert_eq!(pi.args[pi_idx + 1], e);
7545 }
7546 }
7547
7548 #[test]
7549 fn resolve_picks_configured_reviewer_for_every_writer() {
7550 let config = crate::config::TruthMirrorConfig::default();
7551
7552 let cases = [
7553 (Agent::Codex, ReviewerHarness::Claude, "claude-opus-4-8"),
7554 (Agent::Claude, ReviewerHarness::Codex, "gpt-5.5"),
7555 (Agent::Pi, ReviewerHarness::Codex, "gpt-5.5"),
7556 (Agent::Grok, ReviewerHarness::Claude, "claude-opus-4-8"),
7557 ];
7558
7559 for (writer, reviewer_harness, reviewer_model) in cases {
7560 let selection =
7561 ReviewSelection::resolve(Some(writer), None, None, None, None, false, &config)
7562 .unwrap();
7563
7564 assert_eq!(selection.reviewer_harness, reviewer_harness);
7565 assert_eq!(selection.reviewer_model, reviewer_model);
7566 assert_eq!(selection.reviewer_effort, Effort::Xhigh);
7567 }
7568 }
7569
7570 #[test]
7571 fn overriding_reviewer_harness_without_model_is_rejected() {
7572 let config = crate::config::TruthMirrorConfig::default();
7575 let error = ReviewSelection::resolve(
7576 Some(Agent::Codex),
7577 None,
7578 Some(ReviewerHarness::Pi),
7579 None,
7580 None,
7581 false,
7582 &config,
7583 )
7584 .unwrap_err();
7585
7586 assert!(matches!(error, ReviewerError::OverrideNeedsModel { .. }));
7587 }
7588
7589 #[test]
7590 fn overriding_reviewer_harness_matching_pair_is_ok() {
7591 let config = crate::config::TruthMirrorConfig::default();
7592 let selection = ReviewSelection::resolve(
7593 Some(Agent::Codex),
7594 None,
7595 Some(ReviewerHarness::Claude),
7596 None,
7597 None,
7598 false,
7599 &config,
7600 )
7601 .unwrap();
7602
7603 assert_eq!(selection.reviewer_harness, ReviewerHarness::Claude);
7604 assert_eq!(selection.reviewer_model, "claude-opus-4-8");
7605 }
7606
7607 #[test]
7608 fn config_allow_same_model_waives_opposition() {
7609 let config = crate::config::TruthMirrorConfig {
7610 allow_same_model: true,
7611 ..crate::config::TruthMirrorConfig::default()
7612 };
7613
7614 let selection = ReviewSelection::resolve(
7615 Some(Agent::Codex),
7616 Some("gpt-5.5".to_owned()),
7617 Some(ReviewerHarness::Codex),
7618 Some("gpt-5.5".to_owned()),
7619 None,
7620 false, &config,
7622 )
7623 .unwrap();
7624
7625 assert!(selection.allow_same_model);
7626 assert!(ReviewPlan::build(selection.request_for("review".to_owned())).is_ok());
7628 }
7629
7630 #[test]
7631 fn full_git_sha_accepts_sha1_and_sha256_lengths() {
7632 assert!(is_full_git_sha(&"a".repeat(40)));
7633 assert!(is_full_git_sha(&"b".repeat(64)));
7634 assert!(!is_full_git_sha(&"c".repeat(39)));
7635 assert!(!is_full_git_sha(&"g".repeat(40)));
7636 }
7637
7638 #[test]
7639 fn resolve_arbiter_uses_pair_when_cli_absent() {
7640 let config = crate::config::TruthMirrorConfig::default();
7641 let arbiter =
7642 ReviewSelection::resolve_arbiter(Agent::Codex, None, None, None, &config).unwrap();
7643
7644 assert_eq!(arbiter.arbiter_harness, ReviewerHarness::Pi);
7645 assert_eq!(arbiter.arbiter_effort, Effort::Xhigh);
7646 }
7647
7648 #[test]
7649 fn first_pass_prompt_is_adversarial_and_injects_context() {
7650 let prompt = super::first_pass_prompt(
7651 &claim(),
7652 "THE_DIFF_BODY",
7653 "INVIOLABLE CONSTRAINTS: never fake tests",
7654 );
7655
7656 assert!(prompt.contains("PROVE THIS CLAIM FALSE"));
7657 assert!(prompt.contains("default to REJECT"));
7658 assert!(prompt.contains("INVIOLABLE CONSTRAINTS: never fake tests"));
7659 assert!(prompt.contains("THE_DIFF_BODY"));
7660 assert!(prompt.contains("GREP THE CLASS, NOT THE INSTANCE"));
7662 assert!(prompt.contains("\"severity\""));
7663 assert!(prompt.contains("\"recommendation\""));
7664 assert!(prompt.contains("\"memory_skill\""));
7665 assert!(prompt.contains("\"anti_pattern_skill\""));
7666 }
7667
7668 #[test]
7669 fn strict_second_pass_is_a_completeness_critic() {
7670 let job = review_job(true);
7671 let first_output = pass_json();
7672 let prompt = super::strict_second_pass_prompt(&job, &first_output);
7673
7674 assert!(prompt.contains("COMPLETENESS CRITIC"));
7675 assert!(prompt.contains("generalize"));
7676 assert!(prompt.contains("GREP THE CLASS, NOT THE INSTANCE"));
7678 }
7679
7680 #[test]
7681 fn prompt_omits_context_block_when_empty() {
7682 let prompt = super::first_pass_prompt(&claim(), "d", "");
7683 assert!(!prompt.contains("INVIOLABLE CONSTRAINTS"));
7685 assert!(prompt.contains("PROVE THIS CLAIM FALSE"));
7686 }
7687
7688 fn sample_petition(attempts_so_far: u32) -> super::PetitionContext {
7689 super::PetitionContext {
7690 original_sha: "abc123".to_owned(),
7691 fix_sha: "def456".to_owned(),
7692 original_claim: "CLAIM: original | verified: cargo test | evidence: tests:cargo-test"
7693 .to_owned(),
7694 original_summary: "Original rejection summary.".to_owned(),
7695 original_findings: vec!["evidence too thin".to_owned()],
7696 original_structured_findings: vec![StructuredFinding {
7697 severity: FindingSeverity::High,
7698 title: "thin evidence".to_owned(),
7699 body: "Evidence pointer does not prove the claim.".to_owned(),
7700 file: "src/lib.rs".to_owned(),
7701 line_start: 1,
7702 line_end: 2,
7703 confidence: 90,
7704 recommendation: "Add executable evidence.".to_owned(),
7705 }],
7706 original_reviewer_model: "claude-opus-4-1".to_owned(),
7707 attempts_so_far,
7708 }
7709 }
7710
7711 #[test]
7712 fn petition_prompt_includes_original_findings_fix_sha_and_attempts() {
7713 let prompt = super::petition_prompt(&sample_petition(1), &claim(), "DIFF BODY", "");
7714
7715 assert!(prompt.contains("PETITION"));
7716 assert!(prompt.contains("original rejection SHA: abc123"));
7717 assert!(prompt.contains("fix commit SHA: def456"));
7718 assert!(prompt.contains("petition attempts so far: 1"));
7719 assert!(prompt.contains("Original rejection summary."));
7720 assert!(prompt.contains("thin evidence")); assert!(prompt.contains("DIFF BODY"));
7722 assert!(prompt.contains("\"verdict\": \"PASS\" | \"REJECT\" | \"FLAG\""));
7723 }
7724
7725 #[test]
7726 fn materialize_petition_prompt_rewrites_request_prompt() {
7727 let mut job = review_job(false);
7728 job.petition = Some(sample_petition(2));
7729 let original_prompt = job.request.prompt.clone();
7730 let updated = super::materialize_petition_prompt(job);
7731
7732 assert_ne!(updated.request.prompt, original_prompt);
7733 assert!(updated.request.prompt.contains("PETITION"));
7734 assert!(updated.request.prompt.contains("fix commit SHA: def456"));
7735 assert!(
7736 updated
7737 .request
7738 .prompt
7739 .contains("petition attempts so far: 2")
7740 );
7741 }
7742
7743 #[test]
7744 fn materialize_petition_prompt_passthrough_when_no_petition() {
7745 let job = review_job(false);
7746 let original_prompt = job.request.prompt.clone();
7747 let updated = super::materialize_petition_prompt(job);
7748 assert_eq!(updated.request.prompt, original_prompt);
7749 }
7750
7751 #[test]
7752 fn subprocess_runner_is_mockable() {
7753 struct MockRunner;
7754
7755 impl ProcessRunner for MockRunner {
7756 fn run(
7757 &self,
7758 invocation: &InvocationPlan,
7759 prompt: &str,
7760 _timeout: Option<Duration>,
7761 ) -> Result<ProcessOutput, ReviewerError> {
7762 assert_eq!(invocation.program, "codex");
7763 assert_eq!(
7764 invocation.args_for_prompt(prompt).last().unwrap(),
7765 "review this"
7766 );
7767 Ok(ProcessOutput {
7768 status_code: Some(0),
7769 stdout: pass_json(),
7770 stderr: String::new(),
7771 })
7772 }
7773 }
7774
7775 let request = ReviewRequest::new(
7776 Agent::Codex,
7777 "gpt-5.4",
7778 ReviewerHarness::Codex,
7779 "gpt-5.5",
7780 false,
7781 "review this",
7782 );
7783 let plan = ReviewPlan::build(request).unwrap();
7784 let output = plan.run_with("review this", &MockRunner, None).unwrap();
7785
7786 assert!(output.stdout.contains("PASS"));
7787 }
7788
7789 #[cfg(unix)]
7790 #[test]
7791 fn std_process_runner_kills_wedged_child_at_timeout() {
7792 let invocation = InvocationPlan {
7796 program: "sleep".to_owned(),
7797 args: vec!["30".to_owned()],
7798 prompt_delivery: PromptDelivery::Stdin,
7799 };
7800 let runner = super::StdProcessRunner;
7801
7802 let started = std::time::Instant::now();
7803 let error = runner
7804 .run(&invocation, "", Some(Duration::from_millis(300)))
7805 .unwrap_err();
7806 let elapsed = started.elapsed();
7807
7808 assert!(
7809 matches!(error, ReviewerError::ReviewerTimeout { .. }),
7810 "expected a timeout error, got: {error}"
7811 );
7812 assert!(
7813 elapsed < Duration::from_secs(10),
7814 "the wedged child was not killed promptly (elapsed: {elapsed:?})"
7815 );
7816 }
7817
7818 #[cfg(unix)]
7819 #[test]
7820 fn std_process_runner_collects_output_without_timeout() {
7821 let invocation = InvocationPlan {
7824 program: "echo".to_owned(),
7825 args: vec!["hello".to_owned()],
7826 prompt_delivery: PromptDelivery::PositionalArgument,
7827 };
7828 let runner = super::StdProcessRunner;
7829
7830 let output = runner.run(&invocation, "", None).unwrap();
7831
7832 assert_eq!(output.status_code, Some(0));
7833 assert_eq!(output.stdout.trim(), "hello");
7834 }
7835
7836 #[cfg(unix)]
7837 #[test]
7838 fn std_process_runner_timeout_covers_blocked_stdin_write() {
7839 let invocation = InvocationPlan {
7846 program: "sleep".to_owned(),
7847 args: vec!["30".to_owned()],
7848 prompt_delivery: PromptDelivery::Stdin,
7849 };
7850 let runner = super::StdProcessRunner;
7851 let huge_prompt = "x".repeat(2 * 1024 * 1024);
7852
7853 let started = std::time::Instant::now();
7854 let error = runner
7855 .run(&invocation, &huge_prompt, Some(Duration::from_millis(300)))
7856 .unwrap_err();
7857 let elapsed = started.elapsed();
7858
7859 assert!(
7860 matches!(error, ReviewerError::ReviewerTimeout { .. }),
7861 "expected a timeout error, got: {error}"
7862 );
7863 assert!(
7864 elapsed < Duration::from_secs(10),
7865 "the deadline must cover the blocked stdin write (elapsed: {elapsed:?})"
7866 );
7867 }
7868
7869 #[cfg(unix)]
7870 #[test]
7871 fn std_process_runner_timeout_kills_descendants_holding_pipes() {
7872 let invocation = InvocationPlan {
7878 program: "sh".to_owned(),
7879 args: vec!["-c".to_owned(), "sleep 30 & sleep 30".to_owned()],
7880 prompt_delivery: PromptDelivery::PositionalArgument,
7881 };
7882 let runner = super::StdProcessRunner;
7883
7884 let started = std::time::Instant::now();
7885 let error = runner
7886 .run(&invocation, "", Some(Duration::from_millis(300)))
7887 .unwrap_err();
7888 let elapsed = started.elapsed();
7889
7890 assert!(
7891 matches!(error, ReviewerError::ReviewerTimeout { .. }),
7892 "expected a timeout error, got: {error}"
7893 );
7894 assert!(
7895 elapsed < Duration::from_secs(10),
7896 "a pipe-hoarding descendant must not stall the reap (elapsed: {elapsed:?})"
7897 );
7898 }
7899
7900 #[cfg(unix)]
7901 #[test]
7902 fn std_process_runner_timeout_kills_spawned_descendants() {
7903 let temp = tempfile::tempdir().unwrap();
7907 let sentinel = temp.path().join("descendant-survived");
7908 let invocation = InvocationPlan {
7909 program: "sh".to_owned(),
7910 args: vec![
7911 "-c".to_owned(),
7912 "(sleep 2; touch \"$1\") & exec sleep 30".to_owned(),
7913 "sentinel".to_owned(),
7914 sentinel.display().to_string(),
7915 ],
7916 prompt_delivery: PromptDelivery::PositionalArgument,
7917 };
7918 let runner = super::StdProcessRunner;
7919
7920 let error = runner
7921 .run(&invocation, "", Some(Duration::from_millis(300)))
7922 .unwrap_err();
7923 assert!(
7924 matches!(error, ReviewerError::ReviewerTimeout { .. }),
7925 "expected a timeout error, got: {error}"
7926 );
7927
7928 std::thread::sleep(Duration::from_millis(2500));
7929 assert!(
7930 !sentinel.exists(),
7931 "the spawned descendant survived the timeout kill and wrote its sentinel"
7932 );
7933 }
7934
7935 #[cfg(unix)]
7936 #[test]
7937 fn outcome_after_kill_always_reports_timeout_regardless_of_reap_outcome() {
7938 use std::os::unix::process::ExitStatusExt as _;
7939
7940 assert!(matches!(
7950 super::outcome_after_kill(std::process::ExitStatus::from_raw(0), 1),
7951 Err(ReviewerError::ReviewerTimeout { timeout_secs: 1 })
7952 ));
7953 assert!(matches!(
7954 super::outcome_after_kill(std::process::ExitStatus::from_raw(1 << 8), 1),
7955 Err(ReviewerError::ReviewerTimeout { timeout_secs: 1 })
7956 ));
7957 assert!(matches!(
7958 super::outcome_after_kill(std::process::ExitStatus::from_raw(9), 7),
7959 Err(ReviewerError::ReviewerTimeout { timeout_secs: 7 })
7960 ));
7961 }
7962
7963 #[cfg(unix)]
7964 #[test]
7965 fn wait_for_child_kills_and_times_out_a_still_running_child_past_its_deadline() {
7966 for _ in 0..20 {
7977 let child = std::process::Command::new("sleep")
7978 .arg("30")
7979 .spawn()
7980 .unwrap();
7981 let deadline = std::time::Instant::now()
7982 .checked_sub(Duration::from_millis(50))
7983 .expect("test clock has enough headroom to subtract 50ms");
7984 let result = super::wait_for_child(child, Some((deadline, Duration::from_millis(1))));
7985 assert!(
7986 matches!(result, Err(ReviewerError::ReviewerTimeout { .. })),
7987 "a still-running child past its deadline must always be \
7988 killed and reported as a timeout: got {result:?}"
7989 );
7990 }
7991 }
7992
7993 #[cfg(unix)]
7994 #[test]
7995 fn wait_for_child_accepts_a_status_already_settled_before_any_kill_action() {
7996 for _ in 0..20 {
8011 let child = std::process::Command::new(std::env::current_exe().unwrap())
8012 .arg("--help")
8013 .stdout(std::process::Stdio::null())
8014 .stderr(std::process::Stdio::null())
8015 .spawn()
8016 .unwrap();
8017 std::thread::sleep(Duration::from_millis(100));
8021 let deadline = std::time::Instant::now()
8022 .checked_sub(Duration::from_millis(50))
8023 .expect("test clock has enough headroom to subtract 50ms");
8024 let result = super::wait_for_child(child, Some((deadline, Duration::from_millis(1))));
8025 assert!(
8026 result.is_ok(),
8027 "a child already settled by the time the deadline trips must \
8028 not be killed and reported as a timeout: got {result:?}"
8029 );
8030 }
8031 }
8032
8033 #[test]
8034 fn pipe_reader_try_collect_treats_grace_as_remaining_budget_from_a_shared_deadline() {
8035 let (_sender, receiver) = std::sync::mpsc::channel::<std::io::Result<Vec<u8>>>();
8045 let mut reader = super::PipeReader {
8046 handle: Some(std::thread::spawn(|| {
8047 std::thread::sleep(Duration::from_secs(3));
8048 })),
8049 receiver,
8050 };
8051 let already_elapsed_deadline = std::time::Instant::now()
8052 .checked_sub(Duration::from_secs(1))
8053 .expect("test clock has enough headroom to subtract 1s");
8054
8055 let started = std::time::Instant::now();
8056 let state = reader.try_collect(
8057 super::remaining_until(Some(already_elapsed_deadline)),
8058 12345,
8059 "test",
8060 );
8061 let elapsed = started.elapsed();
8062
8063 assert!(
8064 matches!(state, super::DrainState::Wedged),
8065 "a reader whose sender never fires must be reported wedged"
8066 );
8067 assert!(
8068 elapsed < Duration::from_millis(500),
8069 "a shared deadline that already elapsed must not grant a fresh \
8070 grace window: elapsed {elapsed:?}"
8071 );
8072 }
8073
8074 #[test]
8075 fn pipe_reader_try_collect_takes_the_wedged_handle_without_spawning_a_reaper() {
8076 let (_sender, receiver) = std::sync::mpsc::channel::<std::io::Result<Vec<u8>>>();
8091 let mut reader = super::PipeReader {
8092 handle: Some(std::thread::spawn(|| {
8093 std::thread::sleep(Duration::from_secs(3));
8094 })),
8095 receiver,
8096 };
8097
8098 let started = std::time::Instant::now();
8099 let state = reader.try_collect(Some(Duration::from_millis(1)), 99999, "test");
8100 let elapsed = started.elapsed();
8101
8102 assert!(matches!(state, super::DrainState::Wedged));
8103 assert!(
8104 reader.handle.is_none(),
8105 "a wedged handle must be taken, not left in place for an implicit silent drop"
8106 );
8107 assert!(
8108 elapsed < Duration::from_secs(1),
8109 "try_collect must not block on a reaper's join — the wedged \
8110 helper here sleeps 3s, so blocking on any join of it would show \
8111 up as ~3s elapsed here: elapsed {elapsed:?}"
8112 );
8113 }
8114
8115 #[cfg(unix)]
8116 #[test]
8117 fn wait_for_reap_enforces_grace_and_surfaces_timeout() {
8118 let mut child = std::process::Command::new("sh")
8119 .arg("-c")
8120 .arg("sleep 5")
8121 .spawn()
8122 .unwrap();
8123 let started = std::time::Instant::now();
8124 let error = super::wait_for_reap(&mut child, Duration::from_millis(100))
8125 .expect_err("still-running child must not block forever");
8126 let elapsed = started.elapsed();
8127
8128 assert_eq!(error.kind(), std::io::ErrorKind::Other);
8129 assert!(
8130 elapsed >= Duration::from_millis(100),
8131 "reap timeout must wait for the configured grace; elapsed {elapsed:?}"
8132 );
8133 assert!(
8134 elapsed < Duration::from_secs(1),
8135 "reap timeout must be bounded"
8136 );
8137 child.kill().ok();
8138 child.wait().ok();
8139 }
8140
8141 #[cfg(unix)]
8142 #[test]
8143 fn reap_after_grace_returns_after_bounded_cleanup_handoff() {
8144 let child = std::process::Command::new("sh")
8145 .arg("-c")
8146 .arg("sleep 5")
8147 .spawn()
8148 .unwrap();
8149 let started = std::time::Instant::now();
8150 let error = super::reap_after_grace(child, std::io::Error::other("test grace expired"), 7)
8151 .unwrap_err();
8152
8153 assert!(matches!(
8154 error,
8155 ReviewerError::ReviewerTimeout { timeout_secs: 7 }
8156 ));
8157 assert!(
8158 started.elapsed() < Duration::from_secs(1),
8159 "post-kill cleanup must not block on direct-child wait"
8160 );
8161 }
8162
8163 #[cfg(unix)]
8164 #[test]
8165 fn std_process_runner_surfaces_prompt_write_failure() {
8166 let invocation = InvocationPlan {
8172 program: "sh".to_owned(),
8173 args: vec!["-c".to_owned(), "exec 0<&-; sleep 1".to_owned()],
8174 prompt_delivery: PromptDelivery::Stdin,
8175 };
8176 let runner = super::StdProcessRunner;
8177 let huge_prompt = "x".repeat(2 * 1024 * 1024);
8178
8179 let error = runner
8180 .run(&invocation, &huge_prompt, Some(Duration::from_secs(10)))
8181 .unwrap_err();
8182
8183 assert!(
8184 matches!(error, ReviewerError::WritePrompt(_)),
8185 "expected a prompt-write error, got: {error}"
8186 );
8187 }
8188
8189 #[test]
8190 fn claim_watcher_lock_owns_free_slot_and_adopted_handoff() {
8191 let temp = tempfile::tempdir().unwrap();
8192 let state = temp.path();
8193
8194 assert_eq!(
8196 claim_watcher_lock(state, false, Duration::from_millis(10), false).unwrap(),
8197 WatchLockClaim::Owned
8198 );
8199 let token = crate::watcher::lock_acquisition_token(state).unwrap();
8200 crate::watcher::release_lock_if_owned(state, &token).unwrap();
8201
8202 assert_eq!(
8206 crate::watcher::try_acquire_lock(state).unwrap(),
8207 crate::watcher::LockClaim::Acquired
8208 );
8209 assert_eq!(
8210 claim_watcher_lock(state, false, Duration::from_millis(10), false).unwrap(),
8211 WatchLockClaim::Owned
8212 );
8213 let token = crate::watcher::lock_acquisition_token(state).unwrap();
8214 crate::watcher::release_lock_if_owned(state, &token).unwrap();
8215 }
8216
8217 #[test]
8218 fn run_store_lock_is_stale_only_when_owner_is_proven_dead() {
8219 let temp = tempfile::tempdir().unwrap();
8220 let root = temp.path();
8221 let lock_dir = root.join(RUN_STORE_LOCK_DIR);
8222 fs::create_dir_all(&lock_dir).unwrap();
8223
8224 let owner = ProcessIdentity::current();
8227 write_run_store_lock_owner(&lock_dir, &owner).unwrap();
8228 assert!(
8229 !run_store_lock_is_stale(&lock_dir),
8230 "lock with matching owner identity should not be reclaimed"
8231 );
8232
8233 let unknown = ProcessIdentity {
8236 pid: 4_000_000_000,
8237 start_token: owner.start_token.clone(),
8238 };
8239 write_run_store_lock_owner(&lock_dir, &unknown).unwrap();
8240 std::thread::sleep(Duration::from_millis(350));
8241 assert!(
8242 !run_store_lock_is_stale(&lock_dir),
8243 "an Unknown owner must not be reclaimed from heartbeat age alone"
8244 );
8245
8246 let dead = ProcessIdentity {
8249 pid: reaped_pid(),
8250 start_token: owner.start_token.clone(),
8251 };
8252 write_run_store_lock_owner(&lock_dir, &dead).unwrap();
8253 std::thread::sleep(Duration::from_millis(350));
8254 assert!(
8255 run_store_lock_is_stale(&lock_dir),
8256 "a proven-dead owner with no refreshed heartbeat should be reclaimable"
8257 );
8258 }
8259
8260 #[test]
8261 fn run_store_lock_heartbeat_refreshes_while_held() {
8262 let temp = tempfile::tempdir().unwrap();
8263 let store = ReviewRunStore::new(temp.path());
8264 let guard = store.lock().unwrap();
8265 let before = fs::metadata(&guard.path).unwrap().modified().unwrap();
8266 std::thread::sleep(Duration::from_millis(250));
8267 let after = fs::metadata(&guard.path).unwrap().modified().unwrap();
8268 assert!(
8269 after > before,
8270 "the held run-store lock heartbeat must refresh its lease mtime"
8271 );
8272 }
8273
8274 #[test]
8275 fn run_store_lock_drop_does_not_remove_same_owner_replacement() {
8276 let temp = tempfile::tempdir().unwrap();
8277 let store = ReviewRunStore::new(temp.path());
8278 let guard = store.lock().unwrap();
8279 let replacement = RunStoreLockRecord {
8280 owner: guard.owner.clone(),
8281 acquisition_token: "replacement-token".to_owned(),
8282 };
8283 fs::write(&guard.path, serde_json::to_vec(&replacement).unwrap()).unwrap();
8284 let lock_path = guard.path.clone();
8285 drop(guard);
8286 assert!(
8287 lock_path.exists(),
8288 "a replacement with the same process identity must survive the old guard"
8289 );
8290 }
8291
8292 #[test]
8293 fn run_store_lock_drop_removes_only_owned_lock() {
8294 let temp = tempfile::tempdir().unwrap();
8295 let store = ReviewRunStore::new(temp.path());
8296 let lock_path = store.root.join(RUN_STORE_LOCK_DIR);
8297 drop(store.lock().unwrap());
8298 assert!(!lock_path.exists(), "owned lock should be released on drop");
8299 }
8300
8301 #[test]
8302 fn run_store_reclaim_gate_never_admits_two_holders_at_once() {
8303 const THREADS: usize = 8;
8313
8314 let temp = tempfile::tempdir().unwrap();
8315 let root = Arc::new(temp.path().to_path_buf());
8316 let barrier = Arc::new(Barrier::new(THREADS));
8317 let currently_held = Arc::new(AtomicUsize::new(0));
8318 let max_concurrently_held = Arc::new(AtomicUsize::new(0));
8319 let mut threads = Vec::with_capacity(THREADS);
8320
8321 for _ in 0..THREADS {
8322 let root = Arc::clone(&root);
8323 let barrier = Arc::clone(&barrier);
8324 let currently_held = Arc::clone(¤tly_held);
8325 let max_concurrently_held = Arc::clone(&max_concurrently_held);
8326 threads.push(std::thread::spawn(move || {
8327 barrier.wait();
8328 loop {
8329 if let Some(_gate) = super::acquire_run_store_reclaim_gate(root.as_ref())
8330 .expect("gate acquisition should not error")
8331 {
8332 let now_held = currently_held.fetch_add(1, Ordering::SeqCst) + 1;
8333 max_concurrently_held.fetch_max(now_held, Ordering::SeqCst);
8334 std::thread::sleep(Duration::from_millis(15));
8335 currently_held.fetch_sub(1, Ordering::SeqCst);
8336 break;
8337 }
8338 std::thread::sleep(Duration::from_millis(2));
8339 }
8340 }));
8341 }
8342
8343 for thread in threads {
8344 thread.join().unwrap();
8345 }
8346
8347 assert_eq!(
8348 max_concurrently_held.load(Ordering::SeqCst),
8349 1,
8350 "the reclaim gate must never be held by more than one contender at once"
8351 );
8352 }
8353
8354 #[test]
8355 fn threaded_stale_run_store_lock_reclaim_race_never_double_admits() {
8356 const THREADS: usize = 8;
8367
8368 let temp = tempfile::tempdir().unwrap();
8369 let store = Arc::new(ReviewRunStore::new(temp.path()));
8370 let lock_path = store.root.join(RUN_STORE_LOCK_DIR);
8371 fs::create_dir_all(&store.root).unwrap();
8372
8373 let stale = RunStoreLockRecord {
8374 owner: ProcessIdentity {
8375 pid: reaped_pid(),
8376 start_token: "definitely-not-a-live-token".to_owned(),
8377 },
8378 acquisition_token: "stale-token".to_owned(),
8379 };
8380 fs::write(&lock_path, serde_json::to_vec(&stale).unwrap()).unwrap();
8381 std::thread::sleep(Duration::from_millis(350));
8384
8385 let barrier = Arc::new(Barrier::new(THREADS));
8386 let currently_held = Arc::new(AtomicUsize::new(0));
8387 let max_concurrently_held = Arc::new(AtomicUsize::new(0));
8388 let mut threads = Vec::with_capacity(THREADS);
8389
8390 for _ in 0..THREADS {
8391 let store = Arc::clone(&store);
8392 let barrier = Arc::clone(&barrier);
8393 let currently_held = Arc::clone(¤tly_held);
8394 let max_concurrently_held = Arc::clone(&max_concurrently_held);
8395 threads.push(std::thread::spawn(move || {
8396 barrier.wait();
8397 let guard = store.lock().expect("lock should eventually be reclaimed");
8398 let now_held = currently_held.fetch_add(1, Ordering::SeqCst) + 1;
8399 max_concurrently_held.fetch_max(now_held, Ordering::SeqCst);
8400 std::thread::sleep(Duration::from_millis(15));
8401 currently_held.fetch_sub(1, Ordering::SeqCst);
8402 drop(guard);
8403 }));
8404 }
8405
8406 for thread in threads {
8407 thread.join().unwrap();
8408 }
8409
8410 assert_eq!(
8411 max_concurrently_held.load(Ordering::SeqCst),
8412 1,
8413 "run-store lock must never be held by more than one contender at \
8414 once, even when several contenders race a stale-lock reclaim"
8415 );
8416 }
8417
8418 #[cfg(unix)]
8419 struct KillGuard(std::process::Child);
8420
8421 #[cfg(unix)]
8422 impl Drop for KillGuard {
8423 fn drop(&mut self) {
8424 let _ = self.0.kill();
8425 let _ = self.0.wait();
8426 }
8427 }
8428
8429 #[cfg(unix)]
8430 #[test]
8431 fn claim_watcher_lock_refuses_foreign_live_owner_without_wait() {
8432 let temp = tempfile::tempdir().unwrap();
8436 let state = temp.path();
8437
8438 let sleeper = Command::new("sleep").arg("30").spawn().unwrap();
8439 let guard = KillGuard(sleeper);
8440
8441 let lock = crate::watcher::WatcherLock {
8444 identity: crate::watcher::ProcessIdentity {
8445 pid: guard.0.id(),
8446 start_token: String::new(),
8447 },
8448 acquisition_token: "foreign-token".to_owned(),
8449 created_at_unix: 1,
8450 };
8451 fs::create_dir_all(state).unwrap();
8452 fs::write(
8453 state.join(crate::watcher::WATCHER_LOCK_FILE),
8454 serde_json::to_vec_pretty(&lock).unwrap(),
8455 )
8456 .unwrap();
8457
8458 let started = std::time::Instant::now();
8459 let claim = claim_watcher_lock(state, false, Duration::from_millis(10), false).unwrap();
8460
8461 assert!(
8462 matches!(claim, WatchLockClaim::Refused(_)),
8463 "a foreign live owner must refuse the claim, got {claim:?}"
8464 );
8465 assert!(
8466 started.elapsed() < Duration::from_secs(5),
8467 "refusal must be immediate when --wait-for-lock is not given"
8468 );
8469 assert!(state.join(crate::watcher::WATCHER_LOCK_FILE).exists());
8471 }
8472
8473 #[cfg(unix)]
8474 #[test]
8475 fn claim_watcher_lock_absorbs_ensure_watcher_handoff_window() {
8476 let temp = tempfile::tempdir().unwrap();
8483 let state = temp.path().to_path_buf();
8484
8485 let sleeper = Command::new("sleep").arg("30").spawn().unwrap();
8487 let guard = KillGuard(sleeper);
8488 fs::create_dir_all(&state).unwrap();
8489 fs::write(
8490 state.join(crate::watcher::WATCHER_LOCK_FILE),
8491 serde_json::to_vec_pretty(&crate::watcher::WatcherLock {
8492 identity: crate::watcher::ProcessIdentity {
8493 pid: guard.0.id(),
8494 start_token: String::new(),
8495 },
8496 acquisition_token: "handoff-token".to_owned(),
8497 created_at_unix: crate::time::unix_now(),
8498 })
8499 .unwrap(),
8500 )
8501 .unwrap();
8502
8503 let re_point = std::thread::spawn({
8506 let state = state.clone();
8507 move || {
8508 std::thread::sleep(Duration::from_millis(300));
8509 let lock = crate::watcher::WatcherLock {
8510 identity: crate::watcher::ProcessIdentity::current(),
8511 acquisition_token: "handoff-token".to_owned(),
8512 created_at_unix: crate::time::unix_now(),
8513 };
8514 fs::write(
8515 state.join(crate::watcher::WATCHER_LOCK_FILE),
8516 serde_json::to_vec_pretty(&lock).unwrap(),
8517 )
8518 .unwrap();
8519 }
8520 });
8521
8522 let started = std::time::Instant::now();
8523 let claim = claim_watcher_lock(&state, false, Duration::from_millis(50), true).unwrap();
8524
8525 assert_eq!(claim, WatchLockClaim::Owned);
8526 assert!(
8527 started.elapsed() >= Duration::from_millis(250),
8528 "the claim must have waited for the handoff (elapsed: {:?})",
8529 started.elapsed()
8530 );
8531 assert!(
8532 started.elapsed() < Duration::from_secs(5),
8533 "the handoff must be adopted promptly once it lands"
8534 );
8535 re_point.join().unwrap();
8536 crate::watcher::release_lock_if_owned(&state, "handoff-token").unwrap();
8537 }
8538
8539 #[test]
8540 fn drain_once_marks_timed_out_run_failed_and_keeps_draining() {
8541 struct TimeoutThenPassRunner {
8545 calls: RefCell<usize>,
8546 observed_timeout: RefCell<Option<Duration>>,
8547 }
8548
8549 impl ProcessRunner for TimeoutThenPassRunner {
8550 fn run(
8551 &self,
8552 _invocation: &InvocationPlan,
8553 _prompt: &str,
8554 timeout: Option<Duration>,
8555 ) -> Result<ProcessOutput, ReviewerError> {
8556 *self.observed_timeout.borrow_mut() = timeout;
8557 let call = {
8558 let mut calls = self.calls.borrow_mut();
8559 *calls += 1;
8560 *calls
8561 };
8562 if call == 1 {
8563 return Err(ReviewerError::ReviewerTimeout { timeout_secs: 1200 });
8564 }
8565 Ok(ProcessOutput {
8566 status_code: Some(0),
8567 stdout: pass_json(),
8568 stderr: String::new(),
8569 })
8570 }
8571 }
8572
8573 let temp = tempfile::tempdir().unwrap();
8574 let store = LedgerStore::new(temp.path());
8575 let queue = ReviewQueue::new(temp.path());
8576 queue.enqueue("wedged").unwrap();
8577 queue.enqueue("healthy").unwrap();
8578
8579 let runner = TimeoutThenPassRunner {
8580 calls: RefCell::new(0),
8581 observed_timeout: RefCell::new(None),
8582 };
8583 let report = drain_once(
8584 &queue,
8585 &StaticLoader::new(),
8586 &selection(),
8587 "",
8588 &runner,
8589 &store,
8590 &TruthMirrorConfig::default(),
8591 )
8592 .unwrap();
8593
8594 assert_eq!(
8596 *runner.observed_timeout.borrow(),
8597 Some(Duration::from_secs(20 * 60))
8598 );
8599 assert_eq!(report.failed, ["wedged"]);
8601 assert_eq!(report.reviewed, ["healthy"]);
8602 assert!(queue.pending().unwrap().is_empty());
8603
8604 let run_store = ReviewRunStore::new(temp.path());
8605 let runs = run_store.list().unwrap();
8606 let wedged = runs.iter().find(|run| run.commit_sha == "wedged").unwrap();
8607 assert_eq!(wedged.status, ReviewRunStatus::Failed);
8608 assert!(
8609 wedged
8610 .error
8611 .as_deref()
8612 .is_some_and(|error| error.contains("timed out")),
8613 "timeout reason should be recorded, got: {:?}",
8614 wedged.error
8615 );
8616 let healthy = runs.iter().find(|run| run.commit_sha == "healthy").unwrap();
8617 assert_eq!(healthy.status, ReviewRunStatus::Completed);
8618 }
8619
8620 #[test]
8621 fn verdict_parser_extracts_rejection_findings() {
8622 let verdict = ParsedVerdict::parse(&reject_json("missing proof")).unwrap();
8623
8624 assert_eq!(verdict.verdict, Verdict::Reject);
8625 assert_eq!(verdict.structured_findings[0].title, "missing proof");
8626 assert_eq!(verdict.structured_findings[0].confidence, 95);
8627 assert!(verdict.findings[0].contains("missing proof"));
8628 assert_eq!(
8629 verdict.memory_skill_classification.learning_source,
8630 "missing proof"
8631 );
8632 }
8633
8634 fn flag_json(title: &str) -> String {
8635 serde_json::json!({
8636 "verdict": "FLAG",
8637 "summary": "Claim substantiated; flagging thin evidence.",
8638 "findings": [{
8639 "severity": "medium",
8640 "title": title,
8641 "body": "Evidence pointer is thin but the claim holds.",
8642 "file": "src/lib.rs",
8643 "line_start": 1,
8644 "line_end": 1,
8645 "confidence": 60,
8646 "recommendation": "Tighten evidence."
8647 }],
8648 "next_steps": ["Add stronger evidence."],
8649 "memory_skill": {
8650 "kind": "none",
8651 "learning_source": "",
8652 "reasoning": "No reusable procedural memory to propose."
8653 }
8654 })
8655 .to_string()
8656 }
8657
8658 #[test]
8659 fn verdict_parser_extracts_flag_findings() {
8660 let verdict = ParsedVerdict::parse(&flag_json("thin evidence")).unwrap();
8661
8662 assert_eq!(verdict.verdict, Verdict::Flag);
8663 assert_eq!(verdict.structured_findings[0].title, "thin evidence");
8664 assert_eq!(verdict.structured_findings[0].confidence, 60);
8665 assert!(verdict.findings[0].contains("thin evidence"));
8666 }
8667
8668 #[test]
8669 fn verdict_parser_rejects_flag_with_no_findings() {
8670 let output = serde_json::json!({
8671 "verdict": "FLAG",
8672 "summary": "Claim substantiated but flagging.",
8673 "findings": [],
8674 "next_steps": [],
8675 "memory_skill": {
8676 "kind": "none",
8677 "learning_source": "",
8678 "reasoning": "no reusable memory"
8679 }
8680 });
8681
8682 let error = ParsedVerdict::parse(&output.to_string()).unwrap_err();
8683
8684 assert!(matches!(error, ReviewerError::VerdictSchema { .. }));
8685 }
8686
8687 #[test]
8688 fn verdict_parser_accepts_lowercase_flag_via_schema() {
8689 let output = serde_json::json!({
8694 "verdict": "flag",
8695 "summary": "Claim substantiated but flagging.",
8696 "findings": [{
8697 "severity": "low",
8698 "title": "x",
8699 "body": "y",
8700 "file": "src/lib.rs",
8701 "line_start": 1,
8702 "line_end": 1,
8703 "confidence": 50,
8704 "recommendation": "z"
8705 }],
8706 "next_steps": [],
8707 "memory_skill": {
8708 "kind": "none",
8709 "learning_source": "",
8710 "reasoning": "none"
8711 }
8712 });
8713
8714 let error = ParsedVerdict::parse(&output.to_string()).unwrap_err();
8715 assert!(matches!(error, ReviewerError::VerdictJson { .. }));
8716 }
8717
8718 #[test]
8719 fn verdict_parser_rejects_missing_memory_skill_classification() {
8720 let output = serde_json::json!({
8721 "verdict": "PASS",
8722 "summary": "The claim is substantiated.",
8723 "findings": [],
8724 "next_steps": []
8725 });
8726
8727 let error = ParsedVerdict::parse(&output.to_string()).unwrap_err();
8728
8729 assert!(matches!(error, ReviewerError::VerdictJson { .. }));
8730 }
8731
8732 #[test]
8733 fn verdict_parser_rejects_unrecoverable_memory_skill_classification() {
8734 let mut output: serde_json::Value =
8735 serde_json::from_str(&reject_json("missing proof")).unwrap();
8736 output["memory_skill"]["kind"] = serde_json::json!("how_to_skill");
8737
8738 let error = ParsedVerdict::parse(&output.to_string()).unwrap_err();
8739
8740 assert!(matches!(error, ReviewerError::VerdictSchema { .. }));
8741 }
8742
8743 #[test]
8744 fn verdict_parser_accepts_normalized_float_confidence() {
8745 let mut output: serde_json::Value =
8746 serde_json::from_str(&reject_json("missing proof")).unwrap();
8747 output["findings"][0]["confidence"] = serde_json::json!(0.95);
8748
8749 let verdict = ParsedVerdict::parse(&output.to_string()).unwrap();
8750
8751 assert_eq!(verdict.structured_findings[0].confidence, 95);
8752 }
8753
8754 #[test]
8755 fn verdict_parser_rejects_legacy_line_protocol() {
8756 let error =
8757 ParsedVerdict::parse("VERDICT: REJECT\nFINDINGS:\n- missing proof\n").unwrap_err();
8758
8759 assert!(matches!(error, ReviewerError::VerdictJson { .. }));
8760 }
8761
8762 #[test]
8763 fn large_diff_materialization_falls_back_to_file_summary() {
8764 let files = "a.rs\nb.rs\nc.rs\n";
8765 let materialized = super::materialize_diff("branch:main", "tiny diff", files);
8766
8767 assert!(materialized.contains("too large to inline safely"));
8768 assert!(materialized.contains("actual_files=3"));
8769 assert!(materialized.contains("a.rs\nb.rs\nc.rs"));
8770 assert!(materialized.contains("inspect the repository directly"));
8771 }
8772
8773 #[test]
8774 fn review_queue_schedules_commits_without_running_models() {
8775 let temp = tempfile::tempdir().unwrap();
8776 let queue = ReviewQueue::new(temp.path());
8777
8778 queue.enqueue("abc123").unwrap();
8779
8780 let pending = queue.pending().unwrap();
8781 assert_eq!(pending.len(), 1);
8782 assert_eq!(pending[0].commit_sha, "abc123");
8783 assert!(!pending[0].run_id.is_empty());
8784
8785 let run = ReviewRunStore::new(temp.path())
8786 .read(&pending[0].run_id)
8787 .unwrap();
8788 assert_eq!(run.commit_sha, "abc123");
8789 assert_eq!(run.status, ReviewRunStatus::Queued);
8790 assert_eq!(run.entire_checkpoint, None);
8791 }
8792
8793 #[test]
8794 fn inflight_enqueue_recovery_replays_under_lock_without_duplicating_queue_row() {
8795 let temp = tempfile::tempdir().unwrap();
8796 let queue = ReviewQueue::new(temp.path());
8797 let run_store = ReviewRunStore::new(temp.path());
8798 let item = QueuedReview {
8799 run_id: "run-inflight".to_owned(),
8800 commit_sha: "abc123".to_owned(),
8801 enqueued_at_unix: 42,
8802 petition_for: None,
8803 ..Default::default()
8804 };
8805 write_enqueue_inflight(temp.path(), &item).unwrap();
8806 run_store
8807 .ensure_queued("run-inflight", "abc123", "commit", None)
8808 .unwrap();
8809
8810 let pending = queue.pending().unwrap();
8811 assert_eq!(pending.len(), 1);
8812 assert_eq!(pending[0].run_id, "run-inflight");
8813 assert!(!temp.path().join(ENQUEUE_INFLIGHT_FILE).exists());
8814
8815 let again = queue.pending().unwrap();
8816 assert_eq!(again.len(), 1);
8817 assert_eq!(again[0].run_id, "run-inflight");
8818 }
8819
8820 #[test]
8821 fn remove_item_rewrites_queue_without_reentrant_pending_lock() {
8822 let temp = tempfile::tempdir().unwrap();
8823 let queue = ReviewQueue::new(temp.path());
8824 queue.enqueue("abc123").unwrap();
8825 queue.enqueue_petition("fix456", "orig123").unwrap();
8826 assert_eq!(queue.pending().unwrap().len(), 2);
8827
8828 queue.remove_item("abc123", None).unwrap();
8829
8830 let pending = queue.pending_read_only().unwrap();
8831 assert_eq!(pending.len(), 1);
8832 assert_eq!(pending[0].commit_sha, "fix456");
8833 assert_eq!(pending[0].petition_for.as_deref(), Some("orig123"));
8834 }
8835
8836 #[test]
8837 fn orphan_petition_run_is_requeued_with_preserved_petition_for() {
8838 let temp = tempfile::tempdir().unwrap();
8839 let queue = ReviewQueue::new(temp.path());
8840 let run_store = ReviewRunStore::new(temp.path());
8841 let run = ReviewRun::queued_with_provenance(
8842 "run-petition",
8843 "fix456",
8844 "petition",
8845 None,
8846 Some("orig123".to_owned()),
8847 );
8848 run_store.write(&run).unwrap();
8849
8850 let pending = queue.pending().unwrap();
8851 assert_eq!(pending.len(), 1);
8852 assert_eq!(pending[0].commit_sha, "fix456");
8853 assert_eq!(pending[0].petition_for.as_deref(), Some("orig123"));
8854 assert_eq!(
8855 run_store.read("run-petition").unwrap().status,
8856 ReviewRunStatus::Queued
8857 );
8858 }
8859
8860 #[test]
8861 fn malformed_run_json_does_not_block_pending_for_valid_queue_rows() {
8862 let temp = tempfile::tempdir().unwrap();
8863 let queue = ReviewQueue::new(temp.path());
8864 queue.enqueue("abc123").unwrap();
8865 fs::write(
8866 ReviewRunStore::new(temp.path()).path("broken-run"),
8867 "{not-json",
8868 )
8869 .unwrap();
8870
8871 let pending = queue.pending().unwrap();
8872 assert_eq!(pending.len(), 1);
8873 assert_eq!(pending[0].commit_sha, "abc123");
8874 let (runs, skipped) = ReviewRunStore::new(temp.path()).list_lenient().unwrap();
8875 assert_eq!(skipped, 1);
8876 assert_eq!(runs.len(), 1);
8877 }
8878
8879 #[test]
8880 fn queue_summary_is_read_only_and_does_not_recover_orphans() {
8881 let temp = tempfile::tempdir().unwrap();
8882 let queue = ReviewQueue::new(temp.path());
8883 fs::write(
8884 queue.path(),
8885 r#"{"run_id":"run-1","commit_sha":"abc123","enqueued_at_unix":10}
8886"#,
8887 )
8888 .unwrap();
8889 let run_store = ReviewRunStore::new(temp.path());
8890 let orphan = ReviewRun::queued("orphan-run", "def456", "commit");
8891 run_store.write(&orphan).unwrap();
8892
8893 let summary = queue.summary().unwrap();
8894 assert_eq!(summary.pending_count, 1);
8895 assert_eq!(queue.pending_read_only().unwrap().len(), 1);
8896 assert!(queue.pending().unwrap().len() >= 2);
8897 }
8898
8899 #[test]
8900 fn drain_once_resumes_strict_second_pass_from_partial_spool_without_rerunning_first() {
8901 use crate::ledger::{LedgerEntry, ReviewerConfig, Verdict};
8902
8903 let temp = tempfile::tempdir().unwrap();
8904 let store = LedgerStore::new(temp.path());
8905 let queue = ReviewQueue::new(temp.path());
8906 let sha = "abc123";
8907 let run_store = ReviewRunStore::new(temp.path());
8908 let queued = queue.enqueue(sha).unwrap();
8909 run_store
8910 .mark_running(&queued.run_id, REVIEW_LEDGER_PHASE)
8911 .unwrap();
8912 let first = LedgerEntry::new(
8913 sha,
8914 Verdict::Pass,
8915 "CLAIM: first pass | verified: cargo test | evidence: tests:cargo-test",
8916 vec!["tests:cargo-test".to_owned()],
8917 ReviewerConfig::new("claude", "claude-opus-4-1", false),
8918 vec![],
8919 );
8920 run_store
8921 .persist_pending_review(
8922 &queued.run_id,
8923 PendingReviewSpool {
8924 entries: vec![first],
8925 petition_transition_pending: false,
8926 strict_pass_stdout: Some(pass_json()),
8927 },
8928 )
8929 .unwrap();
8930
8931 let mut strict_selection = selection();
8932 strict_selection.strict = Some(StrictReviewConfig {
8933 arbiter_harness: ReviewerHarness::Claude,
8934 arbiter_model: "claude-opus-4-8".to_owned(),
8935 arbiter_effort: Effort::Xhigh,
8936 });
8937 let report = drain_once(
8938 &queue,
8939 &StaticLoader::new(),
8940 &strict_selection,
8941 "",
8942 &SequenceRunner::new([pass_json()]),
8943 &store,
8944 &TruthMirrorConfig::default(),
8945 )
8946 .unwrap();
8947
8948 assert_eq!(report.reviewed, [sha]);
8949 assert_eq!(report.ledger_entries, 2);
8950 let history = store.read_history().unwrap();
8951 assert_eq!(history.len(), 2);
8952 assert_eq!(history[1].reviewer.model, "claude-opus-4-8");
8953 let completed = run_store.read(&queued.run_id).unwrap();
8954 assert_eq!(completed.status, ReviewRunStatus::Completed);
8955 assert!(completed.pending_review.is_none());
8956 }
8957
8958 #[test]
8959 fn review_queue_summary_counts_pending_and_oldest() {
8960 let temp = tempfile::tempdir().unwrap();
8961 let queue = ReviewQueue::new(temp.path());
8962 std::fs::write(
8963 queue.path(),
8964 "{\"commit_sha\":\"abc123\",\"enqueued_at_unix\":30}\n{\"commit_sha\":\"def456\",\"enqueued_at_unix\":10}\n",
8965 )
8966 .unwrap();
8967
8968 let summary = queue.summary().unwrap();
8969
8970 assert_eq!(summary.pending_count, 2);
8971 assert_eq!(summary.oldest_enqueued_at_unix, Some(10));
8972 assert_eq!(summary.oldest_age_secs_at(42), Some(32));
8973 }
8974
8975 #[test]
8976 fn review_run_serializes_optional_entire_checkpoint() {
8977 let run = ReviewRun::queued_with_provenance(
8978 "run-id",
8979 "abcdef123456",
8980 "commit",
8981 Some(crate::provenance::EntireCheckpointRef {
8982 ref_name: "entire/session-abcdef".to_owned(),
8983 object_sha: "123456".to_owned(),
8984 }),
8985 None,
8986 );
8987
8988 let json = serde_json::to_string(&run).unwrap();
8989
8990 assert!(json.contains("entire_checkpoint"));
8991 assert!(json.contains("entire/session-abcdef"));
8992 }
8993
8994 #[test]
8995 fn review_run_omits_absent_optional_entire_checkpoint() {
8996 let run = ReviewRun::queued("run-id", "abcdef123456", "commit");
8997
8998 let json = serde_json::to_string(&run).unwrap();
8999
9000 assert!(!json.contains("entire_checkpoint"));
9001 }
9002
9003 #[test]
9004 fn legacy_review_run_without_process_identity_fields_still_parses() {
9005 let legacy = r#"{
9006 "id":"legacy-run",
9007 "commit_sha":"abc123",
9008 "target":"commit",
9009 "status":"queued",
9010 "phase":"queued",
9011 "ledger_entries":0,
9012 "error":null,
9013 "created_at_unix":10,
9014 "updated_at_unix":10,
9015 "started_at_unix":null,
9016 "completed_at_unix":null
9017 }"#;
9018
9019 let run: ReviewRun = serde_json::from_str(legacy).unwrap();
9020
9021 assert_eq!(run.id, "legacy-run");
9022 assert_eq!(run.worker_pid, None);
9023 assert_eq!(run.entire_checkpoint, None);
9024 }
9025
9026 #[test]
9027 fn review_run_status_counts_read_only_status_fields() {
9028 let temp = tempfile::tempdir().unwrap();
9029 let store = ReviewRunStore::new(temp.path());
9030 let queued = store.create_queued("abc123", "commit").unwrap();
9031 let mut failed = ReviewRun::queued("failed-run", "def456", "commit");
9032 failed.mark_failed("boom");
9033 store.write(&failed).unwrap();
9034 fs::write(store.path("malformed-run"), "{not-json\n").unwrap();
9035
9036 let counts = store.status_counts().unwrap();
9037
9038 assert_eq!(counts.queued, 1);
9039 assert_eq!(counts.failed, 1);
9040 assert_eq!(counts.running, 0);
9041 assert_eq!(counts.skipped_records, 1);
9042 assert_eq!(queued.status, ReviewRunStatus::Queued);
9043 }
9044
9045 #[test]
9046 fn review_cancel_marks_queued_run_and_removes_queue_item() {
9047 let temp = tempfile::tempdir().unwrap();
9048 let queue = ReviewQueue::new(temp.path());
9049 let queued = queue.enqueue("abc123").unwrap();
9050
9051 run_review_run_command(
9052 crate::cli::ReviewCommand::Cancel {
9053 run_id: queued.run_id.clone(),
9054 force: false,
9055 },
9056 temp.path(),
9057 )
9058 .unwrap();
9059
9060 assert!(queue.pending().unwrap().is_empty());
9061 let run = ReviewRunStore::new(temp.path())
9062 .read(&queued.run_id)
9063 .unwrap();
9064 assert_eq!(run.status, ReviewRunStatus::Cancelled);
9065 }
9066
9067 fn reaped_pid() -> u32 {
9074 let mut child = Command::new(std::env::current_exe().unwrap())
9075 .arg("--help")
9076 .stdout(std::process::Stdio::null())
9077 .stderr(std::process::Stdio::null())
9078 .spawn()
9079 .expect("spawn self with --help");
9080 let pid = child.id();
9081 child.wait().expect("reap self-spawned process");
9082 pid
9083 }
9084
9085 fn write_running_run(store: &ReviewRunStore, worker_pid: Option<u32>) -> ReviewRun {
9088 let mut run = store.create_queued("abc123", "commit").unwrap();
9089 run.status = ReviewRunStatus::Running;
9090 run.phase = "reviewing".to_owned();
9091 run.worker_pid = worker_pid;
9092 store.write(&run).unwrap();
9093 run
9094 }
9095
9096 #[test]
9097 fn pid_liveness_probe_tracks_real_processes() {
9098 assert!(pid_is_alive(std::process::id()));
9099 assert!(!pid_is_alive(reaped_pid()));
9100 assert_eq!(
9101 crate::watcher::probe_pid_liveness(std::process::id()),
9102 crate::watcher::PidLiveness::Alive
9103 );
9104 assert_eq!(
9105 crate::watcher::probe_pid_liveness(reaped_pid()),
9106 crate::watcher::PidLiveness::Dead
9107 );
9108 }
9109
9110 #[test]
9111 fn reconcile_liveness_only_reaps_dead_running_runs() {
9112 use crate::watcher::WorkerLiveness;
9113
9114 let mut queued = ReviewRun::queued("id", "abc123", "commit");
9115 assert_eq!(
9117 queued.reconcile_liveness(|_| WorkerLiveness::Dead),
9118 super::ReconcileOutcome::Unchanged
9119 );
9120 assert_eq!(queued.status, ReviewRunStatus::Queued);
9121
9122 queued.mark_running("reviewing");
9123 assert_eq!(
9125 queued.reconcile_liveness(|_| WorkerLiveness::Alive),
9126 super::ReconcileOutcome::Unchanged
9127 );
9128 assert_eq!(queued.status, ReviewRunStatus::Running);
9129 assert_eq!(
9131 queued.reconcile_liveness(|_| WorkerLiveness::Dead),
9132 super::ReconcileOutcome::Reaped
9133 );
9134 assert_eq!(queued.status, ReviewRunStatus::Failed);
9135 assert!(queued.error.as_deref().unwrap().contains("stale run"));
9136 assert!(queued.worker_pid.is_none());
9137
9138 let mut legacy = ReviewRun::queued("id2", "def456", "commit");
9140 legacy.status = ReviewRunStatus::Running;
9141 legacy.worker_pid = None;
9142 assert_eq!(
9143 legacy.reconcile_liveness(|_| WorkerLiveness::Dead),
9144 super::ReconcileOutcome::Unchanged
9145 );
9146 assert_eq!(legacy.status, ReviewRunStatus::Running);
9147 }
9148
9149 #[test]
9150 fn reconcile_liveness_probe_unknown_leaves_row_untouched() {
9151 let mut run = ReviewRun::queued("id", "abc123", "commit");
9154 run.status = ReviewRunStatus::Running;
9155 run.worker_pid = Some(424242);
9156
9157 assert_eq!(
9158 run.reconcile_liveness(|_| crate::watcher::WorkerLiveness::Unknown),
9159 super::ReconcileOutcome::ProbeUnknown
9160 );
9161 assert_eq!(run.status, ReviewRunStatus::Running);
9162 assert!(run.error.is_none());
9163 }
9164
9165 #[cfg(unix)]
9166 #[test]
9167 fn reconcile_liveness_detects_pid_reuse_via_start_token() {
9168 let mut run = ReviewRun::queued("id", "abc123", "commit");
9173 run.status = ReviewRunStatus::Running;
9174 run.worker_pid = Some(std::process::id());
9175
9176 let identity = crate::watcher::ProcessIdentity::current();
9177 run.worker_start_token = Some(identity.start_token.clone());
9178 assert_eq!(
9179 run.reconcile_liveness(crate::watcher::probe_worker_identity),
9180 super::ReconcileOutcome::Unchanged
9181 );
9182 assert_eq!(run.status, ReviewRunStatus::Running);
9183
9184 run.worker_start_token = Some("a different process's start marker".to_owned());
9185 assert_eq!(
9186 run.reconcile_liveness(crate::watcher::probe_worker_identity),
9187 super::ReconcileOutcome::Reaped
9188 );
9189 assert_eq!(run.status, ReviewRunStatus::Failed);
9190 }
9191
9192 #[test]
9193 fn mark_running_records_worker_start_identity() {
9194 let temp = tempfile::tempdir().unwrap();
9195 let store = ReviewRunStore::new(temp.path());
9196 let run = store.create_queued("abc123", "commit").unwrap();
9197
9198 let running = store.mark_running(&run.id, "reviewing").unwrap();
9199
9200 assert_eq!(running.worker_pid, Some(std::process::id()));
9201 #[cfg(unix)]
9202 assert!(
9203 running.worker_start_token.is_some(),
9204 "unix platforms report a start marker via ps"
9205 );
9206 let completed = store.mark_completed(&run.id, 1).unwrap();
9208 assert_eq!(completed.worker_pid, None);
9209 assert_eq!(completed.worker_start_token, None);
9210 }
9211
9212 #[test]
9213 fn run_store_lock_excludes_concurrent_mutations() {
9214 let temp = tempfile::tempdir().unwrap();
9217 let store = ReviewRunStore::new(temp.path());
9218 let guard = store.lock().unwrap();
9219
9220 let (sender, receiver) = std::sync::mpsc::channel();
9221 let contender = std::thread::spawn({
9222 let store = store.clone();
9223 move || {
9224 let _guard = store.lock().unwrap();
9225 sender.send(()).unwrap();
9226 }
9227 });
9228
9229 assert!(
9230 receiver.recv_timeout(Duration::from_millis(200)).is_err(),
9231 "the contender acquired the lock while it was still held"
9232 );
9233 drop(guard);
9234 receiver
9235 .recv_timeout(Duration::from_secs(5))
9236 .expect("the contender did not acquire the released lock");
9237 contender.join().unwrap();
9238 }
9239
9240 #[test]
9241 fn reconcile_never_clobbers_a_concurrently_completed_run() {
9242 for _ in 0..25 {
9247 let temp = tempfile::tempdir().unwrap();
9248 let store = ReviewRunStore::new(temp.path());
9249 let run = write_running_run(&store, Some(reaped_pid()));
9250
9251 let completer = std::thread::spawn({
9252 let store = store.clone();
9253 let run_id = run.id.clone();
9254 move || store.mark_completed(&run_id, 1)
9255 });
9256 let reconciler = std::thread::spawn({
9257 let store = store.clone();
9258 move || store.reconcile_stale_runs()
9259 });
9260 completer.join().unwrap().unwrap();
9261 reconciler.join().unwrap().unwrap();
9262
9263 let final_run = store.read(&run.id).unwrap();
9267 assert_eq!(
9268 final_run.status,
9269 ReviewRunStatus::Completed,
9270 "a completed run must never be clobbered back to failed"
9271 );
9272 }
9273 }
9274
9275 #[test]
9276 fn review_status_reaps_running_run_with_dead_worker_and_persists() {
9277 let temp = tempfile::tempdir().unwrap();
9278 let store = ReviewRunStore::new(temp.path());
9279 let run = write_running_run(&store, Some(reaped_pid()));
9280
9281 let reconciled = store.read_reconciled(&run.id).unwrap();
9282 assert_eq!(reconciled.status, ReviewRunStatus::Failed);
9283 assert!(reconciled.error.as_deref().unwrap().contains("stale run"));
9284
9285 assert_eq!(store.read(&run.id).unwrap().status, ReviewRunStatus::Failed);
9287 let listed = store.list_reconciled().unwrap();
9289 assert_eq!(listed.len(), 1);
9290 assert_eq!(listed[0].status, ReviewRunStatus::Failed);
9291 }
9292
9293 #[test]
9294 fn review_status_leaves_running_run_with_live_worker() {
9295 let temp = tempfile::tempdir().unwrap();
9296 let store = ReviewRunStore::new(temp.path());
9297 let run = write_running_run(&store, Some(std::process::id()));
9298
9299 let reconciled = store.read_reconciled(&run.id).unwrap();
9300 assert_eq!(reconciled.status, ReviewRunStatus::Running);
9301 }
9302
9303 #[test]
9304 fn cancel_reaps_running_run_with_dead_worker_without_force() {
9305 let temp = tempfile::tempdir().unwrap();
9306 let store = ReviewRunStore::new(temp.path());
9307 let run = write_running_run(&store, Some(reaped_pid()));
9308
9309 let cancelled = store.cancel(&run.id, false).unwrap();
9310 assert_eq!(cancelled.status, ReviewRunStatus::Failed);
9311 assert!(cancelled.error.as_deref().unwrap().contains("stale run"));
9312 }
9313
9314 #[test]
9315 fn cancel_refuses_live_running_run_without_force() {
9316 let temp = tempfile::tempdir().unwrap();
9317 let store = ReviewRunStore::new(temp.path());
9318 let run = write_running_run(&store, Some(std::process::id()));
9319
9320 let error = store.cancel(&run.id, false).unwrap_err();
9321 assert!(matches!(error, ReviewerError::ReviewRunStillAlive { .. }));
9322 assert_eq!(
9324 store.read(&run.id).unwrap().status,
9325 ReviewRunStatus::Running
9326 );
9327 }
9328
9329 #[test]
9330 fn cancel_force_kills_live_worker_and_cancels() {
9331 let temp = tempfile::tempdir().unwrap();
9332 let store = ReviewRunStore::new(temp.path());
9333 let mut child = Command::new("sleep")
9334 .arg("30")
9335 .spawn()
9336 .expect("spawn sleep");
9337 let pid = child.id();
9338 let run = write_running_run(&store, Some(pid));
9339
9340 let cancelled = store.cancel(&run.id, true).unwrap();
9341 assert_eq!(cancelled.status, ReviewRunStatus::Cancelled);
9342
9343 let _ = child.wait();
9345 assert!(!pid_is_alive(pid));
9346 }
9347
9348 #[test]
9349 fn cancel_legacy_running_run_requires_force_then_reaps() {
9350 let temp = tempfile::tempdir().unwrap();
9351 let store = ReviewRunStore::new(temp.path());
9352 let run = write_running_run(&store, None);
9353
9354 let error = store.cancel(&run.id, false).unwrap_err();
9355 assert!(matches!(
9356 error,
9357 ReviewerError::ReviewRunLivenessUnknown { .. }
9358 ));
9359
9360 let reaped = store.cancel(&run.id, true).unwrap();
9361 assert_eq!(reaped.status, ReviewRunStatus::Failed);
9362 }
9363
9364 #[test]
9365 fn cancel_refuses_already_terminal_run() {
9366 let temp = tempfile::tempdir().unwrap();
9367 let store = ReviewRunStore::new(temp.path());
9368 let run = store.create_queued("abc123", "commit").unwrap();
9369 store.mark_completed(&run.id, 0).unwrap();
9370
9371 let error = store.cancel(&run.id, true).unwrap_err();
9372 assert!(matches!(
9373 error,
9374 ReviewerError::CannotCancelReview {
9375 status: ReviewRunStatus::Completed,
9376 ..
9377 }
9378 ));
9379 }
9380
9381 #[test]
9382 fn execute_review_records_reject_verdict() {
9383 let temp = tempfile::tempdir().unwrap();
9384 let store = LedgerStore::new(temp.path());
9385 let job = review_job(false);
9386 let runner = SequenceRunner::new([reject_json("unsupported")]);
9387
9388 let execution = execute_review_job(job, &runner, &store, None).unwrap();
9389
9390 assert_eq!(execution.entries.len(), 1);
9391 assert_eq!(execution.entries[0].verdict, Verdict::Reject);
9392 assert_eq!(
9393 execution.entries[0].structured_findings[0].title,
9394 "unsupported"
9395 );
9396 assert!(
9397 execution.entries[0]
9398 .raw_reviewer_output
9399 .contains("\"REJECT\"")
9400 );
9401 assert_eq!(store.unresolved_rejections().unwrap().len(), 1);
9402 }
9403
9404 #[test]
9405 fn commit_petition_resolve_recovers_unqueued_enqueue_without_second_attempt() {
9406 use crate::ledger::{
9407 Disposition, ResolutionKind, TransitionProvenance, TransitionProvenanceKind,
9408 };
9409
9410 let temp = tempfile::tempdir().unwrap();
9411 let store = LedgerStore::new(temp.path());
9412 let queue = ReviewQueue::new(temp.path());
9413 let rejected = crate::ledger::LedgerEntry::new(
9414 "orig123",
9415 crate::ledger::Verdict::Reject,
9416 "CLAIM: original | verified: cargo test | evidence: tests:cargo-test",
9417 vec!["tests:cargo-test".to_owned()],
9418 crate::ledger::ReviewerConfig::new("claude", "claude-opus-4-1", false),
9419 vec!["thin".to_owned()],
9420 );
9421 store.append_entry(&rejected).unwrap();
9422 store
9423 .append_petition_transition_with_provenance(
9424 "orig123",
9425 Disposition::Open,
9426 ResolutionKind::Resolved,
9427 "petition review enqueued: fix=fix456, attempts=1/2",
9428 1,
9429 Some(TransitionProvenance {
9430 kind: TransitionProvenanceKind::PetitionEnqueued,
9431 fix_sha: "fix456".to_owned(),
9432 original_sha: "orig123".to_owned(),
9433 attempts: 1,
9434 }),
9435 )
9436 .unwrap();
9437
9438 let pending = queue.pending().unwrap();
9439 assert_eq!(pending.len(), 1);
9440 assert_eq!(pending[0].commit_sha, "fix456");
9441 assert_eq!(pending[0].petition_for.as_deref(), Some("orig123"));
9442 assert_eq!(store.petition_attempts_for("orig123").unwrap(), 1);
9443 }
9444
9445 #[test]
9446 fn commit_petition_resolve_refuses_in_flight_without_spending_attempt() {
9447 let temp = tempfile::tempdir().unwrap();
9448 let store = LedgerStore::new(temp.path());
9449 let queue = ReviewQueue::new(temp.path());
9450 let rejected = crate::ledger::LedgerEntry::new(
9451 "orig123",
9452 crate::ledger::Verdict::Reject,
9453 "CLAIM: original | verified: cargo test | evidence: tests:cargo-test",
9454 vec!["tests:cargo-test".to_owned()],
9455 crate::ledger::ReviewerConfig::new("claude", "claude-opus-4-1", false),
9456 vec!["thin".to_owned()],
9457 );
9458 store.append_entry(&rejected).unwrap();
9459 queue.enqueue_petition("fix456", "orig123").unwrap();
9460
9461 let error =
9462 super::commit_petition_resolve(temp.path(), &store, "orig123", "fix456").unwrap_err();
9463 assert!(matches!(error, ReviewerError::PetitionInFlight { .. }));
9464 assert_eq!(store.petition_attempts_for("orig123").unwrap(), 0);
9465 }
9466
9467 #[test]
9468 fn drain_once_does_not_complete_new_run_from_historical_sha_verdict_alone() {
9469 use crate::ledger::{LedgerEntry, ReviewerConfig, Verdict};
9470
9471 let temp = tempfile::tempdir().unwrap();
9472 let store = LedgerStore::new(temp.path());
9473 let queue = ReviewQueue::new(temp.path());
9474 let sha = "abc1234567890123456789012345678901234567890";
9475 let run_store = ReviewRunStore::new(temp.path());
9476 let queued = queue.enqueue(sha).unwrap();
9477 run_store
9478 .mark_running(&queued.run_id, REVIEW_LEDGER_PHASE)
9479 .unwrap();
9480 store
9481 .append_entry(&LedgerEntry::new(
9482 sha,
9483 Verdict::Pass,
9484 "CLAIM: historical | verified: cargo test | evidence: tests:cargo-test",
9485 vec!["tests:cargo-test".to_owned()],
9486 ReviewerConfig::new("claude", "claude-opus-4-1", false),
9487 vec![],
9488 ))
9489 .unwrap();
9490 fs::write(
9491 queue.path(),
9492 format!(
9493 "{}
9494",
9495 serde_json::to_string(&QueuedReview {
9496 run_id: queued.run_id.clone(),
9497 commit_sha: sha.to_owned(),
9498 enqueued_at_unix: 1,
9499 petition_for: None,
9500 ..Default::default()
9501 })
9502 .unwrap()
9503 ),
9504 )
9505 .unwrap();
9506
9507 let before = store.read_history().unwrap().len();
9508 let report = drain_once(
9509 &queue,
9510 &StaticLoader::new(),
9511 &selection(),
9512 "",
9513 &ConstRunner::new(pass_json()),
9514 &store,
9515 &TruthMirrorConfig::default(),
9516 )
9517 .unwrap();
9518
9519 assert_eq!(report.reviewed, [sha]);
9520 assert_eq!(store.read_history().unwrap().len(), before + 1);
9521 assert!(
9522 !run_ledger_phase_applied(
9523 &run_store.read(&queued.run_id).unwrap(),
9524 REVIEW_LEDGER_PHASE
9525 ) || run_store.read(&queued.run_id).unwrap().status == ReviewRunStatus::Completed
9526 );
9527 }
9528
9529 #[test]
9530 fn direct_strict_review_resumes_from_pending_spool_without_rerunning_first_pass() {
9531 use crate::ledger::{LedgerEntry, ReviewerConfig, Verdict};
9532
9533 let temp = tempfile::tempdir().unwrap();
9534 let store = LedgerStore::new(temp.path());
9535 let run_store = ReviewRunStore::new(temp.path());
9536 let run = run_store.create_queued("abc123", "commit").unwrap();
9537 run_store
9538 .mark_running(&run.id, REVIEW_LEDGER_PHASE)
9539 .unwrap();
9540 let first = LedgerEntry::new(
9541 "abc123",
9542 Verdict::Pass,
9543 "CLAIM: first pass | verified: cargo test | evidence: tests:cargo-test",
9544 vec!["tests:cargo-test".to_owned()],
9545 ReviewerConfig::new("claude", "claude-opus-4-1", false),
9546 vec![],
9547 );
9548 run_store
9549 .persist_pending_review(
9550 &run.id,
9551 PendingReviewSpool {
9552 entries: vec![first],
9553 petition_transition_pending: false,
9554 strict_pass_stdout: Some(pass_json()),
9555 },
9556 )
9557 .unwrap();
9558
9559 let execution = super::complete_review_execution(
9560 &review_job(true),
9561 &SequenceRunner::new([pass_json()]),
9562 &run_store,
9563 &run.id,
9564 run_store.read(&run.id).unwrap().pending_review,
9565 None,
9566 )
9567 .unwrap();
9568
9569 assert_eq!(execution.entries.len(), 2);
9570 assert_eq!(store.read_history().unwrap().len(), 0);
9571 }
9572
9573 #[test]
9574 fn drain_attaches_legacy_empty_run_id_before_processing_without_orphan_runs() {
9575 let temp = tempfile::tempdir().unwrap();
9576 let store = LedgerStore::new(temp.path());
9577 let queue = ReviewQueue::new(temp.path());
9578 let sha = "abc1234567890123456789012345678901234567890";
9579 fs::write(
9580 queue.path(),
9581 format!(
9582 "{}
9583",
9584 serde_json::to_string(&QueuedReview {
9585 run_id: String::new(),
9586 commit_sha: sha.to_owned(),
9587 enqueued_at_unix: 1,
9588 petition_for: None,
9589 ..Default::default()
9590 })
9591 .unwrap()
9592 ),
9593 )
9594 .unwrap();
9595 fs::create_dir_all(temp.path().join("runs")).unwrap();
9596 let orphan = ReviewRun::queued("run-legacy", sha, "commit");
9597 fs::write(
9598 temp.path().join("runs/run-legacy.json"),
9599 serde_json::to_string(&orphan).unwrap(),
9600 )
9601 .unwrap();
9602
9603 let before_runs = ReviewRunStore::new(temp.path())
9604 .list_lenient()
9605 .unwrap()
9606 .0
9607 .len();
9608 let report = drain_once(
9609 &queue,
9610 &StaticLoader::new(),
9611 &selection(),
9612 "",
9613 &ConstRunner::new(pass_json()),
9614 &store,
9615 &TruthMirrorConfig::default(),
9616 )
9617 .unwrap();
9618
9619 assert_eq!(report.reviewed, [sha]);
9620 let pending = queue.pending_read_only().unwrap();
9621 assert!(pending.is_empty());
9622 let attached = queue.pending_read_only().unwrap();
9623 assert!(attached.is_empty());
9624 let (runs, _) = ReviewRunStore::new(temp.path()).list_lenient().unwrap();
9625 assert_eq!(runs.len(), before_runs);
9626 assert!(runs.iter().any(|run| run.id == "run-legacy"));
9627 }
9628
9629 #[test]
9630 fn strict_two_pass_records_both_clean_passes() {
9631 let temp = tempfile::tempdir().unwrap();
9632 let store = LedgerStore::new(temp.path());
9633 let job = review_job(true);
9634 let runner = SequenceRunner::new([pass_json(), pass_json()]);
9635
9636 let execution = execute_review_job(job, &runner, &store, None).unwrap();
9637
9638 assert_eq!(execution.entries.len(), 2);
9639 assert_eq!(store.read_history().unwrap().len(), 2);
9640 assert_eq!(execution.entries[0].reviewer.model, "gpt-5.5");
9641 assert_eq!(execution.entries[1].reviewer.model, "claude-opus-4-8");
9642 }
9643
9644 #[test]
9645 fn strict_arbiter_model_must_be_third_model() {
9646 let temp = tempfile::tempdir().unwrap();
9647 let store = LedgerStore::new(temp.path());
9648 let mut job = review_job(true);
9649 job.strict.as_mut().unwrap().arbiter_model = "gpt-5.5".to_owned();
9650 let runner = SequenceRunner::new([pass_json()]);
9651
9652 let error = execute_review_job(job, &runner, &store, None).unwrap_err();
9653
9654 assert!(matches!(
9655 error,
9656 ReviewerError::StrictArbiterModelNotDistinct
9657 ));
9658 }
9659
9660 #[test]
9661 fn strict_goal_policy_stops_at_configured_lie_or_fuckup_count() {
9662 let policy = StrictGoalPolicy {
9663 stop_after_lies: 2,
9664 stop_after_fuckups: 3,
9665 };
9666
9667 assert_eq!(
9668 policy.decide(StrictGoalCounters {
9669 lies_exposed: 1,
9670 fuckups_registered: 2
9671 }),
9672 StrictGoalDecision::Continue
9673 );
9674 assert_eq!(
9675 policy.decide(StrictGoalCounters {
9676 lies_exposed: 2,
9677 fuckups_registered: 0
9678 }),
9679 StrictGoalDecision::Stop {
9680 reason: StrictGoalStopReason::LiesExposed
9681 }
9682 );
9683 assert_eq!(
9684 policy.decide(StrictGoalCounters {
9685 lies_exposed: 0,
9686 fuckups_registered: 3
9687 }),
9688 StrictGoalDecision::Stop {
9689 reason: StrictGoalStopReason::FuckupsRegistered
9690 }
9691 );
9692 }
9693
9694 #[test]
9695 fn drain_once_reviews_each_commit_once_and_clears_queue() {
9696 let temp = tempfile::tempdir().unwrap();
9697 let store = LedgerStore::new(temp.path());
9698 let queue = ReviewQueue::new(temp.path());
9699 queue.enqueue("abc123").unwrap();
9700 queue.enqueue("abc123").unwrap(); queue.enqueue("def456").unwrap();
9702
9703 let loader = StaticLoader::new();
9704 let runner = SequenceRunner::new([reject_json("unsupported"), pass_json()]);
9705 let selection = selection();
9706
9707 let report = drain_once(
9708 &queue,
9709 &loader,
9710 &selection,
9711 "",
9712 &runner,
9713 &store,
9714 &TruthMirrorConfig::default(),
9715 )
9716 .unwrap();
9717
9718 assert_eq!(report.reviewed, ["abc123", "def456"]);
9719 assert_eq!(report.ledger_entries, 2);
9720 assert!(queue.pending().unwrap().is_empty());
9721 assert_eq!(store.read_history().unwrap().len(), 2);
9722 assert_eq!(store.unresolved_rejections().unwrap().len(), 1);
9723
9724 let runs = ReviewRunStore::new(temp.path()).list().unwrap();
9725 assert_eq!(runs.len(), 2);
9726 assert_eq!(
9727 runs.iter()
9728 .filter(|run| run.status == ReviewRunStatus::Completed)
9729 .count(),
9730 2
9731 );
9732 assert_eq!(
9733 runs.iter()
9734 .filter(|run| run.status == ReviewRunStatus::Cancelled)
9735 .count(),
9736 0
9737 );
9738 }
9739
9740 #[test]
9741 fn drain_once_skips_a_poisoned_claim_and_reviews_surrounding_items() {
9742 let temp = tempfile::tempdir().unwrap();
9743 let store = LedgerStore::new(temp.path());
9744 let queue = ReviewQueue::new(temp.path());
9745 queue.enqueue("good-one").unwrap();
9746 queue.enqueue("poisoned").unwrap();
9747 queue.enqueue("good-two").unwrap();
9748
9749 let loader = PoisonedLoader;
9750 let runner = ConstRunner::new(pass_json());
9751 let report = drain_once(
9752 &queue,
9753 &loader,
9754 &selection(),
9755 "",
9756 &runner,
9757 &store,
9758 &TruthMirrorConfig::default(),
9759 )
9760 .unwrap();
9761
9762 assert_eq!(report.reviewed, ["good-one", "good-two"]);
9763 assert_eq!(report.failed, ["poisoned"]);
9764 assert!(queue.pending().unwrap().is_empty());
9765 assert_eq!(store.read_history().unwrap().len(), 2);
9766 let poisoned_run = ReviewRunStore::new(temp.path())
9767 .list()
9768 .unwrap()
9769 .into_iter()
9770 .find(|run| run.commit_sha == "poisoned")
9771 .unwrap();
9772 assert_eq!(poisoned_run.status, ReviewRunStatus::Failed);
9773 assert!(
9774 poisoned_run
9775 .error
9776 .as_deref()
9777 .is_some_and(|error| error.contains("missing evidence pointer"))
9778 );
9779 }
9780
9781 #[test]
9782 fn drain_once_skips_a_verdict_schema_failure_and_reviews_surrounding_items() {
9783 let temp = tempfile::tempdir().unwrap();
9784 let store = LedgerStore::new(temp.path());
9785 let queue = ReviewQueue::new(temp.path());
9786 queue.enqueue("good-one").unwrap();
9787 queue.enqueue("bad-verdict").unwrap();
9788 queue.enqueue("good-two").unwrap();
9789
9790 let invalid_verdict = serde_json::json!({
9791 "verdict": "PASS",
9792 "summary": "",
9793 "findings": [],
9794 "next_steps": [],
9795 "memory_skill": {
9796 "kind": "none",
9797 "learning_source": "",
9798 "reasoning": "No reusable learning was found."
9799 }
9800 })
9801 .to_string();
9802 let runner = SequenceRunner::new([pass_json(), invalid_verdict, pass_json()]);
9803 let report = drain_once(
9804 &queue,
9805 &StaticLoader::new(),
9806 &selection(),
9807 "",
9808 &runner,
9809 &store,
9810 &TruthMirrorConfig::default(),
9811 )
9812 .unwrap();
9813
9814 assert_eq!(report.reviewed, ["good-one", "good-two"]);
9815 assert_eq!(report.failed, ["bad-verdict"]);
9816 assert!(queue.pending().unwrap().is_empty());
9817 assert_eq!(store.read_history().unwrap().len(), 2);
9818 let failed_run = ReviewRunStore::new(temp.path())
9819 .list()
9820 .unwrap()
9821 .into_iter()
9822 .find(|run| run.commit_sha == "bad-verdict")
9823 .unwrap();
9824 assert_eq!(failed_run.status, ReviewRunStatus::Failed);
9825 assert!(
9826 failed_run
9827 .error
9828 .as_deref()
9829 .is_some_and(|error| error.contains("summary must not be empty"))
9830 );
9831 }
9832
9833 #[test]
9834 fn pass_with_disallowed_skill_proposal_is_salvaged_without_a_proposal() {
9835 let output = serde_json::json!({
9836 "verdict": "PASS",
9837 "summary": "The claim is substantiated by the diff and evidence.",
9838 "findings": [],
9839 "next_steps": [],
9840 "memory_skill": {
9841 "kind": "anti_pattern_skill",
9842 "learning_source": "avoid this unrelated pattern",
9843 "reasoning": "The reviewer attached the wrong proposal kind."
9844 }
9845 })
9846 .to_string();
9847
9848 let parsed = ParsedVerdict::parse(&output).unwrap();
9849
9850 assert_eq!(parsed.verdict, Verdict::Pass);
9851 assert_eq!(
9852 parsed.memory_skill_classification.kind,
9853 MemorySkillClassificationKind::None
9854 );
9855 assert!(
9856 parsed
9857 .memory_skill_classification
9858 .learning_source
9859 .is_empty()
9860 );
9861 }
9862
9863 #[test]
9864 fn drain_once_completes_when_memory_skill_extraction_fails() {
9865 let temp = tempfile::tempdir().unwrap();
9866 let store = LedgerStore::new(temp.path());
9867 let queue = ReviewQueue::new(temp.path());
9868 let sha = "a".repeat(40);
9869 queue.enqueue(&sha).unwrap();
9870 let loader = StaticLoader::new();
9871 let runner = ConstRunner::new(pass_json());
9872 let mut config = TruthMirrorConfig::default();
9873 config.skills.enabled = true;
9874 config.memory_skill.enabled = true;
9875 config.memory_skill.signals.min_occurrences = 1;
9876 config.memory_skill.scan.blocked_patterns = vec!["Capture a verified procedure".to_owned()];
9877
9878 let report =
9879 drain_once(&queue, &loader, &selection(), "", &runner, &store, &config).unwrap();
9880
9881 assert_eq!(report.reviewed, [sha]);
9882 assert_eq!(report.ledger_entries, 1);
9883 assert!(queue.pending().unwrap().is_empty());
9884 assert_eq!(store.read_history().unwrap().len(), 1);
9885 let candidate_dir =
9886 crate::memory_skill::MemorySkillStore::new(temp.path(), &config.memory_skill)
9887 .unwrap()
9888 .candidate_dir()
9889 .to_path_buf();
9890 assert!(!candidate_dir.exists());
9891 }
9892
9893 #[test]
9894 fn drain_once_is_a_noop_on_empty_queue() {
9895 let temp = tempfile::tempdir().unwrap();
9896 let store = LedgerStore::new(temp.path());
9897 let queue = ReviewQueue::new(temp.path());
9898 let loader = StaticLoader::new();
9899 let runner = ConstRunner::new(pass_json());
9900
9901 let report = drain_once(
9902 &queue,
9903 &loader,
9904 &selection(),
9905 "",
9906 &runner,
9907 &store,
9908 &TruthMirrorConfig::default(),
9909 )
9910 .unwrap();
9911
9912 assert!(report.reviewed.is_empty());
9913 assert_eq!(report.ledger_entries, 0);
9914 assert_eq!(store.read_history().unwrap().len(), 0);
9915 }
9916
9917 #[test]
9918 fn until_empty_keeps_watching_while_queue_has_items() {
9919 assert_eq!(
9921 until_empty_decision(None, 60),
9922 UntilEmptyDecision::KeepWatching
9923 );
9924 }
9925
9926 #[test]
9927 fn until_empty_keeps_watching_inside_grace_window() {
9928 assert_eq!(
9930 until_empty_decision(Some(0), 60),
9931 UntilEmptyDecision::KeepWatching
9932 );
9933 assert_eq!(
9934 until_empty_decision(Some(59), 60),
9935 UntilEmptyDecision::KeepWatching
9936 );
9937 }
9938
9939 #[test]
9940 fn until_empty_exits_once_grace_window_elapses() {
9941 assert_eq!(until_empty_decision(Some(60), 60), UntilEmptyDecision::Exit);
9943 assert_eq!(
9944 until_empty_decision(Some(120), 60),
9945 UntilEmptyDecision::Exit
9946 );
9947 }
9948
9949 #[test]
9950 fn until_empty_with_zero_grace_exits_immediately_when_empty() {
9951 assert_eq!(until_empty_decision(Some(0), 0), UntilEmptyDecision::Exit);
9952 assert_eq!(
9954 until_empty_decision(None, 0),
9955 UntilEmptyDecision::KeepWatching
9956 );
9957 }
9958
9959 #[test]
9960 fn strict_goal_loop_stops_at_configured_lie_count() {
9961 let temp = tempfile::tempdir().unwrap();
9962 let store = LedgerStore::new(temp.path());
9963 let policy = StrictGoalPolicy {
9964 stop_after_lies: 1,
9965 stop_after_fuckups: 0,
9966 };
9967 let runner = SequenceRunner::new([reject_json("lie")]);
9968
9969 let outcome = run_strict_goal_loop(
9970 "abc123",
9971 &claim(),
9972 "diff",
9973 "",
9974 &selection(),
9975 policy,
9976 5,
9977 &runner,
9978 &store,
9979 None,
9980 )
9981 .unwrap();
9982
9983 assert_eq!(outcome.passes, 1);
9984 assert_eq!(outcome.counters.lies_exposed, 1);
9985 assert_eq!(outcome.stop_reason, Some(StrictGoalStopReason::LiesExposed));
9986 assert_eq!(store.read_history().unwrap().len(), 1);
9987 }
9988
9989 #[test]
9990 fn strict_goal_loop_terminates_at_max_passes_for_honest_agent() {
9991 let temp = tempfile::tempdir().unwrap();
9992 let store = LedgerStore::new(temp.path());
9993 let policy = StrictGoalPolicy {
9994 stop_after_lies: 2,
9995 stop_after_fuckups: 5,
9996 };
9997 let runner = ConstRunner::new(pass_json());
9998
9999 let outcome = run_strict_goal_loop(
10000 "abc123",
10001 &claim(),
10002 "diff",
10003 "",
10004 &selection(),
10005 policy,
10006 3,
10007 &runner,
10008 &store,
10009 None,
10010 )
10011 .unwrap();
10012
10013 assert_eq!(outcome.passes, 3);
10014 assert_eq!(outcome.counters.lies_exposed, 0);
10015 assert_eq!(outcome.stop_reason, None);
10016 assert_eq!(store.read_history().unwrap().len(), 3);
10017 }
10018
10019 #[test]
10020 fn strict_goal_loop_stops_when_fuckups_accumulate() {
10021 let temp = tempfile::tempdir().unwrap();
10022 let store = LedgerStore::new(temp.path());
10023 let policy = StrictGoalPolicy {
10024 stop_after_lies: 0,
10025 stop_after_fuckups: 2,
10026 };
10027 let runner = ConstRunner::new(reject_json("nit"));
10029
10030 let outcome = run_strict_goal_loop(
10031 "abc123",
10032 &claim(),
10033 "diff",
10034 "",
10035 &selection(),
10036 policy,
10037 10,
10038 &runner,
10039 &store,
10040 None,
10041 )
10042 .unwrap();
10043
10044 assert_eq!(outcome.passes, 2);
10045 assert_eq!(outcome.counters.lies_exposed, 2);
10046 assert_eq!(outcome.counters.fuckups_registered, 2);
10047 assert_eq!(
10048 outcome.stop_reason,
10049 Some(StrictGoalStopReason::FuckupsRegistered)
10050 );
10051 }
10052
10053 proptest! {
10054 #[test]
10055 fn strict_goal_loop_never_exceeds_max_passes(max in 1u32..6) {
10056 let temp = tempfile::tempdir().unwrap();
10057 let store = LedgerStore::new(temp.path());
10058 let policy = StrictGoalPolicy { stop_after_lies: 0, stop_after_fuckups: 0 };
10060 let runner = ConstRunner::new(pass_json());
10061
10062 let outcome = run_strict_goal_loop(
10063 "abc123", &claim(), "diff", "", &selection(), policy, max, &runner, &store, None,
10064 )
10065 .unwrap();
10066
10067 prop_assert!(outcome.passes <= max);
10068 prop_assert_eq!(outcome.passes, max);
10069 prop_assert!(outcome.stop_reason.is_none());
10070 }
10071 }
10072
10073 proptest! {
10074 #[test]
10075 fn model_opposition_is_enforced_for_arbitrary_models(
10076 watched in "[A-Za-z0-9._/-]{1,32}",
10077 reviewer in "[A-Za-z0-9._/-]{1,32}",
10078 ) {
10079 let request = ReviewRequest::new(
10080 Agent::Codex,
10081 watched.clone(),
10082 ReviewerHarness::Codex,
10083 reviewer.clone(),
10084 false,
10085 "review this",
10086 );
10087 let result = ReviewPlan::build(request);
10088
10089 if normalized_model(watched.trim()) == normalized_model(reviewer.trim()) {
10093 let blocked = matches!(result, Err(ReviewerError::SameModelWithoutWaiver { .. }));
10094 prop_assert!(blocked);
10095 } else {
10096 prop_assert!(result.is_ok());
10097 }
10098 }
10099 }
10100
10101 fn claim() -> Claim {
10102 Claim::new(
10103 "add review",
10104 "cargo test",
10105 vec![EvidenceRef::parse("tests:cargo-test").unwrap()],
10106 )
10107 .unwrap()
10108 }
10109 #[test]
10110 fn batched_planner_keeps_queue_order_and_assigns_each_batch_one_job_slot() {
10111 let pending = vec![
10112 QueuedReview {
10113 run_id: "normal-1".to_owned(),
10114 commit_sha: "normal-1".to_owned(),
10115 ..Default::default()
10116 },
10117 QueuedReview {
10118 run_id: "petition-b".to_owned(),
10119 commit_sha: "fix-sha".to_owned(),
10120 petition_for: Some("rejection-b".to_owned()),
10121 ..Default::default()
10122 },
10123 QueuedReview {
10124 run_id: "petition-a".to_owned(),
10125 commit_sha: "fix-sha".to_owned(),
10126 petition_for: Some("rejection-a".to_owned()),
10127 ..Default::default()
10128 },
10129 QueuedReview {
10130 run_id: "normal-2".to_owned(),
10131 commit_sha: "normal-2".to_owned(),
10132 ..Default::default()
10133 },
10134 ];
10135
10136 let planned = plan_batched_drain(&pending, Path::new("/repo"), &selection(), "context", 32);
10137
10138 assert!(matches!(
10139 &planned[..],
10140 [
10141 PlannedDrainJob::Single(first),
10142 PlannedDrainJob::PetitionBatch(batch),
10143 PlannedDrainJob::Single(last),
10144 ] if first.commit_sha == "normal-1"
10145 && last.commit_sha == "normal-2"
10146 && batch.iter().map(|item| item.petition_for.as_deref()).collect::<Vec<_>>()
10147 == vec![Some("rejection-a"), Some("rejection-b")]
10148 ));
10149 assert_eq!(
10150 planned.len(),
10151 3,
10152 "one petition batch consumes one worker slot"
10153 );
10154 }
10155
10156 fn selection() -> ReviewSelection {
10157 ReviewSelection {
10158 watched_agent: Agent::Codex,
10159 watched_model: "gpt-5.4".to_owned(),
10160 reviewer_harness: ReviewerHarness::Codex,
10161 reviewer_model: "gpt-5.5".to_owned(),
10162 reviewer_effort: Effort::Xhigh,
10163 allow_same_model: false,
10164 strict: None,
10165 }
10166 }
10167
10168 fn synthetic_commit_shas(count: usize) -> Vec<String> {
10172 assert!(count >= 1);
10173 let tree = git_output(["rev-parse", "HEAD^{tree}"])
10174 .unwrap()
10175 .trim()
10176 .to_owned();
10177 let mut parent = git_output(["rev-parse", "HEAD"]).unwrap().trim().to_owned();
10178 let mut shas = vec![parent.clone()];
10179 for index in 1..count {
10180 let message = format!("truth-mirror-test-fixture-{index}");
10181 let output = Command::new("git")
10182 .env("GIT_AUTHOR_NAME", "truth-mirror-test")
10183 .env("GIT_AUTHOR_EMAIL", "truth-mirror-test@example.com")
10184 .env("GIT_COMMITTER_NAME", "truth-mirror-test")
10185 .env("GIT_COMMITTER_EMAIL", "truth-mirror-test@example.com")
10186 .args(["commit-tree", &tree, "-p", &parent, "-m", &message])
10187 .output()
10188 .expect("spawn git commit-tree");
10189 assert!(
10190 output.status.success(),
10191 "git commit-tree failed: {}",
10192 String::from_utf8_lossy(&output.stderr)
10193 );
10194 parent = String::from_utf8_lossy(&output.stdout).trim().to_owned();
10195 shas.push(parent.clone());
10196 }
10197 shas
10198 }
10199
10200 #[test]
10201 fn parallel_drain_caps_workers_uses_isolated_cwds_and_preserves_queue_order() {
10202 let temp = tempfile::tempdir().unwrap();
10203 let queue = ReviewQueue::new(temp.path());
10204 let store = LedgerStore::new(temp.path());
10205 let shas = synthetic_commit_shas(3);
10206 let first_sha = shas[0].clone();
10207 let second_sha = shas[1].clone();
10208 let third_sha = shas[2].clone();
10209 queue.enqueue(&first_sha).unwrap();
10210 queue.enqueue(&second_sha).unwrap();
10211 queue.enqueue(&third_sha).unwrap();
10212
10213 let mut config = TruthMirrorConfig::default();
10214 config.reviewer.max_concurrent_runs = 2;
10215 let runner = ConcurrentRunner::with_overlap(2);
10216 let report = drain_once_parallel(
10217 &queue,
10218 &StaticLoader::new(),
10219 &selection(),
10220 "",
10221 &runner,
10222 &store,
10223 &config,
10224 )
10225 .unwrap();
10226
10227 assert_eq!(
10228 report.reviewed,
10229 vec![first_sha, second_sha, third_sha],
10230 "report order follows the snapshot queue order, not completion order"
10231 );
10232 assert_eq!(
10233 runner.max_active.load(Ordering::SeqCst),
10234 2,
10235 "worker cap must bind concurrent reviewer processes"
10236 );
10237 let dirs = runner.dirs.lock().unwrap().clone();
10238 assert_eq!(dirs.len(), 3);
10239 let unique = dirs.iter().collect::<std::collections::BTreeSet<_>>();
10240 assert_eq!(
10241 unique.len(),
10242 3,
10243 "each job must use an isolated worktree cwd"
10244 );
10245 assert!(
10246 dirs.iter()
10247 .all(|dir| { dir.starts_with(temp.path().join("worktrees")) && !dir.exists() })
10248 );
10249 assert!(queue.pending().unwrap().is_empty());
10250 assert!(
10251 fs::read_dir(temp.path().join("worktrees"))
10252 .unwrap()
10253 .next()
10254 .is_none()
10255 );
10256 }
10257
10258 #[test]
10259 fn serial_default_runs_one_job_at_a_time_without_worktree_cwd() {
10260 let temp = tempfile::tempdir().unwrap();
10261 let queue = ReviewQueue::new(temp.path());
10262 let store = LedgerStore::new(temp.path());
10263 let shas = synthetic_commit_shas(2);
10264 let first_sha = shas[0].clone();
10265 let second_sha = shas[1].clone();
10266 queue.enqueue(&first_sha).unwrap();
10267 queue.enqueue(&second_sha).unwrap();
10268
10269 let config = TruthMirrorConfig::default();
10270 assert_eq!(config.reviewer.max_concurrent_runs, 1);
10271 let runner = ConcurrentRunner::new();
10272 let report = drain_once(
10273 &queue,
10274 &StaticLoader::new(),
10275 &selection(),
10276 "",
10277 &runner,
10278 &store,
10279 &config,
10280 )
10281 .unwrap();
10282
10283 assert_eq!(report.reviewed, vec![first_sha, second_sha]);
10284 assert_eq!(runner.max_active.load(Ordering::SeqCst), 1);
10285 assert!(
10286 runner.dirs.lock().unwrap().is_empty(),
10287 "serial default must keep the shared checkout (no per-job worktree cwd)"
10288 );
10289 assert!(!temp.path().join("worktrees").exists());
10290 assert!(queue.pending().unwrap().is_empty());
10291 }
10292
10293 #[test]
10294 fn parallel_once_snapshot_leaves_late_enqueues_for_later_invocation() {
10295 let temp = tempfile::tempdir().unwrap();
10296 let queue = ReviewQueue::new(temp.path());
10297 let store = LedgerStore::new(temp.path());
10298 let shas = synthetic_commit_shas(3);
10299 let first_sha = shas[0].clone();
10300 let second_sha = shas[1].clone();
10301 let late_sha = shas[2].clone();
10302 queue.enqueue(&first_sha).unwrap();
10303 queue.enqueue(&second_sha).unwrap();
10304
10305 let mut config = TruthMirrorConfig::default();
10306 config.reviewer.max_concurrent_runs = 2;
10307 let runner = SnapshotRunner::new(queue.clone(), late_sha.clone());
10308 let report = drain_once_parallel(
10309 &queue,
10310 &StaticLoader::new(),
10311 &selection(),
10312 "",
10313 &runner,
10314 &store,
10315 &config,
10316 )
10317 .unwrap();
10318
10319 assert_eq!(report.reviewed, vec![first_sha, second_sha]);
10320 let pending = queue.pending().unwrap();
10321 assert_eq!(pending.len(), 1);
10322 assert_eq!(pending[0].commit_sha, late_sha);
10323 }
10324
10325 #[test]
10326 fn durable_claim_precedes_spawn_and_stale_claim_recovers_without_losing_queue_row() {
10327 let temp = tempfile::tempdir().unwrap();
10328 let queue = ReviewQueue::new(temp.path());
10329 let run_store = ReviewRunStore::new(temp.path());
10330 let sha = "abc123deadbeef".to_owned();
10331 let item = queue.enqueue(&sha).unwrap();
10332 let job = PlannedDrainJob::Single(item);
10333 let claimed = claim_planned_job(&queue, &run_store, job).unwrap();
10334 let PlannedDrainJob::Single(claimed_item) = claimed else {
10335 panic!("expected single job");
10336 };
10337 assert!(!claimed_item.run_id.is_empty());
10338 let run = run_store.read(&claimed_item.run_id).unwrap();
10339 assert_eq!(run.status, ReviewRunStatus::Running);
10340 assert_eq!(run.phase, CLAIMED_PHASE);
10341 assert!(run.claimed_at_unix.is_some());
10342 assert_eq!(queue.pending_read_only().unwrap().len(), 1);
10343
10344 let mut crashed = run;
10345 crashed.worker_pid = Some(reaped_pid());
10346 run_store.write(&crashed).unwrap();
10347 let pending = queue.pending().unwrap();
10348 assert_eq!(pending.len(), 1);
10349 let recovered = run_store.read(&claimed_item.run_id).unwrap();
10350 assert_eq!(recovered.status, ReviewRunStatus::Queued);
10351 assert!(recovered.claimed_at_unix.is_none());
10352 assert!(recovered.worker_pid.is_none());
10353 }
10354
10355 #[test]
10356 fn provider_backoff_persists_without_spending_petition_attempts_or_removing_queue() {
10357 let error = ReviewerError::ReviewerProcessFailed {
10358 status: Some(429),
10359 stderr: "rate_limit exceeded: try again later".to_owned(),
10360 };
10361 assert!(is_retryable_provider_failure(&error));
10362
10363 let temp = tempfile::tempdir().unwrap();
10364 let queue = ReviewQueue::new(temp.path());
10365 let run_store = ReviewRunStore::new(temp.path());
10366 let store = LedgerStore::new(temp.path());
10367 let original = "a".repeat(40);
10368 let fix = "b".repeat(40);
10369 seed_petition(temp.path(), &store, &original, &fix, "finding");
10370 let pending_before = queue.pending().unwrap();
10371 assert_eq!(pending_before.len(), 1);
10372 let run_id = pending_before[0].run_id.clone();
10373 let before_attempts = store.petition_attempts_for(&original).unwrap();
10374
10375 run_store
10376 .mark_provider_backoff(&run_id, error.to_string(), DEFAULT_PROVIDER_BACKOFF_SECS)
10377 .unwrap();
10378 let run = run_store.read(&run_id).unwrap();
10379 assert_eq!(run.status, ReviewRunStatus::Queued);
10380 assert!(run.next_attempt_at_unix.is_some());
10381 assert_eq!(queue.pending_read_only().unwrap().len(), 1);
10382
10383 let filtered = filter_pending_ready_for_attempt(
10384 queue.pending_read_only().unwrap(),
10385 &run_store,
10386 unix_now(),
10387 )
10388 .unwrap();
10389 assert!(filtered.is_empty(), "backoff must defer the attempt");
10390
10391 let after_attempts = store.petition_attempts_for(&original).unwrap();
10392 assert_eq!(before_attempts, after_attempts);
10393
10394 let eligible = filter_pending_ready_for_attempt(
10395 queue.pending_read_only().unwrap(),
10396 &run_store,
10397 run.next_attempt_at_unix.unwrap(),
10398 )
10399 .unwrap();
10400 assert_eq!(eligible.len(), 1);
10401 }
10402
10403 #[test]
10404 fn strict_arbiter_provider_backoff_persists_without_terminating_drain() {
10405 let temp = tempfile::tempdir().unwrap();
10406 let store = LedgerStore::new(temp.path());
10407 let queue = ReviewQueue::new(temp.path());
10408 let run_store = ReviewRunStore::new(temp.path());
10409 seed_petition(
10410 temp.path(),
10411 &store,
10412 "orig-a",
10413 "fix-strict-backoff",
10414 "finding-a",
10415 );
10416 seed_petition(
10418 temp.path(),
10419 &store,
10420 "orig-b",
10421 "fix-strict-backoff",
10422 "finding-b",
10423 );
10424 let before_a = store.petition_attempts_for("orig-a").unwrap();
10425 let before_b = store.petition_attempts_for("orig-b").unwrap();
10426 let runner = RecordingBatchRunner::new([
10427 BatchReply::Results(vec![
10428 batch_result("orig-a", &pass_json()),
10429 batch_result("orig-b", &flag_json("non-accepting debt")),
10430 ]),
10431 BatchReply::ProcessFailed {
10432 status: Some(429),
10433 stderr: "rate_limit exceeded: try again later".to_owned(),
10434 },
10435 ]);
10436 let mut strict_selection = selection();
10437 strict_selection.strict = Some(StrictReviewConfig {
10438 arbiter_harness: ReviewerHarness::Claude,
10439 arbiter_model: "claude-opus-4-8".to_owned(),
10440 arbiter_effort: Effort::Xhigh,
10441 });
10442
10443 let report = drain_once(
10444 &queue,
10445 &StaticLoader::new(),
10446 &strict_selection,
10447 "",
10448 &runner,
10449 &store,
10450 &batching_config(32),
10451 )
10452 .expect("strict arbiter 429 must not terminate drain/watch");
10453
10454 assert!(report.failed.is_empty());
10455 assert_eq!(report.reviewed, ["fix-strict-backoff"]);
10457 assert_eq!(store.petition_attempts_for("orig-a").unwrap(), before_a);
10459 assert_eq!(store.petition_attempts_for("orig-b").unwrap(), before_b);
10460
10461 let pending = queue.pending_read_only().unwrap();
10462 assert_eq!(pending.len(), 1);
10463 assert_eq!(pending[0].petition_for.as_deref(), Some("orig-a"));
10464 let run = run_store.read(&pending[0].run_id).unwrap();
10465 assert_eq!(run.status, ReviewRunStatus::Queued);
10466 assert_eq!(run.phase, PROVIDER_BACKOFF_PHASE);
10467 assert!(run.next_attempt_at_unix.is_some());
10468
10469 let deferred = filter_pending_ready_for_attempt(
10470 queue.pending_read_only().unwrap(),
10471 &run_store,
10472 unix_now(),
10473 )
10474 .unwrap();
10475 assert!(
10476 deferred.is_empty(),
10477 "strict 429 backoff must defer next attempt"
10478 );
10479
10480 let eligible = filter_pending_ready_for_attempt(
10481 queue.pending_read_only().unwrap(),
10482 &run_store,
10483 run.next_attempt_at_unix.unwrap(),
10484 )
10485 .unwrap();
10486 assert_eq!(eligible.len(), 1);
10487 assert_eq!(runner.prompts().len(), 2);
10488 }
10489
10490 #[test]
10491 fn orphan_worktree_maintenance_removes_terminal_paths_with_non_fatal_telemetry() {
10492 let temp = tempfile::tempdir().unwrap();
10493 let repository =
10494 PathBuf::from(git_output(["rev-parse", "--show-toplevel"]).unwrap().trim());
10495 let orphan = temp.path().join("worktrees").join("orphan-run");
10496 fs::create_dir_all(orphan.parent().unwrap()).unwrap();
10497 let sha = git_output(["rev-parse", "HEAD"]).unwrap().trim().to_owned();
10498 create_review_worktree(&repository, &orphan, &sha).unwrap();
10499 assert!(orphan.exists());
10500 maintain_terminal_worktrees(temp.path(), &repository);
10501 assert!(
10502 !orphan.exists(),
10503 "orphan worktree should be removed by maintenance cleanup"
10504 );
10505 }
10506
10507 #[test]
10508 fn scheduler_status_read_only_reports_owner_claimed_inflight_workers_and_batches() {
10509 let temp = tempfile::tempdir().unwrap();
10510 let run_store = ReviewRunStore::new(temp.path());
10511 let mut claimed = run_store.create_queued("sha-claimed", "commit").unwrap();
10512 claimed.mark_claimed();
10513 run_store.write(&claimed).unwrap();
10514 let mut inflight = run_store.create_queued("sha-inflight", "commit").unwrap();
10515 inflight.mark_claimed();
10516 inflight.mark_running(REVIEW_LEDGER_PHASE);
10517 inflight.batch = Some(super::BatchRunRef {
10518 batch_id: "batch-status".to_owned(),
10519 kind: super::BatchKind::Petition,
10520 member_index: 0,
10521 member_count: 1,
10522 fix_sha: "sha-inflight".to_owned(),
10523 });
10524 run_store.write(&inflight).unwrap();
10525
10526 let snapshot = run_store
10527 .scheduler_status_read_only(Some(4242), ["batch-queue".to_owned()])
10528 .unwrap();
10529 assert_eq!(snapshot.owner_pid, Some(4242));
10530 assert_eq!(snapshot.claimed, 2);
10531 assert_eq!(snapshot.inflight, 1);
10532 assert_eq!(snapshot.active_workers, 1);
10533 assert_eq!(snapshot.batches, 2);
10534 }
10535
10536 struct StaticLoader {
10537 claim: Claim,
10538 diff: String,
10539 }
10540
10541 impl StaticLoader {
10542 fn new() -> Self {
10543 Self {
10544 claim: claim(),
10545 diff: "diff --git a/src/lib.rs b/src/lib.rs".to_owned(),
10546 }
10547 }
10548 }
10549
10550 impl MaterialLoader for StaticLoader {
10551 fn load(&self, _sha: &str) -> Result<(Claim, String), ReviewerError> {
10552 Ok((self.claim.clone(), self.diff.clone()))
10553 }
10554 }
10555
10556 struct CountingLoader {
10557 calls: RefCell<usize>,
10558 inner: StaticLoader,
10559 }
10560
10561 impl CountingLoader {
10562 fn new() -> Self {
10563 Self {
10564 calls: RefCell::new(0),
10565 inner: StaticLoader::new(),
10566 }
10567 }
10568
10569 fn calls(&self) -> usize {
10570 *self.calls.borrow()
10571 }
10572 }
10573
10574 impl MaterialLoader for CountingLoader {
10575 fn load(&self, sha: &str) -> Result<(Claim, String), ReviewerError> {
10576 *self.calls.borrow_mut() += 1;
10577 self.inner.load(sha)
10578 }
10579 }
10580
10581 fn seed_petition(
10582 state_dir: &std::path::Path,
10583 store: &LedgerStore,
10584 original_sha: &str,
10585 fix_sha: &str,
10586 finding: &str,
10587 ) {
10588 let rejected = crate::ledger::LedgerEntry::new(
10589 original_sha,
10590 Verdict::Reject,
10591 format!(
10592 "CLAIM: original {original_sha} | verified: cargo test | evidence: tests:cargo-test"
10593 ),
10594 vec!["tests:cargo-test".to_owned()],
10595 crate::ledger::ReviewerConfig::new("claude", "claude-opus-4-1", false),
10596 vec![finding.to_owned()],
10597 );
10598 store.append_entry(&rejected).unwrap();
10599 super::commit_petition_resolve(state_dir, store, original_sha, fix_sha).unwrap();
10600 }
10601
10602 fn batching_config(max_petition_batch_size: usize) -> TruthMirrorConfig {
10603 let mut config = TruthMirrorConfig::default();
10604 config.reviewer.batch_petitions = true;
10605 config.reviewer.max_petition_batch_size = max_petition_batch_size;
10606 config
10607 }
10608
10609 struct PoisonedLoader;
10610
10611 impl MaterialLoader for PoisonedLoader {
10612 fn load(&self, sha: &str) -> Result<(Claim, String), ReviewerError> {
10613 if sha == "poisoned" {
10614 return Err(crate::claim::ClaimError::MissingEvidence.into());
10615 }
10616 StaticLoader::new().load(sha)
10617 }
10618 }
10619
10620 struct ConstRunner {
10621 output: String,
10622 }
10623
10624 impl ConstRunner {
10625 fn new(output: impl Into<String>) -> Self {
10626 Self {
10627 output: output.into(),
10628 }
10629 }
10630 }
10631
10632 impl ProcessRunner for ConstRunner {
10633 fn run(
10634 &self,
10635 _invocation: &InvocationPlan,
10636 _prompt: &str,
10637 _timeout: Option<Duration>,
10638 ) -> Result<ProcessOutput, ReviewerError> {
10639 Ok(ProcessOutput {
10640 status_code: Some(0),
10641 stdout: self.output.clone(),
10642 stderr: String::new(),
10643 })
10644 }
10645 }
10646
10647 struct ConcurrentRunner {
10648 active: AtomicUsize,
10649 max_active: AtomicUsize,
10650 arrivals: AtomicUsize,
10651 overlap: usize,
10652 dirs: Mutex<Vec<PathBuf>>,
10653 }
10654
10655 impl ConcurrentRunner {
10656 fn new() -> Self {
10657 Self::with_overlap(1)
10658 }
10659
10660 fn with_overlap(overlap: usize) -> Self {
10661 Self {
10662 active: AtomicUsize::new(0),
10663 max_active: AtomicUsize::new(0),
10664 arrivals: AtomicUsize::new(0),
10665 overlap: overlap.max(1),
10666 dirs: Mutex::new(Vec::new()),
10667 }
10668 }
10669
10670 fn record_max(&self, active: usize) {
10671 let mut observed = self.max_active.load(Ordering::Relaxed);
10672 while active > observed {
10673 match self.max_active.compare_exchange(
10674 observed,
10675 active,
10676 Ordering::Relaxed,
10677 Ordering::Relaxed,
10678 ) {
10679 Ok(_) => break,
10680 Err(current) => observed = current,
10681 }
10682 }
10683 }
10684
10685 fn observe(&self, current_dir: Option<&Path>) {
10686 if let Some(dir) = current_dir {
10687 self.dirs.lock().unwrap().push(dir.to_owned());
10688 }
10689 let ticket = self.arrivals.fetch_add(1, Ordering::SeqCst);
10690 let active = self.active.fetch_add(1, Ordering::SeqCst) + 1;
10691 self.record_max(active);
10692 if current_dir.is_some() && ticket < self.overlap {
10693 let started = std::time::Instant::now();
10694 while self.arrivals.load(Ordering::SeqCst) < self.overlap
10695 && started.elapsed() < Duration::from_secs(10)
10696 {
10697 thread::sleep(Duration::from_millis(10));
10698 }
10699 self.record_max(self.active.load(Ordering::SeqCst));
10700 thread::sleep(Duration::from_millis(50));
10701 } else {
10702 thread::sleep(Duration::from_millis(20));
10703 }
10704 self.active.fetch_sub(1, Ordering::SeqCst);
10705 }
10706 }
10707
10708 impl ProcessRunner for ConcurrentRunner {
10709 fn run(
10710 &self,
10711 _invocation: &InvocationPlan,
10712 _prompt: &str,
10713 _timeout: Option<Duration>,
10714 ) -> Result<ProcessOutput, ReviewerError> {
10715 self.observe(None);
10716 Ok(ProcessOutput {
10717 status_code: Some(0),
10718 stdout: pass_json(),
10719 stderr: String::new(),
10720 })
10721 }
10722
10723 fn run_in_dir(
10724 &self,
10725 invocation: &InvocationPlan,
10726 prompt: &str,
10727 timeout: Option<Duration>,
10728 current_dir: Option<&Path>,
10729 ) -> Result<ProcessOutput, ReviewerError> {
10730 let _ = timeout;
10731 if current_dir.is_some() {
10732 self.observe(current_dir);
10733 Ok(ProcessOutput {
10734 status_code: Some(0),
10735 stdout: pass_json(),
10736 stderr: String::new(),
10737 })
10738 } else {
10739 self.run(invocation, prompt, timeout)
10740 }
10741 }
10742 }
10743
10744 struct SnapshotRunner {
10745 queue: ReviewQueue,
10746 late_sha: String,
10747 enqueued_late: AtomicBool,
10748 inner: ConcurrentRunner,
10749 }
10750
10751 impl SnapshotRunner {
10752 fn new(queue: ReviewQueue, late_sha: String) -> Self {
10753 Self {
10754 queue,
10755 late_sha,
10756 enqueued_late: AtomicBool::new(false),
10757 inner: ConcurrentRunner::new(),
10758 }
10759 }
10760
10761 fn maybe_enqueue_late(&self) {
10762 if self
10763 .enqueued_late
10764 .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
10765 .is_ok()
10766 {
10767 self.queue.enqueue(&self.late_sha).unwrap();
10768 }
10769 }
10770 }
10771
10772 impl ProcessRunner for SnapshotRunner {
10773 fn run(
10774 &self,
10775 invocation: &InvocationPlan,
10776 prompt: &str,
10777 timeout: Option<Duration>,
10778 ) -> Result<ProcessOutput, ReviewerError> {
10779 self.maybe_enqueue_late();
10780 self.inner.run(invocation, prompt, timeout)
10781 }
10782
10783 fn run_in_dir(
10784 &self,
10785 invocation: &InvocationPlan,
10786 prompt: &str,
10787 timeout: Option<Duration>,
10788 current_dir: Option<&Path>,
10789 ) -> Result<ProcessOutput, ReviewerError> {
10790 self.maybe_enqueue_late();
10791 self.inner
10792 .run_in_dir(invocation, prompt, timeout, current_dir)
10793 }
10794 }
10795
10796 struct PanicRunner;
10797
10798 impl ProcessRunner for PanicRunner {
10799 fn run(
10800 &self,
10801 _invocation: &InvocationPlan,
10802 _prompt: &str,
10803 _timeout: Option<Duration>,
10804 ) -> Result<ProcessOutput, ReviewerError> {
10805 panic!("provider must not run when pending_review spool is durable");
10806 }
10807 }
10808
10809 fn review_job(strict: bool) -> ReviewJob {
10810 let claim = claim();
10811 ReviewJob {
10812 commit_sha: "abc123".to_owned(),
10813 diff: "diff --git a/src/lib.rs b/src/lib.rs".to_owned(),
10814 context: String::new(),
10815 request: ReviewRequest::new(
10816 Agent::Codex,
10817 "gpt-5.4",
10818 ReviewerHarness::Codex,
10819 "gpt-5.5",
10820 false,
10821 "review this",
10822 ),
10823 claim,
10824 strict: strict.then_some(StrictReviewConfig {
10825 arbiter_harness: ReviewerHarness::Claude,
10826 arbiter_model: "claude-opus-4-8".to_owned(),
10827 arbiter_effort: Effort::Xhigh,
10828 }),
10829 petition: None,
10830 }
10831 }
10832
10833 struct SequenceRunner {
10834 outputs: RefCell<VecDeque<String>>,
10835 }
10836
10837 impl SequenceRunner {
10838 fn new<I, S>(outputs: I) -> Self
10839 where
10840 I: IntoIterator<Item = S>,
10841 S: Into<String>,
10842 {
10843 Self {
10844 outputs: RefCell::new(outputs.into_iter().map(Into::into).collect()),
10845 }
10846 }
10847 }
10848
10849 impl ProcessRunner for SequenceRunner {
10850 fn run(
10851 &self,
10852 _invocation: &InvocationPlan,
10853 _prompt: &str,
10854 _timeout: Option<Duration>,
10855 ) -> Result<ProcessOutput, ReviewerError> {
10856 let stdout = self.outputs.borrow_mut().pop_front().unwrap();
10857 Ok(ProcessOutput {
10858 status_code: Some(0),
10859 stdout,
10860 stderr: String::new(),
10861 })
10862 }
10863 }
10864
10865 enum BatchReply {
10866 Results(Vec<serde_json::Value>),
10867 PassEveryRequestedMember,
10868 MalformedResultsArray,
10869 Raw(String),
10870 ProcessFailed { status: Option<i32>, stderr: String },
10871 }
10872
10873 struct RecordingBatchRunner {
10874 replies: RefCell<VecDeque<BatchReply>>,
10875 prompts: RefCell<Vec<String>>,
10876 }
10877
10878 impl RecordingBatchRunner {
10879 fn new(replies: impl IntoIterator<Item = BatchReply>) -> Self {
10880 Self {
10881 replies: RefCell::new(replies.into_iter().collect()),
10882 prompts: RefCell::new(Vec::new()),
10883 }
10884 }
10885
10886 fn prompts(&self) -> Vec<String> {
10887 self.prompts.borrow().clone()
10888 }
10889 }
10890
10891 fn batch_prompt_value(prompt: &str, prefix: &str) -> String {
10892 prompt
10893 .lines()
10894 .find_map(|line| line.strip_prefix(prefix))
10895 .unwrap_or_else(|| panic!("batch prompt missing {prefix:?}"))
10896 .to_owned()
10897 }
10898
10899 fn batch_prompt_requested_shas(prompt: &str) -> Vec<String> {
10900 prompt
10901 .lines()
10902 .filter_map(|line| line.split_once(". ORIGINAL REJECTION SHA: "))
10903 .map(|(_, sha)| sha.to_owned())
10904 .collect()
10905 }
10906
10907 impl ProcessRunner for RecordingBatchRunner {
10908 fn run(
10909 &self,
10910 _invocation: &InvocationPlan,
10911 prompt: &str,
10912 _timeout: Option<Duration>,
10913 ) -> Result<ProcessOutput, ReviewerError> {
10914 self.prompts.borrow_mut().push(prompt.to_owned());
10915 let reply = self.replies.borrow_mut().pop_front().unwrap();
10916 if let BatchReply::ProcessFailed { status, stderr } = reply {
10917 return Ok(ProcessOutput {
10918 status_code: status,
10919 stdout: String::new(),
10920 stderr,
10921 });
10922 }
10923 let stdout = match reply {
10924 BatchReply::Raw(output) => output,
10925 reply => {
10926 let batch_id = batch_prompt_value(prompt, "- batch ID: ");
10927 let fix_sha = batch_prompt_value(prompt, "- fix commit SHA: ");
10928 let results = match reply {
10929 BatchReply::Results(results) => serde_json::Value::Array(results),
10930 BatchReply::PassEveryRequestedMember => serde_json::Value::Array(
10931 batch_prompt_requested_shas(prompt)
10932 .into_iter()
10933 .map(|sha| batch_result(&sha, &pass_json()))
10934 .collect(),
10935 ),
10936 BatchReply::MalformedResultsArray => {
10937 serde_json::json!({"not": "an array"})
10938 }
10939 BatchReply::Raw(_) | BatchReply::ProcessFailed { .. } => unreachable!(),
10940 };
10941 serde_json::json!({
10942 "batch_id": batch_id,
10943 "fix_sha": fix_sha,
10944 "results": results,
10945 })
10946 .to_string()
10947 }
10948 };
10949 Ok(ProcessOutput {
10950 status_code: Some(0),
10951 stdout,
10952 stderr: String::new(),
10953 })
10954 }
10955 }
10956
10957 #[test]
10958 fn petition_batch_key_covers_repository_fix_selection_strict_and_context() {
10959 let base = selection();
10960 let mut different_reviewer = base.clone();
10961 different_reviewer.reviewer_model = "different-reviewer".to_owned();
10962 let mut strict = base.clone();
10963 strict.strict = Some(StrictReviewConfig {
10964 arbiter_harness: ReviewerHarness::Claude,
10965 arbiter_model: "claude-opus-4-8".to_owned(),
10966 arbiter_effort: Effort::High,
10967 });
10968 let key =
10969 super::PetitionBatchKey::new(std::path::Path::new("repo-a"), "fix-a", &base, "ctx-a");
10970
10971 assert_ne!(
10972 key,
10973 super::PetitionBatchKey::new(std::path::Path::new("repo-b"), "fix-a", &base, "ctx-a")
10974 );
10975 assert_ne!(
10976 key,
10977 super::PetitionBatchKey::new(std::path::Path::new("repo-a"), "fix-b", &base, "ctx-a")
10978 );
10979 assert_ne!(
10980 key,
10981 super::PetitionBatchKey::new(
10982 std::path::Path::new("repo-a"),
10983 "fix-a",
10984 &different_reviewer,
10985 "ctx-a"
10986 )
10987 );
10988 assert_ne!(
10989 key,
10990 super::PetitionBatchKey::new(std::path::Path::new("repo-a"), "fix-a", &strict, "ctx-a")
10991 );
10992 assert_ne!(
10993 key,
10994 super::PetitionBatchKey::new(std::path::Path::new("repo-a"), "fix-a", &base, "ctx-b")
10995 );
10996 }
10997
10998 #[test]
10999 fn petition_batch_groups_same_fix_once_and_applies_sha_keyed_outcomes() {
11000 let temp = tempfile::tempdir().unwrap();
11001 let store = LedgerStore::new(temp.path());
11002 let queue = ReviewQueue::new(temp.path());
11003 for (original, finding) in [
11004 ("orig-c", "finding-c"),
11005 ("orig-a", "finding-a"),
11006 ("orig-b", "finding-b"),
11007 ] {
11008 seed_petition(temp.path(), &store, original, "fix-1", finding);
11009 }
11010 let loader = CountingLoader::new();
11011 let runner = RecordingBatchRunner::new([BatchReply::Results(vec![
11012 batch_result("orig-b", &reject_json("still broken")),
11013 batch_result("orig-a", &pass_json()),
11014 batch_result("orig-c", &flag_json("debt remains")),
11015 ])]);
11016
11017 let report = drain_once(
11018 &queue,
11019 &loader,
11020 &selection(),
11021 "MATERIALIZED CONTEXT",
11022 &runner,
11023 &store,
11024 &batching_config(32),
11025 )
11026 .unwrap();
11027
11028 assert_eq!(loader.calls(), 1);
11029 assert_eq!(runner.prompts().len(), 1);
11030 assert_eq!(report.reviewed, ["fix-1", "fix-1", "fix-1"]);
11031 assert!(report.failed.is_empty());
11032 let prompt = &runner.prompts()[0];
11033 assert_eq!(
11034 prompt
11035 .matches("diff --git a/src/lib.rs b/src/lib.rs")
11036 .count(),
11037 1
11038 );
11039 assert!(prompt.contains("finding-a"));
11040 assert!(prompt.contains("finding-b"));
11041 assert!(prompt.contains("finding-c"));
11042 let a = prompt.find("ORIGINAL REJECTION SHA: orig-a").unwrap();
11043 let b = prompt.find("ORIGINAL REJECTION SHA: orig-b").unwrap();
11044 let c = prompt.find("ORIGINAL REJECTION SHA: orig-c").unwrap();
11045 assert!(a < b && b < c);
11046
11047 assert_eq!(
11048 store.show("orig-a").unwrap().disposition,
11049 crate::ledger::Disposition::Resolved
11050 );
11051 assert_eq!(
11052 store.show("orig-b").unwrap().disposition,
11053 crate::ledger::Disposition::Open
11054 );
11055 assert_eq!(
11056 store.show("orig-c").unwrap().disposition,
11057 crate::ledger::Disposition::Open
11058 );
11059 for original in ["orig-a", "orig-b", "orig-c"] {
11060 assert_eq!(store.petition_attempts_for(original).unwrap(), 1);
11061 }
11062 let petition_entries: Vec<_> = store
11063 .read_history()
11064 .unwrap()
11065 .into_iter()
11066 .filter(|entry| entry.commit_sha == "fix-1")
11067 .collect();
11068 assert_eq!(petition_entries.len(), 3);
11069 assert!(
11070 petition_entries
11071 .iter()
11072 .all(|entry| !entry.raw_reviewer_output.contains("\"results\""))
11073 );
11074 assert!(queue.pending().unwrap().is_empty());
11075 }
11076
11077 #[test]
11078 fn petition_batch_isolates_duplicate_missing_unknown_and_malformed_members() {
11079 let temp = tempfile::tempdir().unwrap();
11080 let store = LedgerStore::new(temp.path());
11081 let queue = ReviewQueue::new(temp.path());
11082 for original in ["orig-a", "orig-b", "orig-c", "orig-d"] {
11083 seed_petition(
11084 temp.path(),
11085 &store,
11086 original,
11087 "fix-isolation",
11088 &format!("finding-{original}"),
11089 );
11090 }
11091 let mut malformed = batch_result("orig-d", &pass_json());
11092 malformed
11093 .as_object_mut()
11094 .unwrap()
11095 .insert("summary".to_owned(), serde_json::json!(""));
11096 let runner = RecordingBatchRunner::new([BatchReply::Results(vec![
11097 batch_result("orig-a", &pass_json()),
11098 batch_result("orig-b", &pass_json()),
11099 batch_result("orig-b", &pass_json()),
11100 malformed,
11101 batch_result("unknown-original", &pass_json()),
11102 ])]);
11103
11104 let report = drain_once(
11105 &queue,
11106 &StaticLoader::new(),
11107 &selection(),
11108 "",
11109 &runner,
11110 &store,
11111 &batching_config(32),
11112 )
11113 .unwrap();
11114
11115 assert_eq!(report.reviewed, ["fix-isolation"]);
11116 assert_eq!(report.failed.len(), 3);
11117 assert_eq!(
11118 store.show("orig-a").unwrap().disposition,
11119 crate::ledger::Disposition::Resolved
11120 );
11121 for original in ["orig-b", "orig-c", "orig-d"] {
11122 assert_eq!(
11123 store.show(original).unwrap().disposition,
11124 crate::ledger::Disposition::Open
11125 );
11126 assert_eq!(store.petition_attempts_for(original).unwrap(), 1);
11127 }
11128 assert_eq!(
11129 store
11130 .read_history()
11131 .unwrap()
11132 .iter()
11133 .filter(|entry| entry.commit_sha == "fix-isolation")
11134 .count(),
11135 1
11136 );
11137 let runs = ReviewRunStore::new(temp.path()).list().unwrap();
11138 for (original, expected_error) in [
11139 ("orig-b", "duplicate batch results"),
11140 ("orig-c", "missing batch result"),
11141 ("orig-d", "summary must not be empty"),
11142 ] {
11143 let run = runs
11144 .iter()
11145 .find(|run| run.petition_for.as_deref() == Some(original))
11146 .unwrap();
11147 assert_eq!(run.status, ReviewRunStatus::Failed);
11148 assert!(run.error.as_deref().unwrap().contains(expected_error));
11149 }
11150 assert!(queue.pending().unwrap().is_empty());
11151 let (_, attempts) =
11152 super::commit_petition_resolve(temp.path(), &store, "orig-b", "fix-retry").unwrap();
11153 assert_eq!(attempts, 2);
11154 let retry = queue.pending().unwrap();
11155 assert_eq!(retry.len(), 1);
11156 assert_eq!(retry[0].commit_sha, "fix-retry");
11157 assert_eq!(retry[0].petition_for.as_deref(), Some("orig-b"));
11158 }
11159
11160 #[test]
11161 fn malformed_outer_petition_batch_fails_every_member_without_a_verdict() {
11162 let temp = tempfile::tempdir().unwrap();
11163 let store = LedgerStore::new(temp.path());
11164 let queue = ReviewQueue::new(temp.path());
11165 for original in ["orig-a", "orig-b"] {
11166 seed_petition(
11167 temp.path(),
11168 &store,
11169 original,
11170 "fix-outer",
11171 &format!("finding-{original}"),
11172 );
11173 }
11174 let runner = RecordingBatchRunner::new([BatchReply::MalformedResultsArray]);
11175
11176 let report = drain_once(
11177 &queue,
11178 &StaticLoader::new(),
11179 &selection(),
11180 "",
11181 &runner,
11182 &store,
11183 &batching_config(32),
11184 )
11185 .unwrap();
11186
11187 assert!(report.reviewed.is_empty());
11188 assert_eq!(report.failed, ["fix-outer", "fix-outer"]);
11189 assert_eq!(
11190 store
11191 .read_history()
11192 .unwrap()
11193 .iter()
11194 .filter(|entry| entry.commit_sha == "fix-outer")
11195 .count(),
11196 0
11197 );
11198 for original in ["orig-a", "orig-b"] {
11199 assert_eq!(store.petition_attempts_for(original).unwrap(), 1);
11200 }
11201 assert!(queue.pending().unwrap().is_empty());
11202 }
11203
11204 #[test]
11205 fn petition_batch_size_splits_deterministically_at_the_configured_cap() {
11206 let temp = tempfile::tempdir().unwrap();
11207 let store = LedgerStore::new(temp.path());
11208 let queue = ReviewQueue::new(temp.path());
11209 for original in ["orig-e", "orig-c", "orig-a", "orig-d", "orig-b"] {
11210 seed_petition(
11211 temp.path(),
11212 &store,
11213 original,
11214 "fix-capped",
11215 &format!("finding-{original}"),
11216 );
11217 }
11218 let runner = RecordingBatchRunner::new([
11219 BatchReply::PassEveryRequestedMember,
11220 BatchReply::PassEveryRequestedMember,
11221 BatchReply::PassEveryRequestedMember,
11222 ]);
11223
11224 drain_once(
11225 &queue,
11226 &StaticLoader::new(),
11227 &selection(),
11228 "",
11229 &runner,
11230 &store,
11231 &batching_config(2),
11232 )
11233 .unwrap();
11234
11235 let requested: Vec<Vec<String>> = runner
11236 .prompts()
11237 .iter()
11238 .map(|prompt| batch_prompt_requested_shas(prompt))
11239 .collect();
11240 assert_eq!(
11241 requested,
11242 [
11243 vec!["orig-a".to_owned(), "orig-b".to_owned()],
11244 vec!["orig-c".to_owned(), "orig-d".to_owned()],
11245 vec!["orig-e".to_owned()],
11246 ]
11247 );
11248 assert!(runner.prompts().iter().all(|prompt| {
11249 prompt
11250 .matches("FIX COMMIT DIFF (included exactly once)")
11251 .count()
11252 == 1
11253 }));
11254 }
11255
11256 #[test]
11257 fn strict_petition_batch_arbitrates_only_provisional_pass_members() {
11258 let temp = tempfile::tempdir().unwrap();
11259 let store = LedgerStore::new(temp.path());
11260 let queue = ReviewQueue::new(temp.path());
11261 for original in ["orig-a", "orig-b"] {
11262 seed_petition(
11263 temp.path(),
11264 &store,
11265 original,
11266 "fix-strict",
11267 &format!("finding-{original}"),
11268 );
11269 }
11270 let runner = RecordingBatchRunner::new([
11271 BatchReply::Results(vec![
11272 batch_result("orig-a", &pass_json()),
11273 batch_result("orig-b", &flag_json("non-accepting debt")),
11274 ]),
11275 BatchReply::Results(vec![batch_result(
11276 "orig-a",
11277 &reject_json("arbiter found gap"),
11278 )]),
11279 ]);
11280 let mut strict_selection = selection();
11281 strict_selection.strict = Some(StrictReviewConfig {
11282 arbiter_harness: ReviewerHarness::Claude,
11283 arbiter_model: "claude-opus-4-8".to_owned(),
11284 arbiter_effort: Effort::Xhigh,
11285 });
11286
11287 drain_once(
11288 &queue,
11289 &StaticLoader::new(),
11290 &strict_selection,
11291 "",
11292 &runner,
11293 &store,
11294 &batching_config(32),
11295 )
11296 .unwrap();
11297
11298 let prompts = runner.prompts();
11299 assert_eq!(prompts.len(), 2);
11300 assert_eq!(
11301 batch_prompt_requested_shas(&prompts[0]),
11302 ["orig-a", "orig-b"]
11303 );
11304 assert_eq!(batch_prompt_requested_shas(&prompts[1]), ["orig-a"]);
11305 assert!(!prompts[1].contains("orig-b"));
11306 assert_eq!(
11307 store.show("orig-a").unwrap().disposition,
11308 crate::ledger::Disposition::Open
11309 );
11310 assert_eq!(
11311 store.show("orig-b").unwrap().disposition,
11312 crate::ledger::Disposition::Open
11313 );
11314 let history = store.read_history().unwrap();
11315 let a_verdicts: Vec<_> = history
11316 .iter()
11317 .filter(|entry| {
11318 entry.commit_sha == "fix-strict" && entry.petition_for.as_deref() == Some("orig-a")
11319 })
11320 .map(|entry| entry.verdict)
11321 .collect();
11322 assert_eq!(a_verdicts, [Verdict::Pass, Verdict::Reject]);
11323 let audit_order: Vec<_> = history
11324 .iter()
11325 .filter(|entry| entry.commit_sha == "fix-strict")
11326 .map(|entry| entry.petition_for.as_deref().unwrap())
11327 .collect();
11328 assert_eq!(audit_order, ["orig-a", "orig-a", "orig-b"]);
11329 assert!(history.iter().any(|entry| {
11330 entry.commit_sha == "fix-strict"
11331 && entry.petition_for.as_deref() == Some("orig-b")
11332 && entry.verdict == Verdict::Flag
11333 }));
11334 }
11335
11336 #[test]
11337 fn enabled_batching_keeps_normal_review_separate_from_same_fix_petitions() {
11338 let temp = tempfile::tempdir().unwrap();
11339 let store = LedgerStore::new(temp.path());
11340 let queue = ReviewQueue::new(temp.path());
11341 queue.enqueue("fix-mixed").unwrap();
11342 for original in ["orig-a", "orig-b"] {
11343 seed_petition(
11344 temp.path(),
11345 &store,
11346 original,
11347 "fix-mixed",
11348 &format!("finding-{original}"),
11349 );
11350 }
11351 let loader = CountingLoader::new();
11352 let runner = RecordingBatchRunner::new([
11353 BatchReply::Raw(pass_json()),
11354 BatchReply::PassEveryRequestedMember,
11355 ]);
11356
11357 let report = drain_once(
11358 &queue,
11359 &loader,
11360 &selection(),
11361 "",
11362 &runner,
11363 &store,
11364 &batching_config(32),
11365 )
11366 .unwrap();
11367
11368 assert_eq!(loader.calls(), 2);
11369 assert_eq!(runner.prompts().len(), 2);
11370 assert!(!runner.prompts()[0].contains("BATCH:"));
11371 assert_eq!(
11372 batch_prompt_requested_shas(&runner.prompts()[1]),
11373 ["orig-a", "orig-b"]
11374 );
11375 assert_eq!(report.reviewed, ["fix-mixed", "fix-mixed", "fix-mixed"]);
11376 let fix_entries: Vec<_> = store
11377 .read_history()
11378 .unwrap()
11379 .into_iter()
11380 .filter(|entry| entry.commit_sha == "fix-mixed")
11381 .collect();
11382 assert_eq!(fix_entries.len(), 3);
11383 assert_eq!(
11384 fix_entries
11385 .iter()
11386 .filter(|entry| entry.petition_for.is_none())
11387 .count(),
11388 1
11389 );
11390 assert!(queue.pending().unwrap().is_empty());
11391 }
11392
11393 #[test]
11394 fn disabled_petition_batching_preserves_one_invocation_per_petition() {
11395 let temp = tempfile::tempdir().unwrap();
11396 let store = LedgerStore::new(temp.path());
11397 let queue = ReviewQueue::new(temp.path());
11398 for original in ["orig-a", "orig-b"] {
11399 seed_petition(
11400 temp.path(),
11401 &store,
11402 original,
11403 "fix-legacy",
11404 &format!("finding-{original}"),
11405 );
11406 }
11407 let loader = CountingLoader::new();
11408 let runner =
11409 RecordingBatchRunner::new([BatchReply::Raw(pass_json()), BatchReply::Raw(pass_json())]);
11410
11411 drain_once(
11412 &queue,
11413 &loader,
11414 &selection(),
11415 "",
11416 &runner,
11417 &store,
11418 &TruthMirrorConfig::default(),
11419 )
11420 .unwrap();
11421
11422 assert_eq!(loader.calls(), 2);
11423 assert_eq!(runner.prompts().len(), 2);
11424 assert!(
11425 runner
11426 .prompts()
11427 .iter()
11428 .all(|prompt| prompt.contains("PETITION:") && !prompt.contains("BATCH:"))
11429 );
11430 }
11431
11432 #[test]
11433 fn extract_verdict_json_handles_raw_fenced_and_prose_wrapped_output() {
11434 let raw = r#"{"verdict":"PASS"}"#;
11436 assert_eq!(super::extract_verdict_json(raw), raw);
11437
11438 let fenced = "Here is my verdict:\n```json\n{\"verdict\":\"PASS\"}\n```\nthanks";
11440 assert_eq!(
11441 super::extract_verdict_json(fenced),
11442 "{\"verdict\":\"PASS\"}"
11443 );
11444
11445 let bare = "verdict below\n```\n{\"a\":1}\n```";
11447 assert_eq!(super::extract_verdict_json(bare), "{\"a\":1}");
11448
11449 let two = "```\nnot json\n```\nthen\n```json\n{\"b\":2}\n```";
11451 assert_eq!(super::extract_verdict_json(two), "{\"b\":2}");
11452
11453 let prose = "The verdict is {\"a\": {\"b\":2}} as required.";
11455 assert_eq!(super::extract_verdict_json(prose), "{\"a\": {\"b\":2}}");
11456
11457 assert_eq!(super::extract_verdict_json(" nope "), "nope");
11460 }
11461
11462 #[test]
11463 fn parsed_verdict_accepts_markdown_fenced_reviewer_output() {
11464 let fenced = format!(
11468 "The fix looks good. Verdict follows.\n\n```json\n{}\n```\n",
11469 pass_json()
11470 );
11471 let parsed = ParsedVerdict::parse(&fenced).unwrap();
11472 assert_eq!(parsed.verdict, Verdict::Pass);
11473 assert_eq!(parsed.raw, fenced);
11475 }
11476
11477 #[test]
11478 fn parsed_verdict_accepts_fenced_json_prefix_with_trailing_prose() {
11479 let fenced = format!(
11480 "Reviewer explanation.\n\n```json\n{}\nThe JSON above is the complete verdict.\n```\n",
11481 pass_json()
11482 );
11483
11484 let parsed = ParsedVerdict::parse(&fenced).unwrap();
11485
11486 assert_eq!(parsed.verdict, Verdict::Pass);
11487 }
11488
11489 #[test]
11490 fn legacy_queued_review_without_optional_fields_still_parses() {
11491 let legacy = r#"{"commit_sha":"abc","enqueued_at_unix":1}"#;
11492 let item: super::QueuedReview = serde_json::from_str(legacy).unwrap();
11493 assert!(item.run_id.is_empty());
11494 assert_eq!(item.petition_for, None);
11495 }
11496
11497 #[test]
11498 fn enqueue_petition_round_trips_petition_for() {
11499 let temp = tempfile::tempdir().unwrap();
11500 let queue = ReviewQueue::new(temp.path());
11501 queue.enqueue("normal1").unwrap();
11502 queue.enqueue_petition("fix456", "orig123").unwrap();
11503
11504 let pending = queue.pending().unwrap();
11505 assert_eq!(pending.len(), 2);
11506 assert_eq!(pending[0].petition_for, None);
11507 assert_eq!(pending[1].commit_sha, "fix456");
11508 assert_eq!(pending[1].petition_for.as_deref(), Some("orig123"));
11509 }
11510
11511 #[test]
11512 fn drain_once_runs_petition_jobs_and_transitions_the_original_rejection() {
11513 use crate::ledger::{Disposition, LedgerEntry, ResolutionKind, ReviewerConfig};
11514
11515 let temp = tempfile::tempdir().unwrap();
11516 let store = LedgerStore::new(temp.path());
11517 let queue = ReviewQueue::new(temp.path());
11518
11519 let rejected = LedgerEntry::new(
11522 "orig123",
11523 Verdict::Reject,
11524 "CLAIM: original | verified: cargo test | evidence: tests:cargo-test",
11525 vec!["tests:cargo-test".to_owned()],
11526 ReviewerConfig::new("claude", "claude-opus-4-1", false),
11527 vec!["evidence too thin".to_owned()],
11528 );
11529 store.append_entry(&rejected).unwrap();
11530 store
11531 .append_petition_transition(
11532 "orig123",
11533 Disposition::Open,
11534 ResolutionKind::Resolved,
11535 "petition review enqueued: fix=fix456, attempts=1/2",
11536 1,
11537 )
11538 .unwrap();
11539 queue.enqueue_petition("fix456", "orig123").unwrap();
11540
11541 let loader = StaticLoader::new();
11542 let runner = ConstRunner::new(pass_json());
11543 let report = drain_once(
11544 &queue,
11545 &loader,
11546 &selection(),
11547 "",
11548 &runner,
11549 &store,
11550 &TruthMirrorConfig::default(),
11551 )
11552 .unwrap();
11553
11554 assert_eq!(report.reviewed, ["fix456"]);
11555 let review = store.show("fix456").unwrap();
11558 assert_eq!(review.petition_for.as_deref(), Some("orig123"));
11559 assert_eq!(review.petition_attempts, 1);
11560 let original = store.show("orig123").unwrap();
11562 assert_eq!(original.disposition, Disposition::Resolved);
11563 assert!(queue.pending().unwrap().is_empty());
11564 assert!(store.blocking_rejections().unwrap().is_empty());
11565 }
11566
11567 #[test]
11568 fn extract_verdict_json_takes_leading_json_before_trailing_prose() {
11569 let leading = "{\"verdict\":\"PASS\"} \n\nHope this helps!";
11572 assert_eq!(
11573 super::extract_verdict_json(leading),
11574 "{\"verdict\":\"PASS\"}"
11575 );
11576
11577 let both = "verdict: {\"a\":1} trailing prose with a stray } brace";
11580 assert_eq!(super::extract_verdict_json(both), "{\"a\":1}");
11581 }
11582
11583 #[test]
11584 fn extract_verdict_json_skips_non_json_brace_to_find_verdict() {
11585 let output = format!(
11589 "The API call getCapabilities({{preview:true}}) returned nothing useful.\n\n{}",
11590 flag_json("burned petition")
11591 );
11592 let parsed = ParsedVerdict::parse(&output).unwrap();
11593 assert_eq!(parsed.verdict, Verdict::Flag);
11594 assert_eq!(parsed.structured_findings[0].title, "burned petition");
11595 }
11596
11597 #[test]
11598 fn extract_verdict_json_prefers_verdict_object_over_stray_json() {
11599 let output = format!(
11601 "For reference, here's an example object: {{\"example\": 1}}.\n\n{}",
11602 reject_json("real finding")
11603 );
11604 let parsed = ParsedVerdict::parse(&output).unwrap();
11605 assert_eq!(parsed.verdict, Verdict::Reject);
11606 assert_eq!(parsed.structured_findings[0].title, "real finding");
11607 }
11608
11609 #[test]
11610 fn drain_once_reviews_normal_and_petition_items_for_the_same_fix_sha() {
11611 use crate::ledger::{Disposition, LedgerEntry, ResolutionKind, ReviewerConfig};
11612
11613 let temp = tempfile::tempdir().unwrap();
11614 let store = LedgerStore::new(temp.path());
11615 let queue = ReviewQueue::new(temp.path());
11616
11617 let rejected = LedgerEntry::new(
11618 "orig123",
11619 Verdict::Reject,
11620 "CLAIM: original | verified: cargo test | evidence: tests:cargo-test",
11621 vec!["tests:cargo-test".to_owned()],
11622 ReviewerConfig::new("claude", "claude-opus-4-1", false),
11623 vec!["evidence too thin".to_owned()],
11624 );
11625 store.append_entry(&rejected).unwrap();
11626 store
11627 .append_petition_transition(
11628 "orig123",
11629 Disposition::Open,
11630 ResolutionKind::Resolved,
11631 "petition review enqueued: fix=fix456, attempts=1/2",
11632 1,
11633 )
11634 .unwrap();
11635
11636 queue.enqueue("fix456").unwrap();
11640 queue.enqueue_petition("fix456", "orig123").unwrap();
11641
11642 let loader = StaticLoader::new();
11643 let runner = ConstRunner::new(pass_json());
11644 let report = drain_once(
11645 &queue,
11646 &loader,
11647 &selection(),
11648 "",
11649 &runner,
11650 &store,
11651 &TruthMirrorConfig::default(),
11652 )
11653 .unwrap();
11654
11655 assert_eq!(report.reviewed, ["fix456", "fix456"]);
11656 assert!(queue.pending().unwrap().is_empty());
11657 let history = store.read_history().unwrap();
11660 let fix_entries: Vec<_> = history
11661 .iter()
11662 .filter(|entry| entry.commit_sha == "fix456")
11663 .collect();
11664 assert_eq!(fix_entries.len(), 2);
11665 assert!(fix_entries.iter().any(|entry| entry.petition_for.is_none()));
11666 assert!(
11667 fix_entries
11668 .iter()
11669 .any(|entry| entry.petition_for.as_deref() == Some("orig123"))
11670 );
11671 assert_eq!(
11673 store.show("orig123").unwrap().disposition,
11674 crate::ledger::Disposition::Resolved
11675 );
11676 }
11677
11678 #[test]
11679 fn pending_does_not_duplicate_legacy_queue_row_for_orphan_run() {
11680 let temp = tempfile::tempdir().unwrap();
11681 let queue = ReviewQueue::new(temp.path());
11682 let sha = "abc1234567890123456789012345678901234567890";
11683 fs::write(
11684 queue.path(),
11685 format!(
11686 "{}\n",
11687 serde_json::to_string(&QueuedReview {
11688 run_id: String::new(),
11689 commit_sha: sha.to_owned(),
11690 enqueued_at_unix: 1,
11691 petition_for: None,
11692 ..Default::default()
11693 })
11694 .unwrap()
11695 ),
11696 )
11697 .unwrap();
11698 fs::create_dir_all(temp.path().join("runs")).unwrap();
11699 let run = ReviewRun::queued("run-1", sha, "commit");
11700 fs::write(
11701 temp.path().join("runs/run-1.json"),
11702 serde_json::to_string(&run).unwrap(),
11703 )
11704 .unwrap();
11705
11706 let pending = queue.pending().unwrap();
11707 assert_eq!(pending.len(), 1);
11708 assert_eq!(pending[0].commit_sha, sha);
11709 assert_eq!(pending[0].run_id, "run-1");
11710 }
11711
11712 #[test]
11713 fn enqueue_inflight_recovery_republishes_queue_after_crash() {
11714 let temp = tempfile::tempdir().unwrap();
11715 let queue = ReviewQueue::new(temp.path());
11716 let sha = "abc1234567890123456789012345678901234567890";
11717 let run_store = ReviewRunStore::new(temp.path());
11718 let run = run_store
11719 .ensure_queued("run-inflight", sha, "commit", None)
11720 .unwrap();
11721 let item = QueuedReview {
11722 run_id: run.id,
11723 commit_sha: sha.to_owned(),
11724 enqueued_at_unix: 1,
11725 petition_for: None,
11726 ..Default::default()
11727 };
11728 write_enqueue_inflight(temp.path(), &item).unwrap();
11729
11730 let pending = queue.pending().unwrap();
11731 assert_eq!(pending.len(), 1);
11732 assert_eq!(pending[0].run_id, "run-inflight");
11733 assert!(!temp.path().join(ENQUEUE_INFLIGHT_FILE).exists());
11734 }
11735
11736 #[test]
11737 fn drain_once_replay_applies_pending_spool_without_rerunning_provider() {
11738 use crate::ledger::{LedgerEntry, ReviewerConfig, Verdict};
11739
11740 let temp = tempfile::tempdir().unwrap();
11741 let store = LedgerStore::new(temp.path());
11742 let queue = ReviewQueue::new(temp.path());
11743 let sha = "abc123";
11744 let run_store = ReviewRunStore::new(temp.path());
11745 let queued = queue.enqueue(sha).unwrap();
11746 run_store
11747 .mark_running(&queued.run_id, REVIEW_LEDGER_PHASE)
11748 .unwrap();
11749 let entry = LedgerEntry::new(
11750 sha,
11751 Verdict::Pass,
11752 "CLAIM: ok | verified: cargo test | evidence: tests:cargo-test",
11753 vec!["tests:cargo-test".to_owned()],
11754 ReviewerConfig::new("claude", "claude-opus-4-1", false),
11755 vec![],
11756 );
11757 run_store
11758 .persist_pending_review(
11759 &queued.run_id,
11760 PendingReviewSpool {
11761 entries: vec![entry],
11762 petition_transition_pending: false,
11763 strict_pass_stdout: None,
11764 },
11765 )
11766 .unwrap();
11767
11768 let report = drain_once(
11769 &queue,
11770 &StaticLoader::new(),
11771 &selection(),
11772 "",
11773 &PanicRunner,
11774 &store,
11775 &TruthMirrorConfig::default(),
11776 )
11777 .unwrap();
11778
11779 assert_eq!(report.reviewed, [sha]);
11780 assert_eq!(report.ledger_entries, 1);
11781 assert_eq!(store.read_history().unwrap().len(), 1);
11782 let completed = run_store.read(&queued.run_id).unwrap();
11783 assert_eq!(completed.status, ReviewRunStatus::Completed);
11784 assert!(completed.pending_review.is_none());
11785 assert_eq!(
11786 completed.ledger_applied_phase.as_deref(),
11787 Some(REVIEW_LEDGER_PHASE)
11788 );
11789 assert!(queue.pending().unwrap().is_empty());
11790 }
11791
11792 #[test]
11793 fn drain_once_replay_applies_pending_petition_transition_without_rerunning_provider() {
11794 use crate::ledger::{Disposition, LedgerEntry, ReviewerConfig, Verdict};
11795
11796 let temp = tempfile::tempdir().unwrap();
11797 let store = LedgerStore::new(temp.path());
11798 let queue = ReviewQueue::new(temp.path());
11799 let run_store = ReviewRunStore::new(temp.path());
11800
11801 let rejected = LedgerEntry::new(
11802 "orig123",
11803 Verdict::Reject,
11804 "CLAIM: original | verified: cargo test | evidence: tests:cargo-test",
11805 vec!["tests:cargo-test".to_owned()],
11806 ReviewerConfig::new("claude", "claude-opus-4-1", false),
11807 vec!["evidence too thin".to_owned()],
11808 );
11809 store.append_entry(&rejected).unwrap();
11810 store
11811 .append_petition_transition(
11812 "orig123",
11813 Disposition::Open,
11814 crate::ledger::ResolutionKind::Resolved,
11815 "petition review enqueued: fix=fix456, attempts=1/2",
11816 1,
11817 )
11818 .unwrap();
11819 let queued = queue.enqueue_petition("fix456", "orig123").unwrap();
11820 run_store
11821 .mark_running(&queued.run_id, REVIEW_LEDGER_PHASE)
11822 .unwrap();
11823 let mut review = LedgerEntry::new(
11824 "fix456",
11825 Verdict::Pass,
11826 "CLAIM: ok | verified: cargo test | evidence: tests:cargo-test",
11827 vec!["tests:cargo-test".to_owned()],
11828 ReviewerConfig::new("claude", "claude-opus-4-1", false),
11829 vec![],
11830 );
11831 review.petition_for = Some("orig123".to_owned());
11832 review.petition_attempts = 1;
11833 run_store
11834 .persist_pending_review(
11835 &queued.run_id,
11836 PendingReviewSpool {
11837 entries: vec![review],
11838 petition_transition_pending: true,
11839 strict_pass_stdout: None,
11840 },
11841 )
11842 .unwrap();
11843
11844 let report = drain_once(
11845 &queue,
11846 &StaticLoader::new(),
11847 &selection(),
11848 "",
11849 &PanicRunner,
11850 &store,
11851 &TruthMirrorConfig::default(),
11852 )
11853 .unwrap();
11854
11855 assert_eq!(report.reviewed, ["fix456"]);
11856 let fix = store.show("fix456").unwrap();
11857 assert_eq!(fix.petition_for.as_deref(), Some("orig123"));
11858 assert_eq!(fix.petition_attempts, 1);
11859 let original = store.show("orig123").unwrap();
11860 assert_eq!(original.disposition, Disposition::Resolved);
11861 let completed = run_store.read(&queued.run_id).unwrap();
11862 assert_eq!(completed.status, ReviewRunStatus::Completed);
11863 assert!(completed.pending_review.is_none());
11864 assert!(queue.pending().unwrap().is_empty());
11865 }
11866
11867 #[test]
11868 fn drain_once_reject_petition_appends_review_transition_after_enqueue_and_replay_is_idempotent()
11869 {
11870 use crate::ledger::{
11871 Disposition, LedgerEntry, ReviewerConfig, TransitionProvenance,
11872 TransitionProvenanceKind, Verdict,
11873 };
11874
11875 let temp = tempfile::tempdir().unwrap();
11876 let store = LedgerStore::new(temp.path());
11877 let queue = ReviewQueue::new(temp.path());
11878 let run_store = ReviewRunStore::new(temp.path());
11879
11880 let rejected = LedgerEntry::new(
11881 "orig123",
11882 Verdict::Reject,
11883 "CLAIM: original | verified: cargo test | evidence: tests:cargo-test",
11884 vec!["tests:cargo-test".to_owned()],
11885 ReviewerConfig::new("claude", "claude-opus-4-1", false),
11886 vec!["evidence too thin".to_owned()],
11887 );
11888 store.append_entry(&rejected).unwrap();
11889 store
11890 .append_petition_transition_with_provenance(
11891 "orig123",
11892 Disposition::Open,
11893 crate::ledger::ResolutionKind::Resolved,
11894 "petition review enqueued: fix=fix456, attempts=1/2",
11895 1,
11896 Some(TransitionProvenance {
11897 kind: TransitionProvenanceKind::PetitionEnqueued,
11898 fix_sha: "fix456".to_owned(),
11899 original_sha: "orig123".to_owned(),
11900 attempts: 1,
11901 }),
11902 )
11903 .unwrap();
11904 let queued = queue.enqueue_petition("fix456", "orig123").unwrap();
11905 run_store
11906 .mark_running(&queued.run_id, REVIEW_LEDGER_PHASE)
11907 .unwrap();
11908 let mut review = LedgerEntry::new(
11909 "fix456",
11910 Verdict::Reject,
11911 "CLAIM: still bad | verified: cargo test | evidence: tests:cargo-test",
11912 vec!["tests:cargo-test".to_owned()],
11913 ReviewerConfig::new("claude", "claude-opus-4-1", false),
11914 vec!["fix insufficient".to_owned()],
11915 );
11916 review.petition_for = Some("orig123".to_owned());
11917 review.petition_attempts = 1;
11918 run_store
11919 .persist_pending_review(
11920 &queued.run_id,
11921 PendingReviewSpool {
11922 entries: vec![review.clone()],
11923 petition_transition_pending: true,
11924 strict_pass_stdout: None,
11925 },
11926 )
11927 .unwrap();
11928
11929 let report = drain_once(
11930 &queue,
11931 &StaticLoader::new(),
11932 &selection(),
11933 "",
11934 &PanicRunner,
11935 &store,
11936 &TruthMirrorConfig::default(),
11937 )
11938 .unwrap();
11939
11940 assert_eq!(report.reviewed, ["fix456"]);
11941 let orig_history: Vec<_> = store
11942 .read_history()
11943 .unwrap()
11944 .into_iter()
11945 .filter(|entry| entry.commit_sha == "orig123")
11946 .collect();
11947 assert_eq!(orig_history.len(), 3);
11948 let original = store.show("orig123").unwrap();
11949 assert_eq!(original.disposition, Disposition::Open);
11950 assert_eq!(original.petition_attempts, 1);
11951 let provenance = original.transition_provenance.as_ref().unwrap();
11952 assert_eq!(provenance.kind, TransitionProvenanceKind::PetitionReview);
11953 assert_eq!(provenance.fix_sha, "fix456");
11954 assert_eq!(provenance.original_sha, "orig123");
11955 assert_eq!(provenance.attempts, 1);
11956
11957 let queued_replay = queue.enqueue_petition("fix456", "orig123").unwrap();
11958 run_store
11959 .mark_running(&queued_replay.run_id, REVIEW_LEDGER_PHASE)
11960 .unwrap();
11961 run_store
11962 .persist_pending_review(
11963 &queued_replay.run_id,
11964 PendingReviewSpool {
11965 entries: vec![review],
11966 petition_transition_pending: true,
11967 strict_pass_stdout: None,
11968 },
11969 )
11970 .unwrap();
11971
11972 let replay = drain_once(
11973 &queue,
11974 &StaticLoader::new(),
11975 &selection(),
11976 "",
11977 &PanicRunner,
11978 &store,
11979 &TruthMirrorConfig::default(),
11980 )
11981 .unwrap();
11982 assert_eq!(replay.reviewed, ["fix456"]);
11983 let orig_history_after: Vec<_> = store
11984 .read_history()
11985 .unwrap()
11986 .into_iter()
11987 .filter(|entry| entry.commit_sha == "orig123")
11988 .collect();
11989 assert_eq!(orig_history_after.len(), 3);
11990 let original_after = store.show("orig123").unwrap();
11991 assert_eq!(
11992 original_after.transition_provenance,
11993 original.transition_provenance
11994 );
11995 }
11996
11997 #[test]
11998 fn drain_once_replay_applies_pending_strict_second_pass_without_rerunning_provider() {
11999 use crate::ledger::{LedgerEntry, ReviewerConfig, Verdict};
12000
12001 let temp = tempfile::tempdir().unwrap();
12002 let store = LedgerStore::new(temp.path());
12003 let queue = ReviewQueue::new(temp.path());
12004 let sha = "abc123";
12005 let run_store = ReviewRunStore::new(temp.path());
12006 let queued = queue.enqueue(sha).unwrap();
12007 run_store
12008 .mark_running(&queued.run_id, REVIEW_LEDGER_PHASE)
12009 .unwrap();
12010 let first = LedgerEntry::new(
12011 sha,
12012 Verdict::Pass,
12013 "CLAIM: first pass | verified: cargo test | evidence: tests:cargo-test",
12014 vec!["tests:cargo-test".to_owned()],
12015 ReviewerConfig::new("claude", "claude-opus-4-1", false),
12016 vec![],
12017 );
12018 let second = LedgerEntry::new(
12019 sha,
12020 Verdict::Pass,
12021 "CLAIM: strict second pass | verified: cargo test | evidence: tests:cargo-test",
12022 vec!["tests:cargo-test".to_owned()],
12023 ReviewerConfig::new("claude", "claude-opus-4-2", false),
12024 vec![],
12025 );
12026 run_store
12027 .persist_pending_review(
12028 &queued.run_id,
12029 PendingReviewSpool {
12030 entries: vec![first, second],
12031 petition_transition_pending: false,
12032 strict_pass_stdout: None,
12033 },
12034 )
12035 .unwrap();
12036
12037 let report = drain_once(
12038 &queue,
12039 &StaticLoader::new(),
12040 &selection(),
12041 "",
12042 &PanicRunner,
12043 &store,
12044 &TruthMirrorConfig::default(),
12045 )
12046 .unwrap();
12047
12048 assert_eq!(report.reviewed, [sha]);
12049 assert_eq!(report.ledger_entries, 2);
12050 let history = store.read_history().unwrap();
12051 assert_eq!(history.len(), 2);
12052 assert!(history[0].claim.contains("first pass"));
12053 assert!(history[1].claim.contains("strict second pass"));
12054 let completed = run_store.read(&queued.run_id).unwrap();
12055 assert_eq!(completed.status, ReviewRunStatus::Completed);
12056 assert!(completed.pending_review.is_none());
12057 assert!(queue.pending().unwrap().is_empty());
12058 }
12059
12060 #[test]
12061 fn drain_once_replay_skips_model_when_ledger_phase_is_durable() {
12062 use crate::ledger::{LedgerEntry, ReviewerConfig, Verdict};
12063
12064 let temp = tempfile::tempdir().unwrap();
12065 let store = LedgerStore::new(temp.path());
12066 let queue = ReviewQueue::new(temp.path());
12067 let sha = "abc1234567890123456789012345678901234567890";
12068 let run_store = ReviewRunStore::new(temp.path());
12069 run_store
12070 .ensure_queued("run-replay", sha, "commit", None)
12071 .unwrap();
12072 run_store
12073 .mark_ledger_applied("run-replay", REVIEW_LEDGER_PHASE, 1)
12074 .unwrap();
12075 store
12076 .append_entry(&LedgerEntry::new(
12077 sha,
12078 Verdict::Pass,
12079 "CLAIM: ok | verified: cargo test | evidence: tests:cargo-test",
12080 vec!["tests:cargo-test".to_owned()],
12081 ReviewerConfig::new("claude", "claude-opus-4-1", false),
12082 vec![],
12083 ))
12084 .unwrap();
12085 fs::write(
12086 queue.path(),
12087 format!(
12088 "{}\n",
12089 serde_json::to_string(&QueuedReview {
12090 run_id: "run-replay".to_owned(),
12091 commit_sha: sha.to_owned(),
12092 enqueued_at_unix: 1,
12093 petition_for: None,
12094 ..Default::default()
12095 })
12096 .unwrap()
12097 ),
12098 )
12099 .unwrap();
12100
12101 let loader = StaticLoader::new();
12102 let runner = ConstRunner::new(pass_json());
12103 let before = store.read_history().unwrap().len();
12104 let report = drain_once(
12105 &queue,
12106 &loader,
12107 &selection(),
12108 "",
12109 &runner,
12110 &store,
12111 &TruthMirrorConfig::default(),
12112 )
12113 .unwrap();
12114
12115 assert_eq!(report.reviewed, [sha]);
12116 assert_eq!(store.read_history().unwrap().len(), before);
12117 assert!(queue.pending().unwrap().is_empty());
12118 }
12119
12120 #[test]
12121 fn drain_once_replay_skips_model_when_running_and_verdict_is_durable() {
12122 let temp = tempfile::tempdir().unwrap();
12123 let store = LedgerStore::new(temp.path());
12124 let queue = ReviewQueue::new(temp.path());
12125 let sha = "abc1234567890123456789012345678901234567890";
12126 let run_store = ReviewRunStore::new(temp.path());
12127 let _run = run_store
12128 .ensure_queued("run-running", sha, "commit", None)
12129 .unwrap();
12130 run_store
12131 .mark_running("run-running", REVIEW_LEDGER_PHASE)
12132 .unwrap();
12133 run_store
12134 .mark_ledger_applied("run-running", REVIEW_LEDGER_PHASE, 1)
12135 .unwrap();
12136 fs::write(
12137 queue.path(),
12138 format!(
12139 "{}
12140",
12141 serde_json::to_string(&QueuedReview {
12142 run_id: "run-running".to_owned(),
12143 commit_sha: sha.to_owned(),
12144 enqueued_at_unix: 1,
12145 petition_for: None,
12146 ..Default::default()
12147 })
12148 .unwrap()
12149 ),
12150 )
12151 .unwrap();
12152
12153 let loader = StaticLoader::new();
12154 let runner = ConstRunner::new(pass_json());
12155 let before = store.read_history().unwrap().len();
12156 let report = drain_once(
12157 &queue,
12158 &loader,
12159 &selection(),
12160 "",
12161 &runner,
12162 &store,
12163 &TruthMirrorConfig::default(),
12164 )
12165 .unwrap();
12166
12167 assert_eq!(report.reviewed, [sha]);
12168 assert_eq!(store.read_history().unwrap().len(), before);
12169 assert!(queue.pending().unwrap().is_empty());
12170 assert!(run_ledger_phase_applied(
12171 &run_store.read("run-running").unwrap(),
12172 REVIEW_LEDGER_PHASE
12173 ));
12174 }
12175
12176 #[test]
12177 fn flag_petition_verdict_does_not_resolve_the_original_rejection() {
12178 use crate::ledger::{Disposition, LedgerEntry, ResolutionKind, ReviewerConfig};
12179
12180 let temp = tempfile::tempdir().unwrap();
12181 let store = LedgerStore::new(temp.path());
12182 let queue = ReviewQueue::new(temp.path());
12183 let rejected = LedgerEntry::new(
12184 "orig123",
12185 Verdict::Reject,
12186 "CLAIM: original | verified: cargo test | evidence: tests:cargo-test",
12187 vec!["tests:cargo-test".to_owned()],
12188 ReviewerConfig::new("claude", "claude-opus-4-1", false),
12189 vec!["evidence too thin".to_owned()],
12190 );
12191 store.append_entry(&rejected).unwrap();
12192 store
12193 .append_petition_transition(
12194 "orig123",
12195 Disposition::Open,
12196 ResolutionKind::Resolved,
12197 "petition review enqueued: fix=fix456, attempts=1/2",
12198 1,
12199 )
12200 .unwrap();
12201 queue.enqueue_petition("fix456", "orig123").unwrap();
12202
12203 let loader = StaticLoader::new();
12205 let runner = ConstRunner::new(flag_json("evidence could be tighter"));
12206 drain_once(
12207 &queue,
12208 &loader,
12209 &selection(),
12210 "",
12211 &runner,
12212 &store,
12213 &TruthMirrorConfig::default(),
12214 )
12215 .unwrap();
12216
12217 assert_eq!(
12218 store.show("orig123").unwrap().disposition,
12219 crate::ledger::Disposition::Open
12220 );
12221 let history = store.read_history().unwrap();
12222 let petition_entry = history
12223 .iter()
12224 .find(|entry| {
12225 entry.petition_for.as_deref() == Some("orig123") && entry.commit_sha == "fix456"
12226 })
12227 .unwrap();
12228 assert_eq!(petition_entry.verdict, Verdict::Flag);
12229 }
12230
12231 #[test]
12232 fn reconcile_stale_runs_reaps_dead_worker_rows_and_counts_live_view() {
12233 let temp = tempfile::tempdir().unwrap();
12236 let store = ReviewRunStore::new(temp.path());
12237
12238 let mut orphan = ReviewRun::queued("run-orphan", "abc123", "commit");
12241 orphan.status = ReviewRunStatus::Running;
12242 orphan.worker_pid = Some(reaped_pid());
12243 store.write(&orphan).unwrap();
12244
12245 let mut live = ReviewRun::queued("run-live", "def456", "commit");
12247 live.mark_running("reviewing");
12248 store.write(&live).unwrap();
12249
12250 assert_eq!(store.stale_running_count().unwrap(), 1);
12252 assert_eq!(
12253 store.read("run-orphan").unwrap().status,
12254 ReviewRunStatus::Running
12255 );
12256
12257 assert_eq!(store.reconcile_stale_runs().unwrap(), 1);
12259 let orphan = store.read("run-orphan").unwrap();
12260 assert_eq!(orphan.status, ReviewRunStatus::Failed);
12261 assert!(
12262 orphan
12263 .error
12264 .as_deref()
12265 .is_some_and(|error| error.contains("stale run")),
12266 "stale reason should be recorded, got: {:?}",
12267 orphan.error
12268 );
12269 assert_eq!(
12270 store.read("run-live").unwrap().status,
12271 ReviewRunStatus::Running
12272 );
12273 assert_eq!(store.stale_running_count().unwrap(), 0);
12274 }
12275
12276 #[test]
12277 fn drain_once_skips_petition_whose_target_is_already_resolved() {
12278 use crate::ledger::{Disposition, LedgerEntry, ResolutionKind, ReviewerConfig};
12284
12285 let temp = tempfile::tempdir().unwrap();
12286 let store = LedgerStore::new(temp.path());
12287 let queue = ReviewQueue::new(temp.path());
12288
12289 let rejected = LedgerEntry::new(
12290 "orig123",
12291 Verdict::Reject,
12292 "CLAIM: original | verified: cargo test | evidence: tests:cargo-test",
12293 vec!["tests:cargo-test".to_owned()],
12294 ReviewerConfig::new("claude", "claude-opus-4-1", false),
12295 vec!["evidence too thin".to_owned()],
12296 );
12297 store.append_entry(&rejected).unwrap();
12298 store
12300 .append_petition_transition(
12301 "orig123",
12302 Disposition::Resolved,
12303 ResolutionKind::Resolved,
12304 "already fixed elsewhere",
12305 1,
12306 )
12307 .unwrap();
12308
12309 queue.enqueue_petition("fix456", "orig123").unwrap();
12310 queue.enqueue("later999").unwrap();
12311
12312 let loader = StaticLoader::new();
12313 let runner = ConstRunner::new(pass_json());
12314 let report = drain_once(
12315 &queue,
12316 &loader,
12317 &selection(),
12318 "",
12319 &runner,
12320 &store,
12321 &TruthMirrorConfig::default(),
12322 )
12323 .unwrap();
12324
12325 assert_eq!(report.failed, ["fix456"]);
12328 assert_eq!(report.reviewed, ["later999"]);
12329 assert!(queue.pending().unwrap().is_empty());
12330
12331 let run_store = ReviewRunStore::new(temp.path());
12333 let runs = run_store.list().unwrap();
12334 let skipped = runs.iter().find(|run| run.commit_sha == "fix456").unwrap();
12335 assert_eq!(skipped.status, ReviewRunStatus::Failed);
12336 assert!(
12337 skipped
12338 .error
12339 .as_deref()
12340 .is_some_and(|error| error.contains("no open rejection")),
12341 "skip reason should be recorded, got: {:?}",
12342 skipped.error
12343 );
12344
12345 assert!(
12347 store
12348 .read_history()
12349 .unwrap()
12350 .iter()
12351 .all(|entry| entry.commit_sha != "fix456")
12352 );
12353 }
12354
12355 #[test]
12356 fn drain_once_survives_petition_target_resolved_mid_review() {
12357 use crate::ledger::{Disposition, LedgerEntry, ResolutionKind, ReviewerConfig};
12363
12364 struct ResolvingRunner {
12365 state_dir: PathBuf,
12366 }
12367
12368 impl ProcessRunner for ResolvingRunner {
12369 fn run(
12370 &self,
12371 _invocation: &InvocationPlan,
12372 _prompt: &str,
12373 _timeout: Option<Duration>,
12374 ) -> Result<ProcessOutput, ReviewerError> {
12375 LedgerStore::new(&self.state_dir)
12378 .append_petition_transition(
12379 "orig123",
12380 Disposition::Resolved,
12381 ResolutionKind::Resolved,
12382 "raced resolve while the petition review ran",
12383 1,
12384 )
12385 .unwrap();
12386 Ok(ProcessOutput {
12387 status_code: Some(0),
12388 stdout: pass_json(),
12389 stderr: String::new(),
12390 })
12391 }
12392 }
12393
12394 let temp = tempfile::tempdir().unwrap();
12395 let store = LedgerStore::new(temp.path());
12396 let queue = ReviewQueue::new(temp.path());
12397
12398 let rejected = LedgerEntry::new(
12399 "orig123",
12400 Verdict::Reject,
12401 "CLAIM: original | verified: cargo test | evidence: tests:cargo-test",
12402 vec!["tests:cargo-test".to_owned()],
12403 ReviewerConfig::new("claude", "claude-opus-4-1", false),
12404 vec!["evidence too thin".to_owned()],
12405 );
12406 store.append_entry(&rejected).unwrap();
12407 store
12408 .append_petition_transition(
12409 "orig123",
12410 Disposition::Open,
12411 ResolutionKind::Resolved,
12412 "petition review enqueued: fix=fix456, attempts=1/2",
12413 1,
12414 )
12415 .unwrap();
12416 queue.enqueue_petition("fix456", "orig123").unwrap();
12417
12418 let runner = ResolvingRunner {
12419 state_dir: temp.path().to_path_buf(),
12420 };
12421 let report = drain_once(
12422 &queue,
12423 &StaticLoader::new(),
12424 &selection(),
12425 "",
12426 &runner,
12427 &store,
12428 &TruthMirrorConfig::default(),
12429 )
12430 .unwrap();
12431
12432 assert_eq!(report.reviewed, ["fix456"]);
12436 assert!(report.failed.is_empty());
12437 assert!(queue.pending().unwrap().is_empty());
12438 assert_eq!(
12439 store.show("orig123").unwrap().disposition,
12440 Disposition::Resolved
12441 );
12442 }
12443}