//! Reviewer process harness, model opposition, async queue, and verdict execution.
use std::{
fs,
io::{self, Read, Write},
path::{Path, PathBuf},
process::{Command, ExitCode, Stdio},
sync::{
Arc,
atomic::{AtomicBool, AtomicU64, Ordering},
},
time::{Duration, Instant, SystemTime, UNIX_EPOCH},
};
use anyhow::Result;
use serde::{Deserialize, Serialize};
use thiserror::Error;
use crate::{
claim::{Claim, EvidenceRef},
cli::{self, Agent, ReviewScope, ReviewerHarness},
config::{self, Effort},
ledger::{
Disposition, LedgerEntry, LedgerError, LedgerStore, MemorySkillClassification,
MemorySkillClassificationKind, ResolutionKind, ReviewerConfig, StructuredFinding,
TransitionProvenance, TransitionProvenanceKind, Verdict,
},
provenance::{self, EntireCheckpointRef},
state::{StateTransaction, atomic_replace},
surface,
time::unix_now,
};
pub const REVIEW_QUEUE_FILE: &str = "review-queue.jsonl";
pub const REVIEW_RUNS_DIR: &str = "runs";
const BATCHES_DIR: &str = "batches";
const ENQUEUE_INFLIGHT_FILE: &str = "enqueue-inflight.json";
/// Scheduler phase after durable claim and before reviewer execution starts.
const CLAIMED_PHASE: &str = "claimed";
/// Run phase while waiting out a provider/rate-limit backoff window.
const PROVIDER_BACKOFF_PHASE: &str = "provider_backoff";
/// Default delay before a retryable provider failure may be attempted again.
const DEFAULT_PROVIDER_BACKOFF_SECS: u64 = 60;
const PETITION_FAILURES_FILE: &str = "petition-failures.jsonl";
const REVIEW_LEDGER_PHASE: &str = "reviewing";
const MAX_INLINE_DIFF_FILES: usize = 2;
const MAX_INLINE_DIFF_BYTES: usize = 256 * 1024;
const MAX_UNTRACKED_FILE_BYTES: u64 = 16 * 1024;
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ReviewRequest {
pub watched_agent: Agent,
pub watched_model: String,
pub reviewer_harness: ReviewerHarness,
pub reviewer_model: String,
pub reviewer_effort: Effort,
pub allow_same_model: bool,
pub prompt: String,
}
impl ReviewRequest {
pub fn new(
watched_agent: Agent,
watched_model: impl Into<String>,
reviewer_harness: ReviewerHarness,
reviewer_model: impl Into<String>,
allow_same_model: bool,
prompt: impl Into<String>,
) -> Self {
Self {
watched_agent,
watched_model: watched_model.into(),
reviewer_harness,
reviewer_model: reviewer_model.into(),
reviewer_effort: Effort::highest(),
allow_same_model,
prompt: prompt.into(),
}
}
pub fn with_effort(mut self, effort: Effort) -> Self {
self.reviewer_effort = effort;
self
}
}
/// Resolved reviewer selection shared by `review` and `watch`. The reviewer is
/// chosen from the writer harness's adversarial pair; explicit CLI values win.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ReviewSelection {
pub watched_agent: Agent,
pub watched_model: String,
pub reviewer_harness: ReviewerHarness,
pub reviewer_model: String,
pub reviewer_effort: Effort,
pub allow_same_model: bool,
pub strict: Option<StrictReviewConfig>,
}
impl ReviewSelection {
/// Resolve the reviewer for a writer harness from its adversarial pair,
/// applying any explicit CLI overrides. `strict` is set separately.
#[allow(clippy::too_many_arguments)]
pub fn resolve(
watched_agent: Option<Agent>,
watched_model: Option<String>,
reviewer_harness: Option<ReviewerHarness>,
reviewer_model: Option<String>,
reviewer_effort: Option<Effort>,
allow_same_model: bool,
config: &config::TruthMirrorConfig,
) -> Result<Self, ReviewerError> {
let watched_agent = match watched_agent {
Some(agent) => agent,
None => agent_from_slug(&config.default_writer)?,
};
let writer_slug = surface::agent_slug(watched_agent);
let pair = config.pair_for(writer_slug);
let harness_from_cli = reviewer_harness.is_some();
let reviewer_harness = match reviewer_harness {
Some(harness) => harness,
None => {
let slug = pair
.map(|pair| pair.reviewer.harness.as_str())
.ok_or_else(|| ReviewerError::NoPairForWriter {
writer: writer_slug.to_owned(),
})?;
harness_from_slug(slug)?
}
};
let reviewer_model = match reviewer_model {
Some(model) => model,
None => {
let pair = pair.ok_or_else(|| ReviewerError::NoPairForWriter {
writer: writer_slug.to_owned(),
})?;
// Don't pair a CLI-overridden harness with a model string meant for
// a different harness — that silently reviews with the wrong tool.
if harness_from_cli
&& !pair
.reviewer
.harness
.eq_ignore_ascii_case(harness_slug(reviewer_harness))
{
return Err(ReviewerError::OverrideNeedsModel {
role: "reviewer".to_owned(),
harness: harness_slug(reviewer_harness).to_owned(),
});
}
pair.reviewer.model.clone()
}
};
let reviewer_effort = reviewer_effort
.or_else(|| pair.map(|pair| pair.reviewer.effort))
.unwrap_or_else(Effort::highest);
Ok(Self {
watched_agent,
watched_model: watched_model.unwrap_or_default(),
reviewer_harness,
reviewer_model,
reviewer_effort,
// Either the CLI flag or config may waive model opposition.
allow_same_model: allow_same_model || config.allow_same_model,
strict: None,
})
}
/// Resolve the second-pass arbiter for the writer, preferring CLI overrides,
/// then the writer pair's `arbiter`.
pub fn resolve_arbiter(
watched_agent: Agent,
arbiter_harness: Option<ReviewerHarness>,
arbiter_model: Option<String>,
arbiter_effort: Option<Effort>,
config: &config::TruthMirrorConfig,
) -> Result<StrictReviewConfig, ReviewerError> {
let pair_arbiter = config
.pair_for(surface::agent_slug(watched_agent))
.and_then(|pair| pair.arbiter.clone());
let harness_from_cli = arbiter_harness.is_some();
let harness = match arbiter_harness {
Some(harness) => harness,
None => {
let slug = pair_arbiter
.as_ref()
.map(|arbiter| arbiter.harness.as_str())
.ok_or(ReviewerError::MissingArbiter)?;
harness_from_slug(slug)?
}
};
let model = match arbiter_model {
Some(model) => model,
None => {
let arbiter = pair_arbiter.as_ref().ok_or(ReviewerError::MissingArbiter)?;
if harness_from_cli && !arbiter.harness.eq_ignore_ascii_case(harness_slug(harness))
{
return Err(ReviewerError::OverrideNeedsModel {
role: "arbiter".to_owned(),
harness: harness_slug(harness).to_owned(),
});
}
arbiter.model.clone()
}
};
let effort = arbiter_effort
.or_else(|| pair_arbiter.as_ref().map(|arbiter| arbiter.effort))
.unwrap_or_else(Effort::highest);
Ok(StrictReviewConfig {
arbiter_harness: harness,
arbiter_model: model,
arbiter_effort: effort,
})
}
fn request_for(&self, prompt: String) -> ReviewRequest {
ReviewRequest::new(
self.watched_agent,
self.watched_model.clone(),
self.reviewer_harness,
self.reviewer_model.clone(),
self.allow_same_model,
prompt,
)
.with_effort(self.reviewer_effort)
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ReviewPlan {
pub watched_agent: Agent,
pub watched_model: String,
pub reviewer_harness: ReviewerHarness,
pub reviewer_model: String,
pub allow_same_model: bool,
pub invocation: InvocationPlan,
}
impl ReviewPlan {
pub fn build(request: ReviewRequest) -> Result<Self, ReviewerError> {
validate_model_present("reviewer", &request.reviewer_model)?;
// The writer model may be unknown (pair-based selection doesn't require it);
// opposition is enforced only when the writer model is actually provided.
if request.watched_model.trim().is_empty() {
if !request.allow_same_model {
// The writer model is unknown so we cannot verify model opposition at
// runtime — the review will proceed, but warn so this doesn't go silent.
eprintln!(
"{} warning: writer model is unknown; model opposition is only pair-based \
for this review. Pass --watched-model to enforce per-model opposition; \
truth-mirror has no persistent writer-model config key yet.",
crate::messages::diagnostic_prefix()
);
}
} else if !request.allow_same_model
&& normalized_model(&request.watched_model) == normalized_model(&request.reviewer_model)
{
return Err(ReviewerError::SameModelWithoutWaiver {
watched_model: request.watched_model,
reviewer_model: request.reviewer_model,
});
}
let invocation = InvocationPlan::for_harness(
request.reviewer_harness,
&request.reviewer_model,
request.reviewer_effort,
)?;
Ok(Self {
watched_agent: request.watched_agent,
watched_model: request.watched_model,
reviewer_harness: request.reviewer_harness,
reviewer_model: request.reviewer_model,
allow_same_model: request.allow_same_model,
invocation,
})
}
pub fn run_with<R: ProcessRunner>(
&self,
prompt: &str,
runner: &R,
timeout: Option<Duration>,
) -> Result<ProcessOutput, ReviewerError> {
runner.run(&self.invocation, prompt, timeout)
}
pub fn run_with_dir<R: ProcessRunner>(
&self,
prompt: &str,
runner: &R,
timeout: Option<Duration>,
current_dir: Option<&Path>,
) -> Result<ProcessOutput, ReviewerError> {
runner.run_in_dir(&self.invocation, prompt, timeout, current_dir)
}
fn reviewer_config(&self) -> ReviewerConfig {
ReviewerConfig::new(
harness_slug(self.reviewer_harness),
self.reviewer_model.clone(),
self.allow_same_model,
)
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct InvocationPlan {
pub program: String,
pub args: Vec<String>,
pub prompt_delivery: PromptDelivery,
}
impl InvocationPlan {
pub fn for_harness(
harness: ReviewerHarness,
model: &str,
effort: Effort,
) -> Result<Self, ReviewerError> {
validate_model_present("reviewer", model)?;
let model = model.trim();
let e = effort.as_str();
// Reasoning-effort flags verified against source: codex ReasoningEffort
// (`-c model_reasoning_effort`), pi `--thinking` (cli/args.js:249), claude
// `--effort`. Gemini/OpenCode have no effort flag, so effort is omitted.
let plan = match harness {
ReviewerHarness::Claude => Self {
program: "claude".to_owned(),
args: vec![
"--print".to_owned(),
"--model".to_owned(),
model.to_owned(),
"--effort".to_owned(),
// Claude has no `minimal`; clamp to a valid level.
effort.claude_value().to_owned(),
],
prompt_delivery: PromptDelivery::Stdin,
},
ReviewerHarness::Codex => Self {
program: "codex".to_owned(),
args: vec![
"exec".to_owned(),
"-m".to_owned(),
model.to_owned(),
"-c".to_owned(),
format!("model_reasoning_effort={e}"),
],
prompt_delivery: PromptDelivery::PositionalArgument,
},
ReviewerHarness::Pi => Self {
program: "pi".to_owned(),
args: vec![
"--model".to_owned(),
model.to_owned(),
"--thinking".to_owned(),
e.to_owned(),
// Read-only tools so the reviewer can grep the repo for the whole
// defect class (Codex `exec` / Claude `-p` have read tools already).
"--tools".to_owned(),
"read,grep,find,ls".to_owned(),
"-p".to_owned(),
],
prompt_delivery: PromptDelivery::Stdin,
},
ReviewerHarness::Gemini => Self {
program: "gemini".to_owned(),
args: vec!["-m".to_owned(), model.to_owned()],
prompt_delivery: PromptDelivery::FlagValue("-p".to_owned()),
},
ReviewerHarness::Opencode => Self {
program: "opencode".to_owned(),
args: vec!["run".to_owned(), "--model".to_owned(), model.to_owned()],
prompt_delivery: PromptDelivery::PositionalArgument,
},
ReviewerHarness::Custom => return Err(ReviewerError::UnsupportedCustomHarness),
};
Ok(plan)
}
pub fn args_for_prompt(&self, prompt: &str) -> Vec<String> {
let mut args = self.args.clone();
match &self.prompt_delivery {
PromptDelivery::Stdin => {}
PromptDelivery::PositionalArgument => args.push(prompt.to_owned()),
PromptDelivery::FlagValue(flag) => {
args.push(flag.clone());
args.push(prompt.to_owned());
}
}
args
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum PromptDelivery {
Stdin,
PositionalArgument,
FlagValue(String),
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ProcessOutput {
pub status_code: Option<i32>,
pub stdout: String,
pub stderr: String,
}
pub trait ProcessRunner {
/// Run the reviewer invocation to completion.
///
/// `timeout` is a per-review wall-clock limit covering the WHOLE
/// invocation: stdin delivery, the review itself, the reap, and the pipe
/// drain. When the reviewer outlives it, the implementation must kill the
/// reviewer's process tree, reap the child, and return
/// [`ReviewerError::ReviewerTimeout`]. `None` means wait indefinitely.
fn run(
&self,
invocation: &InvocationPlan,
prompt: &str,
timeout: Option<Duration>,
) -> Result<ProcessOutput, ReviewerError>;
/// Run a reviewer in a disposable per-job checkout when one is available.
/// Test runners may keep the default implementation; production's standard
/// runner always honors the requested directory.
fn run_in_dir(
&self,
invocation: &InvocationPlan,
prompt: &str,
timeout: Option<Duration>,
current_dir: Option<&Path>,
) -> Result<ProcessOutput, ReviewerError> {
let _ = current_dir;
self.run(invocation, prompt, timeout)
}
}
#[derive(Clone, Copy, Debug, Default)]
pub struct StdProcessRunner;
impl ProcessRunner for StdProcessRunner {
fn run(
&self,
invocation: &InvocationPlan,
prompt: &str,
timeout: Option<Duration>,
) -> Result<ProcessOutput, ReviewerError> {
self.run_in_dir(invocation, prompt, timeout, None)
}
fn run_in_dir(
&self,
invocation: &InvocationPlan,
prompt: &str,
timeout: Option<Duration>,
current_dir: Option<&Path>,
) -> Result<ProcessOutput, ReviewerError> {
let mut command = Command::new(&invocation.program);
command.args(invocation.args_for_prompt(prompt));
command.stdout(Stdio::piped()).stderr(Stdio::piped());
if let Some(current_dir) = current_dir {
command.current_dir(current_dir);
}
if invocation.prompt_delivery == PromptDelivery::Stdin {
command.stdin(Stdio::piped());
}
// unix: the reviewer runs as its own process-group leader, so the
// timeout kill can take the WHOLE group. A descendant that inherited
// the output pipes must not survive the reviewer and hold them open —
// 0.9.2 froze the queue exactly that way.
#[cfg(unix)]
{
use std::os::unix::process::CommandExt as _;
command.process_group(0);
}
// The deadline covers everything from spawn on — stdin delivery, the
// review itself, the reap, and the pipe drain. The first timeout fix
// started the clock only after a blocking `write_all` of the prompt
// returned, so a reviewer that never read stdin (or stalled against a
// full pipe buffer before reading) wedged the queue forever despite
// the configured timeout.
let deadline = timeout.and_then(|limit| {
Instant::now()
.checked_add(limit)
.map(|instant| (instant, limit))
});
let mut child = command.spawn().map_err(ReviewerError::Spawn)?;
// Deliver the prompt on a helper thread so a reviewer that never
// reads stdin cannot outrun the deadline: the main thread proceeds
// straight to the deadline-enforcing wait, and the kill that fires on
// timeout closes the pipe and unblocks the writer. Dropping the write
// end after a completed write preserves the EOF contract.
let mut stdin_writer = if invocation.prompt_delivery == PromptDelivery::Stdin {
let pipe = child.stdin.take().ok_or(ReviewerError::MissingStdinPipe)?;
Some(StdinWriter::spawn(pipe, prompt.as_bytes().to_vec()))
} else {
None
};
// Drain both pipes on helper threads so a chatty reviewer never
// deadlocks against a full pipe buffer while we poll for exit (this is
// what `wait_with_output` does internally — split out so a timeout can
// kill the child and still collect, or boundedly abandon, the pipes).
let mut stdout_reader = child.stdout.take().map(spawn_pipe_reader);
let mut stderr_reader = child.stderr.take().map(spawn_pipe_reader);
let child_pid = child.id();
let status = match wait_for_child(child, deadline) {
Ok(status) => status,
Err(error) => {
// The child was killed and reaped (or the kill surfaced an
// error). The readers hit EOF once the process tree is down;
// their partial output is discarded — a timed out review
// records the timeout, not half a verdict. One shared
// absolute deadline covers all three collections — not a
// fresh grace per pipe — so a pipe-hoarding descendant cannot
// hold the queue past the configured grace no matter how many
// pipes it holds.
let cleanup_deadline = deadline.map(|_| Instant::now() + PIPE_DRAIN_GRACE);
if let Some(reader) = stdout_reader.as_mut() {
let _ =
reader.try_collect(remaining_until(cleanup_deadline), child_pid, "stdout");
}
if let Some(reader) = stderr_reader.as_mut() {
let _ =
reader.try_collect(remaining_until(cleanup_deadline), child_pid, "stderr");
}
if let Some(writer) = stdin_writer.as_mut() {
let _ =
writer.try_collect(remaining_until(cleanup_deadline), child_pid, "stdin");
}
return Err(error);
}
};
// One shared absolute deadline for the first collection attempt too:
// three sequential `try_collect` calls each getting their own fresh
// `PIPE_DRAIN_GRACE` could let a review exceed its whole-invocation
// timeout by up to 3x the grace before this fix.
let cleanup_deadline = deadline.map(|_| Instant::now() + PIPE_DRAIN_GRACE);
let mut prompt_write = stdin_writer.as_mut().map(|writer| {
writer.try_collect(remaining_until(cleanup_deadline), child_pid, "stdin")
});
let mut stdout = stdout_reader.as_mut().map(|reader| {
reader.try_collect(remaining_until(cleanup_deadline), child_pid, "stdout")
});
let mut stderr = stderr_reader.as_mut().map(|reader| {
reader.try_collect(remaining_until(cleanup_deadline), child_pid, "stderr")
});
if pipes_wedged(&prompt_write, &stdout, &stderr) {
// The reviewer exited but a spawned descendant inherited a pipe
// and holds it open, so the readers never saw EOF. Take the
// process tree down (best-effort; the group may already be gone)
// and give the stragglers one final — again shared, not
// per-pipe — grace before declaring the wedge: an unbounded join
// here re-creates the 0.9.2 freeze.
let _ = platform_tree_kill(child_pid);
let retry_deadline = deadline.map(|_| Instant::now() + PIPE_DRAIN_GRACE);
if prompt_write.as_ref().is_some_and(WriteState::is_wedged) {
prompt_write = stdin_writer.as_mut().map(|writer| {
writer.try_collect(remaining_until(retry_deadline), child_pid, "stdin")
});
}
if stdout.as_ref().is_some_and(DrainState::is_wedged) {
stdout = stdout_reader.as_mut().map(|reader| {
reader.try_collect(remaining_until(retry_deadline), child_pid, "stdout")
});
}
if stderr.as_ref().is_some_and(DrainState::is_wedged) {
stderr = stderr_reader.as_mut().map(|reader| {
reader.try_collect(remaining_until(retry_deadline), child_pid, "stderr")
});
}
}
if pipes_wedged(&prompt_write, &stdout, &stderr) {
return Err(ReviewerError::ReviewerOutputWedged {
grace_secs: PIPE_DRAIN_GRACE.as_secs(),
});
}
// A prompt-delivery failure invalidates the review even when the
// reviewer went on to exit cleanly: the verdict answered a truncated
// question. Surfaced, never swallowed.
if let Some(WriteState::Finished(Err(source))) = prompt_write {
return Err(ReviewerError::WritePrompt(source));
}
Ok(ProcessOutput {
status_code: status.code(),
stdout: String::from_utf8_lossy(&stdout.and_then(drain_bytes).unwrap_or_default())
.into_owned(),
stderr: String::from_utf8_lossy(&stderr.and_then(drain_bytes).unwrap_or_default())
.into_owned(),
})
}
}
/// Result of a bounded pipe-drain collect.
enum DrainState {
/// The reader thread delivered: drained bytes, or `None` when the read
/// itself failed (treated as empty output — the process result is the
/// authoritative signal, not the captured text).
Drained(Option<Vec<u8>>),
/// The grace window expired with the pipe still open: a descendant of the
/// reviewer inherited the write end and is holding it open.
Wedged,
}
impl DrainState {
fn is_wedged(&self) -> bool {
matches!(self, Self::Wedged)
}
}
/// Extract bytes from a completed drain; a wedge is handled by the caller
/// before this point, so it degrades to empty here.
fn drain_bytes(state: DrainState) -> Option<Vec<u8>> {
match state {
DrainState::Drained(bytes) => bytes,
DrainState::Wedged => None,
}
}
/// Whether any of the three pipe collections is still wedged.
fn pipes_wedged(
prompt_write: &Option<WriteState>,
stdout: &Option<DrainState>,
stderr: &Option<DrainState>,
) -> bool {
prompt_write.as_ref().is_some_and(WriteState::is_wedged)
|| stdout.as_ref().is_some_and(DrainState::is_wedged)
|| stderr.as_ref().is_some_and(DrainState::is_wedged)
}
/// Result of a bounded stdin-writer collect.
enum WriteState {
Finished(io::Result<()>),
Wedged,
}
impl WriteState {
fn is_wedged(&self) -> bool {
matches!(self, Self::Wedged)
}
}
/// A pipe-draining thread whose output is collected through a channel, so the
/// join can be bounded by a grace window. `JoinHandle::join` alone cannot
/// time out — the 0.9.2 queue freeze was an unbounded join on a pipe whose
/// write end a surviving descendant held open.
struct PipeReader {
handle: Option<std::thread::JoinHandle<()>>,
receiver: std::sync::mpsc::Receiver<io::Result<Vec<u8>>>,
}
fn spawn_pipe_reader(mut pipe: impl Read + Send + 'static) -> PipeReader {
let (sender, receiver) = std::sync::mpsc::channel();
let handle = std::thread::spawn(move || {
let mut buffer = Vec::new();
let result = pipe.read_to_end(&mut buffer).map(|_| buffer);
let _ = sender.send(result);
});
PipeReader {
handle: Some(handle),
receiver,
}
}
impl PipeReader {
/// Collect the drained bytes. `Some(grace)` bounds the wait; `None`
/// blocks indefinitely (the explicit no-timeout mode). Once the value is
/// received the thread has finished, so the join cannot block. On an
/// expired grace the thread is still blocked in `read_to_end`; see
/// `reap_wedged_pipe_thread` for why that's logged and left to exit on
/// its own rather than joined from a second thread.
fn try_collect(
&mut self,
grace: Option<Duration>,
pid: u32,
label: &'static str,
) -> DrainState {
use std::sync::mpsc::RecvTimeoutError;
let received = match grace {
Some(grace) => self.receiver.recv_timeout(grace),
None => self
.receiver
.recv()
.map_err(|_| RecvTimeoutError::Disconnected),
};
match received {
Ok(result) => {
if let Some(handle) = self.handle.take() {
let _ = handle.join();
}
DrainState::Drained(result.ok())
}
Err(RecvTimeoutError::Timeout | RecvTimeoutError::Disconnected) => {
if let Some(handle) = self.handle.take() {
reap_wedged_pipe_thread(pid, label, handle);
}
DrainState::Wedged
}
}
}
}
/// A prompt-delivery thread with the same bounded-collect contract as
/// [`PipeReader`].
struct StdinWriter {
handle: Option<std::thread::JoinHandle<()>>,
receiver: std::sync::mpsc::Receiver<io::Result<()>>,
}
impl StdinWriter {
fn spawn(mut pipe: std::process::ChildStdin, bytes: Vec<u8>) -> Self {
let (sender, receiver) = std::sync::mpsc::channel();
let handle = std::thread::spawn(move || {
let result = pipe.write_all(&bytes);
// Dropping the write end signals EOF to the reviewer.
drop(pipe);
let _ = sender.send(result);
});
Self {
handle: Some(handle),
receiver,
}
}
fn try_collect(
&mut self,
grace: Option<Duration>,
pid: u32,
label: &'static str,
) -> WriteState {
use std::sync::mpsc::RecvTimeoutError;
let received = match grace {
Some(grace) => self.receiver.recv_timeout(grace),
None => self
.receiver
.recv()
.map_err(|_| RecvTimeoutError::Disconnected),
};
match received {
Ok(result) => {
if let Some(handle) = self.handle.take() {
let _ = handle.join();
}
WriteState::Finished(result)
}
Err(RecvTimeoutError::Timeout | RecvTimeoutError::Disconnected) => {
if let Some(handle) = self.handle.take() {
reap_wedged_pipe_thread(pid, label, handle);
}
WriteState::Wedged
}
}
}
}
/// Compute the remaining budget until a shared absolute cleanup deadline.
/// `None` means "wait indefinitely" (no timeout configured for this
/// invocation at all); `Some(Duration::ZERO)` means the deadline has already
/// elapsed, so the caller's next collect must resolve immediately or be
/// declared wedged rather than getting a fresh grace window.
fn remaining_until(deadline: Option<Instant>) -> Option<Duration> {
deadline.map(|deadline| deadline.saturating_duration_since(Instant::now()))
}
/// A pipe-drain or stdin-write helper thread that outlived its grace window
/// is still blocked in a syscall, most often because a descendant of the
/// reviewer inherited its pipe and holds it open. Dropping the `JoinHandle`
/// at this point would silently detach it: the OS thread keeps running,
/// untracked and unlogged, potentially forever if that pipe never closes —
/// exactly the "up to three leaked threads per review" failure mode.
/// Instead, move the join to a dedicated background thread: the queue is
/// never blocked waiting on it, but it is still joined, and both the
/// detection and the eventual (or permanently absent) completion are logged.
fn reap_wedged_pipe_thread(pid: u32, label: &'static str, handle: std::thread::JoinHandle<()>) {
// CodeRabbit/Codex round-2 (flagged independently by both, twice each):
// spawning a background thread to `.join()` this handle does not bound
// anything. If the helper is blocked because a descendant escaped the
// process-tree kill and still holds the pipe open, the join blocks
// forever too — each wedge then leaks TWO threads instead of one, which
// is worse than round 1's silent-drop problem, not better. There is no
// way to make a std `JoinHandle::join()` time out, so the only way to
// avoid retaining an unbounded thread here is to not spawn one at all:
// log the detection once and let the handle drop. This costs no
// additional thread; the original helper still exits on its own
// whenever its blocking read/write eventually returns, which may be
// never if the descendant never exits — a limitation this crate already
// documents elsewhere as best-effort process-tree coverage, not a
// completeness guarantee.
tracing::warn!(
pid,
label,
"reviewer pipe helper thread still blocked after its grace window (a descendant likely \
escaped the process-tree kill and still holds the pipe open); not spawning a reaper for \
it, since that reaper's own join could block forever too — leaving it to exit on its own"
);
drop(handle);
}
/// How often to poll the child while waiting under a timeout. Small enough
/// that a killed reviewer is reaped promptly, large enough to stay off the
/// CPU for a 20-minute default window.
const CHILD_POLL_INTERVAL: Duration = Duration::from_millis(100);
/// How long to wait after a kill before surfacing an unreaped child as a hard
/// error. This is bounded so a process-tree kill race cannot wedge the queue
/// even if the child survives longer than the request timeout.
const KILL_REAP_GRACE: Duration = Duration::from_secs(1);
/// Grace for collecting reviewer output after the child resolves (clean exit
/// or kill + reap). Pipe drains are kernel-buffer copies that finish in
/// milliseconds once the write end closes; the bound exists so a descendant
/// holding a pipe open can never wedge the queue the way 0.9.2 did.
const PIPE_DRAIN_GRACE: Duration = Duration::from_secs(5);
/// Wait for `child`, enforcing an optional wall-clock deadline.
///
/// With no deadline this is a plain blocking wait. With a deadline the child
/// is polled; once it passes, the reviewer's whole process tree is killed and
/// the child is reaped within a bounded grace. If the direct child still does
/// not report exit after that grace, cleanup is handed to a background reaper
/// before returning [`ReviewerError::ReviewerTimeout`] so the queue hot path
/// never blocks on `wait`.
fn wait_for_child(
mut child: std::process::Child,
deadline: Option<(Instant, Duration)>,
) -> Result<std::process::ExitStatus, ReviewerError> {
let Some((deadline, limit)) = deadline else {
return child.wait().map_err(ReviewerError::Wait);
};
loop {
// Deadline first, then try_wait: a poll tick that lands after the
// deadline has already passed must never CONTINUE POLLING and accept
// a status the child happens to reach in some LATER tick as a
// "clean" success — that reintroduces exactly the nondeterministic
// over-deadline success the wall-clock contract promises never
// happens. Once `kill_and_reap` is actually invoked, every path
// funnels through it, which always reports the timeout regardless of
// how the child resolves — that part of the round-1 fix is
// unconditional and untouched here.
if Instant::now() >= deadline {
// Codex round-2 P2 (reviewer.rs ~806): CHILD_POLL_INTERVAL (100ms)
// is coarser than a short configured timeout can be (mocks,
// fast-fail tests, or any limit < the poll interval), and even a
// generous timeout can have the child exit during the FINAL sleep
// that straddles the deadline instant. Without this one-time
// final glance, a child that genuinely finished within its
// configured budget gets misreported as timed out purely because
// our own poll granularity hadn't looked yet — a self-inflicted
// timeout, not the child's fault. This is not "continuing to
// wait": it is a single non-blocking check of what has ALREADY
// happened, made before any kill action is taken. If the child is
// still running, we proceed to kill_and_reap exactly as before.
if let Some(status) = child.try_wait().map_err(ReviewerError::Wait)? {
return Ok(status);
}
return kill_and_reap(child, limit);
}
if let Some(status) = child.try_wait().map_err(ReviewerError::Wait)? {
return Ok(status);
}
std::thread::sleep(CHILD_POLL_INTERVAL);
}
}
/// The deadline passed with the child still running: kill the reviewer's
/// process tree, reap the child, and decide the outcome. The short grace is a
/// bounded diagnostic window; if it expires, a final direct kill is attempted
/// and any remaining reap is moved off the queue hot path.
fn kill_and_reap(
child: std::process::Child,
limit: Duration,
) -> Result<std::process::ExitStatus, ReviewerError> {
let mut child = child;
kill_process_tree(&mut child)?;
let status = match wait_for_reap(&mut child, KILL_REAP_GRACE) {
Ok(status) => status,
Err(error) => return reap_after_grace(child, error, limit.as_secs()),
};
outcome_after_kill(status, limit.as_secs())
}
fn reap_after_grace(
mut child: std::process::Child,
grace_error: io::Error,
timeout_secs: u64,
) -> Result<std::process::ExitStatus, ReviewerError> {
let pid = child.id();
tracing::warn!(
pid,
%grace_error,
"reviewer kill grace expired; handing direct-child reap to background cleanup"
);
match child.kill() {
Ok(()) => {}
Err(error) if error.kind() == io::ErrorKind::InvalidInput => {}
Err(source) => {
return Err(ReviewerError::KillReviewer {
pid,
tree_error: "direct-child reap fallback".to_owned(),
source,
});
}
}
if let Some(status) = child.try_wait().map_err(ReviewerError::Wait)? {
return outcome_after_kill(status, timeout_secs);
}
let reaper = std::thread::Builder::new()
.name(format!("truth-mirror-reaper-{pid}"))
.spawn(move || {
if let Err(error) = child.wait() {
tracing::debug!(pid, %error, "background reviewer reaper stopped");
}
});
if let Err(error) = reaper {
tracing::warn!(pid, %error, "could not spawn background reviewer reaper");
}
Err(ReviewerError::ReviewerTimeout { timeout_secs })
}
fn wait_for_reap(
child: &mut std::process::Child,
grace: Duration,
) -> io::Result<std::process::ExitStatus> {
let started = Instant::now();
loop {
if let Some(status) = child.try_wait()? {
return Ok(status);
}
if started.elapsed() >= grace {
return Err(io::Error::other(format!(
"killed child did not exit within {grace:?} (pid={})",
child.id()
)));
}
std::thread::sleep(CHILD_POLL_INTERVAL);
}
}
/// Decide the outcome after the timeout kill landed.
///
/// Once `wait_for_child` has decided the deadline was crossed and committed
/// to `kill_and_reap`, the result is always a timeout — regardless of how the
/// reap resolves. An earlier version let a child that exited on its own
/// (unix: no signal on the reaped status) "win" the kill race and return
/// `Ok`, treating the review as having produced a valid verdict. That made
/// the reported outcome depend on exactly when the poll loop happened to
/// observe the exit relative to the deadline — an over-deadline review could
/// succeed or time out depending on scheduling, contradicting the documented
/// wall-clock contract (a review either finishes within its budget or it
/// times out, never both depending on luck). The exit status is still
/// reaped and logged for diagnostics; it just never overrides the timeout.
fn outcome_after_kill(
status: std::process::ExitStatus,
timeout_secs: u64,
) -> Result<std::process::ExitStatus, ReviewerError> {
tracing::debug!(
?status,
timeout_secs,
"reviewer reaped after deadline kill; reporting timeout regardless of reap outcome"
);
Err(ReviewerError::ReviewerTimeout { timeout_secs })
}
/// Kill the reviewer and every descendant it spawned.
///
/// unix: the child was spawned as a process-group leader, so a direct
/// process-group `SIGKILL` via rustix signals the whole group — descendants
/// cannot survive to hold the output pipes open *for that group*. Descendants
/// that escape into a different process group (for example by `setsid`) are
/// outside this signal window, so this is best-effort coverage, not complete
/// process-tree proof.
///
/// windows: `taskkill /F /T` attempts whole-tree termination. A direct kill
/// fallback still applies when taskkill cannot run. Non-unix hosts have only a
/// direct child kill fallback, so full-tree cleanup is not guaranteed there.
///
/// In every case, a genuine kill failure is surfaced, never swallowed. The
/// direct child is always reaped; process-tree cleanup remains best-effort for
/// descendants that escape the platform's tree primitive.
fn kill_process_tree(child: &mut std::process::Child) -> Result<(), ReviewerError> {
let pid = child.id();
if let Err(tree_error) = platform_tree_kill(pid) {
// The tree kill failed — most often because the process already
// exited and took the group with it. A direct kill still covers the
// child itself.
match child.kill() {
Ok(()) => {}
Err(direct_error) => {
// std reports InvalidInput when the child was already reaped:
// the kill/reap race, not a failure — the subsequent wait
// surfaces the real exit status.
if direct_error.kind() != io::ErrorKind::InvalidInput {
return Err(ReviewerError::KillReviewer {
pid,
tree_error: tree_error.to_string(),
source: direct_error,
});
}
}
}
}
Ok(())
}
#[cfg(unix)]
fn platform_tree_kill(pid: u32) -> io::Result<()> {
// The reviewer is spawned as its own process-group leader
// (`process_group(0)`), so the pgid equals the child pid. Signal the
// whole group directly via rustix instead of shelling out to `kill`:
// Depot Linux CI can fail the external-command path even when the
// process group itself is killable.
let Some(group) = rustix::process::Pid::from_raw(pid as i32) else {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
format!("invalid reviewer process group pid: {pid}"),
));
};
rustix::process::kill_process_group(group, rustix::process::Signal::KILL)
.map_err(io::Error::from)
}
#[cfg(windows)]
fn platform_tree_kill(pid: u32) -> io::Result<()> {
// /T takes down the whole process tree rooted at the reviewer.
let status = Command::new("taskkill")
.args(["/F", "/T", "/PID", &pid.to_string()])
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null())
.status()?;
if status.success() {
Ok(())
} else {
Err(io::Error::other(format!(
"taskkill /F /T /PID {pid} exited with {status}"
)))
}
}
#[cfg(not(any(unix, windows)))]
fn platform_tree_kill(_pid: u32) -> io::Result<()> {
// Non-unix hosts have no process-tree kill primitive here, so callers can
// only attempt a best-effort direct child kill after this returns.
Err(io::Error::other(
"process-tree kill is not supported on this platform",
))
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ReviewJob {
pub commit_sha: String,
pub claim: Claim,
pub diff: String,
/// Ground-truth constraints + recent trajectory injected into review prompts.
pub context: String,
pub request: ReviewRequest,
pub strict: Option<StrictReviewConfig>,
/// When set, this job is a petition re-review for the listed original
/// rejection SHA. The reviewer is asked "does <fix sha> materially address
/// each finding?", and a `PASS` verdict (the petition's accept — there is
/// no separate `ACCEPT` variant on the wire) transitions the original
/// rejection to `resolved`; a non-PASS verdict leaves the attempt spent
/// (escalating to `needs-human` once the counter hits the bound).
pub petition: Option<PetitionContext>,
}
/// Original rejection payload bundled into a petition re-review.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct PetitionContext {
pub original_sha: String,
pub fix_sha: String,
pub original_claim: String,
pub original_summary: String,
pub original_findings: Vec<String>,
pub original_structured_findings: Vec<StructuredFinding>,
pub original_reviewer_model: String,
pub attempts_so_far: u32,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct StrictReviewConfig {
pub arbiter_harness: ReviewerHarness,
pub arbiter_model: String,
pub arbiter_effort: Effort,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ReviewExecution {
pub entries: Vec<LedgerEntry>,
}
/// If the job carries a petition context, rebuild the request with the
/// petition-specific prompt so the reviewer is asked the right question.
/// The reviewer invocation plan is otherwise unchanged; only the prompt
/// text the reviewer receives is replaced.
fn materialize_petition_prompt(mut job: ReviewJob) -> ReviewJob {
let Some(petition) = &job.petition else {
return job;
};
job.request.prompt = petition_prompt(petition, &job.claim, &job.diff, &job.context);
job
}
pub fn execute_review_job<R: ProcessRunner>(
job: ReviewJob,
runner: &R,
store: &LedgerStore,
timeout: Option<Duration>,
) -> Result<ReviewExecution, ReviewerError> {
let job = materialize_petition_prompt(job);
let execution = draft_review_job(&job, runner, timeout)?;
apply_review_execution(&job, &execution, store)?;
Ok(execution)
}
fn strict_pass_needed(job: &ReviewJob, first_verdict: &ParsedVerdict) -> bool {
job.strict.is_some()
&& first_verdict.verdict == Verdict::Pass
&& first_verdict.findings.is_empty()
}
fn draft_review_first_pass_in_dir<R: ProcessRunner>(
job: &ReviewJob,
runner: &R,
timeout: Option<Duration>,
current_dir: Option<&Path>,
) -> Result<(LedgerEntry, String), ReviewerError> {
let first_plan = ReviewPlan::build(job.request.clone())?;
let first_output =
first_plan.run_with_dir(&job.request.prompt, runner, timeout, current_dir)?;
ensure_process_success(&first_output)?;
let first_verdict = ParsedVerdict::parse(&first_output.stdout)?;
let first_entry = entry_from_verdict(job, &first_plan, &first_verdict);
Ok((first_entry, first_output.stdout))
}
fn draft_review_strict_pass_in_dir<R: ProcessRunner>(
job: &ReviewJob,
runner: &R,
first_stdout: &str,
timeout: Option<Duration>,
current_dir: Option<&Path>,
) -> Result<LedgerEntry, ReviewerError> {
let strict = job
.strict
.as_ref()
.expect("strict pass requested without strict config");
validate_strict_arbiter(&job.request, strict)?;
let strict_prompt = strict_second_pass_prompt(job, first_stdout);
let strict_request = ReviewRequest::new(
job.request.watched_agent,
job.request.watched_model.clone(),
strict.arbiter_harness,
strict.arbiter_model.clone(),
false,
strict_prompt,
)
.with_effort(strict.arbiter_effort);
let strict_plan = ReviewPlan::build(strict_request.clone())?;
let strict_output =
strict_plan.run_with_dir(&strict_request.prompt, runner, timeout, current_dir)?;
ensure_process_success(&strict_output)?;
let strict_verdict = ParsedVerdict::parse(&strict_output.stdout)?;
Ok(entry_from_verdict(job, &strict_plan, &strict_verdict))
}
pub(crate) fn draft_review_job<R: ProcessRunner>(
job: &ReviewJob,
runner: &R,
timeout: Option<Duration>,
) -> Result<ReviewExecution, ReviewerError> {
draft_review_job_in_dir(job, runner, timeout, None)
}
fn draft_review_job_in_dir<R: ProcessRunner>(
job: &ReviewJob,
runner: &R,
timeout: Option<Duration>,
current_dir: Option<&Path>,
) -> Result<ReviewExecution, ReviewerError> {
let (first_entry, first_stdout) =
draft_review_first_pass_in_dir(job, runner, timeout, current_dir)?;
let first_verdict = ParsedVerdict::parse(&first_stdout)?;
let mut entries = vec![first_entry];
if strict_pass_needed(job, &first_verdict) {
entries.push(draft_review_strict_pass_in_dir(
job,
runner,
&first_stdout,
timeout,
current_dir,
)?);
}
Ok(ReviewExecution { entries })
}
fn ledger_entry_canonical_line(entry: &LedgerEntry) -> Result<String, ReviewerError> {
Ok(
String::from_utf8_lossy(&serde_json::to_vec(entry).map_err(ReviewerError::RunJson)?)
.into_owned(),
)
}
fn ledger_contains_exact_entry(
store: &LedgerStore,
entry: &LedgerEntry,
) -> Result<bool, ReviewerError> {
let needle = ledger_entry_canonical_line(entry)?;
Ok(store
.read_history()
.map_err(ReviewerError::Ledger)?
.iter()
.any(|existing| {
ledger_entry_canonical_line(existing)
.ok()
.is_some_and(|line| line == needle)
}))
}
fn append_entry_idempotent(store: &LedgerStore, entry: &LedgerEntry) -> Result<(), ReviewerError> {
if ledger_contains_exact_entry(store, entry)? {
return Ok(());
}
store.append_entry(entry).map_err(ReviewerError::Ledger)
}
fn petition_provenance_matches(
original: &LedgerEntry,
petition: &PetitionContext,
attempts: u32,
) -> bool {
if original.petition_attempts != attempts {
return false;
}
if let Some(provenance) = &original.transition_provenance {
return provenance.kind == crate::ledger::TransitionProvenanceKind::PetitionReview
&& provenance.fix_sha == petition.fix_sha
&& provenance.original_sha == petition.original_sha
&& provenance.attempts == attempts;
}
original.resolution.as_ref().is_some_and(|resolution| {
resolution.reason.contains("petition review of fix")
&& resolution.reason.contains(&petition.fix_sha)
&& resolution.reason.contains(&petition.original_sha)
})
}
fn reviewer_petition_open_transition_applied(
original: &LedgerEntry,
petition: &PetitionContext,
attempts: u32,
) -> bool {
original.is_unresolved_rejection() && petition_provenance_matches(original, petition, attempts)
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum ExpectedPetitionOutcome {
Resolved,
NeedsHuman,
OpenAttempt,
}
fn expected_petition_outcome(
verdict_entry: &LedgerEntry,
) -> Result<ExpectedPetitionOutcome, ReviewerError> {
let max_attempts = crate::resolve::MAX_PETITION_ATTEMPTS;
if crate::resolve::petition_accepts(verdict_entry.verdict) {
return Ok(ExpectedPetitionOutcome::Resolved);
}
if verdict_entry.petition_attempts >= max_attempts {
return Ok(ExpectedPetitionOutcome::NeedsHuman);
}
Ok(ExpectedPetitionOutcome::OpenAttempt)
}
fn petition_outcome_already_applied(
original: &LedgerEntry,
petition: &PetitionContext,
verdict_entry: &LedgerEntry,
) -> Result<bool, ReviewerError> {
let expected = expected_petition_outcome(verdict_entry)?;
match expected {
ExpectedPetitionOutcome::Resolved => Ok(original.disposition == Disposition::Resolved
&& petition_provenance_matches(original, petition, verdict_entry.petition_attempts)),
ExpectedPetitionOutcome::NeedsHuman => Ok(original.is_needs_human()
&& petition_provenance_matches(original, petition, verdict_entry.petition_attempts)),
ExpectedPetitionOutcome::OpenAttempt => Ok(reviewer_petition_open_transition_applied(
original,
petition,
verdict_entry.petition_attempts,
)),
}
}
fn apply_petition_transition_idempotent(
job: &ReviewJob,
entry: &LedgerEntry,
store: &LedgerStore,
) -> Result<(), ReviewerError> {
let Some(petition) = &job.petition else {
return Ok(());
};
apply_petition_context_transition_idempotent(petition, entry, store)
}
fn apply_petition_context_transition_idempotent(
petition: &PetitionContext,
entry: &LedgerEntry,
store: &LedgerStore,
) -> Result<(), ReviewerError> {
let original = store
.show(&petition.original_sha)
.map_err(ReviewerError::Ledger)?;
if petition_outcome_already_applied(&original, petition, entry)? {
return Ok(());
}
if original.is_unresolved_rejection() || original.is_needs_human() {
return apply_petition_context_transition(petition, entry, store);
}
Ok(())
}
fn apply_review_execution(
job: &ReviewJob,
execution: &ReviewExecution,
store: &LedgerStore,
) -> Result<(), ReviewerError> {
for entry in &execution.entries {
append_entry_idempotent(store, entry)?;
}
if let Some(last) = execution.entries.last() {
apply_petition_transition_idempotent(job, last, store)?;
}
Ok(())
}
fn apply_petition_batch_execution(
petition: &PetitionContext,
execution: &ReviewExecution,
store: &LedgerStore,
) -> Result<(), ReviewerError> {
for entry in &execution.entries {
append_entry_idempotent(store, entry)?;
}
if let Some(last) = execution.entries.last() {
apply_petition_context_transition_idempotent(petition, last, store)?;
}
Ok(())
}
/// After a petition reviewer's verdict is recorded, transition the original
/// rejection: `PASS` (the petition's accept) -> `resolved` (with provenance); anything else ->
/// increment the attempt counter; on the bounded-th attempt, escalate to
/// `needs-human`. The bounded max lives in `crate::resolve::MAX_PETITION_ATTEMPTS`.
fn apply_petition_context_transition(
petition: &PetitionContext,
entry: &LedgerEntry,
store: &LedgerStore,
) -> Result<(), ReviewerError> {
let max_attempts = crate::resolve::MAX_PETITION_ATTEMPTS;
let next_attempts = entry.petition_attempts;
let reason = format!(
"petition review of fix {} against rejection {} (attempts {})",
petition.fix_sha, petition.original_sha, next_attempts
);
let provenance = Some(crate::ledger::TransitionProvenance {
kind: crate::ledger::TransitionProvenanceKind::PetitionReview,
fix_sha: petition.fix_sha.clone(),
original_sha: petition.original_sha.clone(),
attempts: next_attempts,
});
if crate::resolve::petition_accepts(entry.verdict) {
// PASS: the fix materially addresses each finding. FLAG remains auditable
// debt on the petition review entry but does not resolve the rejection.
store
.append_petition_transition_with_provenance(
&petition.original_sha,
crate::ledger::Disposition::Resolved,
crate::ledger::ResolutionKind::Resolved,
&reason,
next_attempts,
provenance.clone(),
)
.map_err(ReviewerError::Ledger)?;
} else if next_attempts >= max_attempts {
store
.escalate_to_needs_human_with_attempts(
&petition.original_sha,
&reason,
next_attempts,
provenance.clone(),
)
.map_err(ReviewerError::Ledger)?;
} else {
store
.append_petition_transition_with_provenance(
&petition.original_sha,
crate::ledger::Disposition::Open,
crate::ledger::ResolutionKind::Resolved,
&reason,
next_attempts,
provenance,
)
.map_err(ReviewerError::Ledger)?;
}
Ok(())
}
/// Rebuild the petition context for a queued petition item from the ledger:
/// the original rejection's claim, findings, and reviewer, plus the current
/// attempt counter. The CLI's `resolve --fixed-by` already counted this
/// attempt when it enqueued, so `attempts_so_far` INCLUDES the in-flight
/// attempt and the verdict entry carries it unchanged (no double increment).
fn petition_context_from_ledger(
original_sha: &str,
fix_sha: &str,
store: &LedgerStore,
) -> Result<PetitionContext, ReviewerError> {
let original = store.show(original_sha).map_err(ReviewerError::Ledger)?;
let attempts_so_far = store
.petition_attempts_for(original_sha)
.map_err(ReviewerError::Ledger)?;
Ok(PetitionContext {
original_sha: original_sha.to_owned(),
fix_sha: fix_sha.to_owned(),
original_claim: original.claim.clone(),
original_summary: original.summary.clone(),
original_findings: original.findings.clone(),
original_structured_findings: original.structured_findings.clone(),
original_reviewer_model: original.reviewer.model.clone(),
attempts_so_far,
})
}
/// Whether a queued petition's target can still accept a petition transition:
/// the rejection exists and is still open (or escalated to needs-human, which
/// remains petitionable). Anything else — already resolved, waived, or never
/// recorded — makes the queued petition a dead letter: the drain must consume
/// it and keep going instead of dying on `NoOpenRejection`, which is exactly
/// how 0.9.2 killed watchers when a petition was enqueued twice.
fn petition_target_is_actionable(
store: &LedgerStore,
original_sha: &str,
) -> Result<bool, ReviewerError> {
match store.show(original_sha) {
Ok(entry) => Ok(entry.is_unresolved_rejection() || entry.is_needs_human()),
Err(LedgerError::NotFound { .. }) => Ok(false),
Err(error) => Err(ReviewerError::Ledger(error)),
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ParsedVerdict {
pub verdict: Verdict,
pub summary: String,
pub findings: Vec<String>,
pub structured_findings: Vec<StructuredFinding>,
pub next_steps: Vec<String>,
pub memory_skill_classification: MemorySkillClassification,
pub raw: String,
}
impl ParsedVerdict {
pub fn parse(output: &str) -> Result<Self, ReviewerError> {
// Reviewer models routinely wrap the verdict in prose and/or a
// ```json fenced block despite the prompt asking for raw JSON.
let candidate = extract_verdict_json(output);
let parsed: ReviewerJsonOutput =
serde_json::from_str(candidate).map_err(|source| ReviewerError::VerdictJson {
source,
output: output.to_owned(),
})?;
Self::from_json(parsed, output.to_owned())
}
fn from_json(mut parsed: ReviewerJsonOutput, raw: String) -> Result<Self, ReviewerError> {
if let Err(error) = parsed.validate() {
if !parsed.salvage_disallowed_pass_skill_proposal() {
return Err(error);
}
tracing::warn!(
verdict = %parsed.verdict,
"reviewer returned a PASS verdict with a disallowed memory-skill proposal; stripped the proposal and accepted the verdict"
);
parsed.validate()?;
}
let findings = parsed
.findings
.iter()
.map(StructuredFinding::display_line)
.collect();
Ok(Self {
verdict: parsed.verdict,
summary: parsed.summary,
findings,
structured_findings: parsed.findings,
next_steps: parsed.next_steps,
memory_skill_classification: parsed.memory_skill,
raw,
})
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
enum ParsedPetitionBatchMember {
Valid(ParsedVerdict),
Invalid(String),
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct ParsedPetitionBatch {
members: std::collections::BTreeMap<String, ParsedPetitionBatchMember>,
unknown_item_ids: Vec<String>,
}
impl ParsedPetitionBatch {
fn parse(
output: &str,
expected_batch_id: &str,
expected_fix_sha: &str,
requested_shas: &[String],
) -> Result<Self, ReviewerError> {
let candidate = extract_verdict_json(output);
let document: serde_json::Value =
serde_json::from_str(candidate).map_err(|source| ReviewerError::VerdictJson {
source,
output: output.to_owned(),
})?;
let object = document
.as_object()
.ok_or_else(|| ReviewerError::BatchVerdictSchema {
message: "batch verdict must be a JSON object".to_owned(),
})?;
let batch_id = object
.get("batch_id")
.and_then(serde_json::Value::as_str)
.ok_or_else(|| ReviewerError::BatchVerdictSchema {
message: "batch_id must be a string".to_owned(),
})?;
if batch_id != expected_batch_id {
return Err(ReviewerError::BatchVerdictSchema {
message: format!(
"batch_id mismatch: expected {expected_batch_id:?}, got {batch_id:?}"
),
});
}
let fix_sha = object
.get("fix_sha")
.and_then(serde_json::Value::as_str)
.ok_or_else(|| ReviewerError::BatchVerdictSchema {
message: "fix_sha must be a string".to_owned(),
})?;
if fix_sha != expected_fix_sha {
return Err(ReviewerError::BatchVerdictSchema {
message: format!(
"fix_sha mismatch: expected {expected_fix_sha:?}, got {fix_sha:?}"
),
});
}
let results = object
.get("results")
.and_then(serde_json::Value::as_array)
.ok_or_else(|| ReviewerError::BatchVerdictSchema {
message: "results must be an array".to_owned(),
})?;
let requested: std::collections::BTreeSet<&str> =
requested_shas.iter().map(String::as_str).collect();
let mut candidates = std::collections::BTreeMap::<&str, Vec<&serde_json::Value>>::new();
let mut unknown_item_ids = Vec::new();
for result in results {
match result
.get("original_rejection_sha")
.and_then(serde_json::Value::as_str)
{
Some(original_sha) if requested.contains(original_sha) => {
candidates.entry(original_sha).or_default().push(result);
}
Some(original_sha) => unknown_item_ids.push(original_sha.to_owned()),
None => unknown_item_ids.push("<malformed original_rejection_sha>".to_owned()),
}
}
let mut members = std::collections::BTreeMap::new();
for requested_sha in requested_shas {
let outcome = match candidates.get(requested_sha.as_str()).map(Vec::as_slice) {
None | Some([]) => ParsedPetitionBatchMember::Invalid(format!(
"missing batch result for original rejection {requested_sha}"
)),
Some([_, _, ..]) => ParsedPetitionBatchMember::Invalid(format!(
"duplicate batch results for original rejection {requested_sha}"
)),
Some([result]) => {
let raw = serde_json::to_string(result).map_err(ReviewerError::RunJson)?;
match serde_json::from_value::<ReviewerJsonOutput>((*result).clone()) {
Ok(parsed) => match ParsedVerdict::from_json(parsed, raw) {
Ok(verdict) => ParsedPetitionBatchMember::Valid(verdict),
Err(error) => ParsedPetitionBatchMember::Invalid(format!(
"invalid batch result for original rejection {requested_sha}: {error}"
)),
},
Err(error) => ParsedPetitionBatchMember::Invalid(format!(
"invalid batch result for original rejection {requested_sha}: {error}"
)),
}
}
};
members.insert(requested_sha.clone(), outcome);
}
Ok(Self {
members,
unknown_item_ids,
})
}
}
/// Pull the verdict JSON out of a reviewer reply that may wrap it in prose
/// and/or markdown fences. Every candidate is VALIDATED as JSON before it is
/// returned — a shape that fails to parse falls through to the next strategy
/// instead of poisoning the caller (leading JSON + trailing prose was exactly
/// that failure: `starts_with('{')` returned the whole text and serde died on
/// the trailing characters). Precedence:
/// 1. the whole (trimmed) output — the documented contract, zero-cost;
/// 2. the longest valid JSON value PREFIX when the output starts with `{`
/// (leading verdict, trailing prose);
/// 3. the first ```json (or bare ```) fenced block that parses as a JSON
/// object containing a `"verdict"` key (later fences are scanned before
/// accepting a non-verdict JSON fragment);
/// 4. every `{` position in the output, preferring the first JSON object that
/// contains a `"verdict"` key and falling back to the first valid JSON
/// value if none do.
///
/// Returns a slice of `output`; the caller still surfaces the full original
/// text in parse errors and stores it verbatim as `raw`.
fn extract_verdict_json(output: &str) -> &str {
fn parses(candidate: &str) -> bool {
serde_json::from_str::<serde_json::Value>(candidate).is_ok()
}
/// Parse the first complete JSON value at the start of `text` and return
/// both its byte length and the parsed value.
fn json_prefix(text: &str) -> Option<(usize, serde_json::Value)> {
let mut stream = serde_json::Deserializer::from_str(text).into_iter::<serde_json::Value>();
match stream.next() {
Some(Ok(value)) => Some((stream.byte_offset(), value)),
_ => None,
}
}
fn json_prefix_len(text: &str) -> Option<usize> {
json_prefix(text).map(|(len, _)| len)
}
fn value_has_verdict(value: &serde_json::Value) -> bool {
matches!(value, serde_json::Value::Object(map) if map.contains_key("verdict"))
}
let trimmed = output.trim();
if trimmed.starts_with('{') {
if parses(trimmed) {
return trimmed;
}
if let Some(end) = json_prefix_len(trimmed) {
return trimmed[..end].trim();
}
}
// Fenced block: find ``` openers, skip the info string (e.g. `json`),
// and take the body up to the closing fence. Prefer a body that parses as
// a verdict object; remember the first valid-but-non-verdict body as a
// fallback so existing behavior for bare JSON fences stays stable.
let mut search_from = 0;
let mut first_valid_fence: Option<&str> = None;
while let Some(open_rel) = output[search_from..].find("```") {
let after_open = search_from + open_rel + 3;
let body_start = match output[after_open..].find('\n') {
Some(newline_rel) => after_open + newline_rel + 1,
None => break,
};
let Some(close_rel) = output[body_start..].find("```") else {
break;
};
let body = output[body_start..body_start + close_rel].trim();
if body.starts_with('{')
&& let Some((end, ref value)) = json_prefix(body)
{
let candidate = body[..end].trim();
if value_has_verdict(value) {
return candidate;
}
if first_valid_fence.is_none() {
first_valid_fence = Some(candidate);
}
}
search_from = body_start + close_rel + 3;
}
// Last resort: scan every `{` position. A stray non-JSON brace like
// `{preview:true}` fails to parse and is skipped; a valid JSON object with
// a `"verdict"` key wins over an earlier non-verdict JSON fragment.
let mut first_valid_brace: Option<&str> = None;
for (brace_idx, _) in output.match_indices('{') {
let tail = &output[brace_idx..];
if let Some((end, value)) = json_prefix(tail) {
let candidate = tail[..end].trim();
if value_has_verdict(&value) {
return candidate;
}
if first_valid_brace.is_none() {
first_valid_brace = Some(candidate);
}
}
}
// No verdict object anywhere: preserve fence-before-brace precedence for
// non-verdict JSON fragments, then fall back to the trimmed prose.
first_valid_fence.or(first_valid_brace).unwrap_or(trimmed)
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq)]
struct ReviewerJsonOutput {
verdict: Verdict,
summary: String,
#[serde(default)]
findings: Vec<StructuredFinding>,
#[serde(default)]
next_steps: Vec<String>,
memory_skill: MemorySkillClassification,
}
impl ReviewerJsonOutput {
/// A PASS may only propose a how-to skill. A reviewer occasionally gets the
/// verdict right while attaching a REJECT-only proposal; retain the valid
/// verdict and discard that proposal rather than throwing away the review.
fn salvage_disallowed_pass_skill_proposal(&mut self) -> bool {
if self.verdict != Verdict::Pass
|| !matches!(
self.memory_skill.kind,
MemorySkillClassificationKind::AntiPatternSkill
| MemorySkillClassificationKind::RemediationSkill
)
{
return false;
}
self.memory_skill.kind = MemorySkillClassificationKind::None;
self.memory_skill.learning_source.clear();
true
}
fn validate(&self) -> Result<(), ReviewerError> {
if self.summary.trim().is_empty() {
return Err(ReviewerError::VerdictSchema {
message: "summary must not be empty".to_owned(),
});
}
self.memory_skill
.validate_for_verdict(self.verdict)
.map_err(|message| ReviewerError::VerdictSchema { message })?;
for finding in &self.findings {
if finding.title.trim().is_empty() {
return Err(ReviewerError::VerdictSchema {
message: "finding title must not be empty".to_owned(),
});
}
if finding.body.trim().is_empty() {
return Err(ReviewerError::VerdictSchema {
message: "finding body must not be empty".to_owned(),
});
}
if finding.file.trim().is_empty() {
return Err(ReviewerError::VerdictSchema {
message: "finding file must not be empty".to_owned(),
});
}
if finding.line_start == 0 || finding.line_end == 0 {
return Err(ReviewerError::VerdictSchema {
message: "finding lines must be one-based".to_owned(),
});
}
if finding.line_end < finding.line_start {
return Err(ReviewerError::VerdictSchema {
message: "finding line_end must be greater than or equal to line_start"
.to_owned(),
});
}
if finding.confidence > 100 {
return Err(ReviewerError::VerdictSchema {
message: "finding confidence must be between 0 and 100".to_owned(),
});
}
if finding.recommendation.trim().is_empty() {
return Err(ReviewerError::VerdictSchema {
message: "finding recommendation must not be empty".to_owned(),
});
}
}
if self.verdict == Verdict::Pass && !self.findings.is_empty() {
return Err(ReviewerError::VerdictSchema {
message: "PASS verdict must not include findings".to_owned(),
});
}
if self.verdict == Verdict::Reject && self.findings.is_empty() {
return Err(ReviewerError::VerdictSchema {
message: "REJECT verdict must include at least one finding".to_owned(),
});
}
if self.verdict == Verdict::Flag && self.findings.is_empty() {
return Err(ReviewerError::VerdictSchema {
message: "FLAG verdict must include at least one finding (the debt being surfaced, in the findings array)"
.to_owned(),
});
}
Ok(())
}
}
/// Parsed reviewer output held durably between provider return and ledger apply.
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
pub struct PendingReviewSpool {
pub entries: Vec<LedgerEntry>,
/// When true, a petition transition must still be applied after entries land.
#[serde(default)]
pub petition_transition_pending: bool,
/// First-pass stdout retained while the strict arbiter pass is still pending.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub strict_pass_stdout: Option<String>,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "kebab-case")]
pub enum ReviewRunStatus {
Queued,
Running,
Completed,
Failed,
Cancelled,
}
impl std::fmt::Display for ReviewRunStatus {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Queued => formatter.write_str("queued"),
Self::Running => formatter.write_str("running"),
Self::Completed => formatter.write_str("completed"),
Self::Failed => formatter.write_str("failed"),
Self::Cancelled => formatter.write_str("cancelled"),
}
}
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "kebab-case")]
pub enum BatchKind {
Petition,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
pub struct BatchRunRef {
pub batch_id: String,
pub kind: BatchKind,
pub member_index: u32,
pub member_count: u32,
pub fix_sha: String,
}
/// Immutable audit record for a batched petition review invocation.
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
pub struct BatchReviewRecord {
pub batch_id: String,
pub fix_sha: String,
pub phase: String,
pub member_run_ids: Vec<String>,
pub member_original_shas: Vec<String>,
pub reviewer_harness: String,
pub reviewer_model: String,
pub raw_stdout: Option<String>,
pub created_at_unix: u64,
pub updated_at_unix: u64,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
pub struct ReviewRun {
pub id: String,
pub commit_sha: String,
pub target: String,
pub status: ReviewRunStatus,
pub phase: String,
pub ledger_entries: usize,
pub error: Option<String>,
/// PID of the process executing this run while it is `Running`. Persisted so a
/// `review status`/`cancel` can tell a live worker from a silently-dead one and
/// reap the zombie. `None` once the run reaches a terminal state (or for legacy
/// records written before pid tracking existed).
#[serde(default)]
pub worker_pid: Option<u32>,
/// OS-reported start marker of the worker process, recorded alongside the pid
/// so a REUSED pid cannot make a dead worker look alive (pid-recycling
/// defense). `None` for legacy rows or when the platform reports no marker.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub worker_start_token: Option<String>,
pub created_at_unix: u64,
pub updated_at_unix: u64,
pub started_at_unix: Option<u64>,
pub completed_at_unix: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub entire_checkpoint: Option<EntireCheckpointRef>,
/// When set, this run reviews a petition fix for the named original rejection.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub petition_for: Option<String>,
/// Phase for which ledger application (review verdicts and petition
/// transitions) is durably recorded. Enables replay-idempotent drain.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub ledger_applied_phase: Option<String>,
/// Provider output parsed but not yet fully applied to the ledger. Cleared
/// only after verdict entries and any petition transition are durable.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub pending_review: Option<PendingReviewSpool>,
/// Scheduler correlation for a batched petition review.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub batch: Option<BatchRunRef>,
/// Unix time when the watcher scheduler durably claimed this run for drain.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub claimed_at_unix: Option<u64>,
/// Earliest unix time a retryable provider failure may be attempted again.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub next_attempt_at_unix: Option<u64>,
/// Detached worktree path while claimed/inflight (maintenance cleanup hint).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub worktree_path: Option<String>,
}
#[derive(Clone, Debug, Default, Eq, PartialEq)]
pub struct ReviewRunStatusCounts {
pub queued: usize,
pub running: usize,
pub completed: usize,
pub failed: usize,
pub cancelled: usize,
pub skipped_records: usize,
}
/// Read-only scheduler observability for `truth-mirror status`.
#[derive(Clone, Debug, Default, Eq, PartialEq)]
pub struct SchedulerStatusSnapshot {
pub owner_pid: Option<u32>,
pub claimed: usize,
pub inflight: usize,
pub active_workers: usize,
pub batches: usize,
}
impl ReviewRunStatusCounts {
fn add(&mut self, status: ReviewRunStatus) {
match status {
ReviewRunStatus::Queued => self.queued += 1,
ReviewRunStatus::Running => self.running += 1,
ReviewRunStatus::Completed => self.completed += 1,
ReviewRunStatus::Failed => self.failed += 1,
ReviewRunStatus::Cancelled => self.cancelled += 1,
}
}
}
#[derive(Deserialize)]
struct ReviewRunStatusRecord {
status: ReviewRunStatus,
}
impl ReviewRun {
#[cfg(test)]
fn queued(
id: impl Into<String>,
commit_sha: impl Into<String>,
target: impl Into<String>,
) -> Self {
Self::queued_with_provenance(id, commit_sha, target, None, None)
}
fn queued_with_provenance(
id: impl Into<String>,
commit_sha: impl Into<String>,
target: impl Into<String>,
entire_checkpoint: Option<EntireCheckpointRef>,
petition_for: Option<String>,
) -> Self {
let timestamp = unix_now();
Self {
id: id.into(),
commit_sha: commit_sha.into(),
target: target.into(),
status: ReviewRunStatus::Queued,
phase: "queued".to_owned(),
ledger_entries: 0,
error: None,
worker_pid: None,
worker_start_token: None,
created_at_unix: timestamp,
updated_at_unix: timestamp,
started_at_unix: None,
completed_at_unix: None,
entire_checkpoint,
petition_for,
ledger_applied_phase: None,
pending_review: None,
batch: None,
claimed_at_unix: None,
next_attempt_at_unix: None,
worktree_path: None,
}
}
fn clear_pending_review(&mut self) {
self.pending_review = None;
self.updated_at_unix = unix_now();
}
fn mark_running(&mut self, phase: impl Into<String>) {
let timestamp = unix_now();
let identity = crate::watcher::ProcessIdentity::current();
self.status = ReviewRunStatus::Running;
self.phase = phase.into();
self.error = None;
self.worker_pid = Some(identity.pid);
self.worker_start_token =
(!identity.start_token.is_empty()).then_some(identity.start_token);
self.updated_at_unix = timestamp;
if self.started_at_unix.is_none() {
self.started_at_unix = Some(timestamp);
}
self.completed_at_unix = None;
// Inflight execution clears a prior backoff window.
self.next_attempt_at_unix = None;
}
/// Durably claim this run under the scheduler before a worker is spawned.
fn mark_claimed(&mut self) {
let timestamp = unix_now();
self.status = ReviewRunStatus::Running;
self.phase = CLAIMED_PHASE.to_owned();
self.error = None;
self.worker_pid = Some(std::process::id());
self.claimed_at_unix = Some(timestamp);
self.next_attempt_at_unix = None;
self.updated_at_unix = timestamp;
self.started_at_unix = Some(timestamp);
self.completed_at_unix = None;
}
/// Persist provider/rate-limit backoff without a terminal failure or queue removal.
fn mark_provider_backoff(&mut self, error: impl Into<String>, backoff_secs: u64) {
let timestamp = unix_now();
self.status = ReviewRunStatus::Queued;
self.phase = PROVIDER_BACKOFF_PHASE.to_owned();
self.error = Some(error.into());
self.worker_pid = None;
self.claimed_at_unix = None;
self.worktree_path = None;
self.next_attempt_at_unix = Some(timestamp.saturating_add(backoff_secs));
self.updated_at_unix = timestamp;
self.completed_at_unix = None;
}
/// Reset a crash-abandoned claim so the queue row can be drained again.
fn release_stale_claim(&mut self, reason: impl Into<String>) {
let timestamp = unix_now();
self.status = ReviewRunStatus::Queued;
self.phase = "queued".to_owned();
self.error = Some(reason.into());
self.worker_pid = None;
self.claimed_at_unix = None;
self.worktree_path = None;
self.updated_at_unix = timestamp;
self.completed_at_unix = None;
}
fn is_claimed(&self) -> bool {
self.status == ReviewRunStatus::Running && self.claimed_at_unix.is_some()
}
fn is_inflight(&self) -> bool {
self.status == ReviewRunStatus::Running
&& self.claimed_at_unix.is_some()
&& self.phase != CLAIMED_PHASE
}
fn backoff_blocks_attempt(&self, now: u64) -> bool {
self.next_attempt_at_unix
.map(|eligible| eligible > now)
.unwrap_or(false)
}
fn mark_ledger_applied(&mut self, phase: impl Into<String>, ledger_entries: usize) {
let timestamp = unix_now();
self.ledger_applied_phase = Some(phase.into());
self.ledger_entries = ledger_entries;
self.updated_at_unix = timestamp;
}
fn mark_completed(&mut self, ledger_entries: usize) {
let timestamp = unix_now();
self.status = ReviewRunStatus::Completed;
self.phase = "completed".to_owned();
self.ledger_entries = ledger_entries;
self.error = None;
self.worker_pid = None;
self.worker_start_token = None;
self.claimed_at_unix = None;
self.next_attempt_at_unix = None;
self.worktree_path = None;
self.updated_at_unix = timestamp;
self.completed_at_unix = Some(timestamp);
}
fn mark_failed(&mut self, error: impl Into<String>) {
let timestamp = unix_now();
self.status = ReviewRunStatus::Failed;
self.phase = "failed".to_owned();
self.error = Some(error.into());
self.worker_pid = None;
self.worker_start_token = None;
self.claimed_at_unix = None;
self.next_attempt_at_unix = None;
self.worktree_path = None;
self.updated_at_unix = timestamp;
self.completed_at_unix = Some(timestamp);
}
fn mark_cancelled(&mut self) {
let timestamp = unix_now();
self.status = ReviewRunStatus::Cancelled;
self.phase = "cancelled".to_owned();
self.error = None;
self.worker_pid = None;
self.worker_start_token = None;
self.claimed_at_unix = None;
self.next_attempt_at_unix = None;
self.worktree_path = None;
self.updated_at_unix = timestamp;
self.completed_at_unix = Some(timestamp);
}
/// Reconcile a `Running` run against worker liveness. If its recorded worker
/// identity is proven dead — pid gone, or pid reused under a different start
/// token — the worker died without recording a verdict: flip the zombie to
/// `Failed` with an explanatory reason.
///
/// The probe is TRI-STATE: [`crate::watcher::WorkerLiveness::Unknown`]
/// (the OS probe itself failed) is never treated as proof of death — the
/// row is left alone and [`ReconcileOutcome::ProbeUnknown`] is returned so
/// the caller can log the inconclusive probe. A run with no recorded pid
/// (legacy record) is likewise left untouched: only an explicit forced
/// cancel reaps it.
fn reconcile_liveness(
&mut self,
probe: impl Fn(&crate::watcher::ProcessIdentity) -> crate::watcher::WorkerLiveness,
) -> ReconcileOutcome {
if self.status != ReviewRunStatus::Running {
return ReconcileOutcome::Unchanged;
}
let Some(pid) = self.worker_pid else {
return ReconcileOutcome::Unchanged;
};
let identity = crate::watcher::ProcessIdentity {
pid,
start_token: self.worker_start_token.clone().unwrap_or_default(),
};
match probe(&identity) {
crate::watcher::WorkerLiveness::Dead => {
self.mark_failed(stale_worker_reason(pid));
ReconcileOutcome::Reaped
}
crate::watcher::WorkerLiveness::Alive => ReconcileOutcome::Unchanged,
crate::watcher::WorkerLiveness::Unknown => ReconcileOutcome::ProbeUnknown,
}
}
}
/// Outcome of reconciling one run against worker liveness.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum ReconcileOutcome {
/// The worker is proven dead; the run was flipped to `Failed`.
Reaped,
/// Nothing to do: not running, no recorded pid, or the worker is alive.
Unchanged,
/// The liveness probe could not prove the worker alive or dead. The row
/// was deliberately left alone — a probe failure is NOT proof of death.
ProbeUnknown,
}
/// Reason recorded when a running review is found abandoned by a dead worker.
fn stale_worker_reason(pid: u32) -> String {
format!("worker process {pid} exited without recording a verdict (stale run)")
}
/// Force-terminate a process with SIGKILL. Used by `review cancel --force` on a
/// worker that is still alive.
fn kill_pid(pid: u32) -> Result<(), ReviewerError> {
let status = Command::new("kill")
.arg("-KILL")
.arg(pid.to_string())
.stdout(Stdio::null())
.stderr(Stdio::null())
.status()
.map_err(ReviewerError::KillWorker)?;
if status.success() || !crate::watcher::pid_is_alive(pid) {
Ok(())
} else {
Err(ReviewerError::KillWorkerFailed { pid })
}
}
#[derive(Clone, Debug)]
pub struct ReviewRunStore {
root: PathBuf,
}
/// Directory name of the run-store mutation mutex inside the state dir.
const RUN_STORE_LOCK_DIR: &str = "review-runs.lock";
const RUN_STORE_LOCK_OWNER_FILE: &str = "review-runs.lock.owner";
const RUN_STORE_LOCK_HEARTBEAT_FILE: &str = "review-runs.lock.heartbeat";
const RUN_STORE_LOCK_LEASE_PREFIX: &str = ".review-runs.lock.lease";
/// Fixed name for the reclaim gate's backing file. Created once and never
/// deleted — see `RunStoreReclaimGate`'s doc comment for why.
const RUN_STORE_LOCK_RECLAIM_FILE: &str = ".review-runs.lock.reclaim";
/// The heartbeat is refreshed while the guard is held. A lock is stale only
/// after this many missed refresh intervals AND a process probe proves that
/// the recorded owner is dead. A fresh heartbeat never overrides a live
/// identity probe, and an Unknown probe is never reclaimed automatically.
const RUN_STORE_LOCK_HEARTBEAT_INTERVAL: Duration = Duration::from_millis(100);
const RUN_STORE_LOCK_STALE_INTERVALS: u64 = 3;
/// Total time to wait for a live contender before failing the mutation.
/// Long enough that ordinary contention (sub-millisecond critical sections)
/// never trips it, short enough to surface a genuine jam.
const RUN_STORE_LOCK_WAIT: Duration = Duration::from_secs(10);
const RUN_STORE_LOCK_POLL: Duration = Duration::from_millis(25);
static RUN_STORE_LOCK_TOKEN_COUNTER: AtomicU64 = AtomicU64::new(0);
#[derive(Clone, Debug, Deserialize, Serialize)]
struct RunStoreLockRecord {
owner: crate::watcher::ProcessIdentity,
acquisition_token: String,
}
/// Mutual-exclusion guard for run-store mutations. Implemented as an atomic
/// `create_new` + hard-link pair. The fixed lock path and the guard's unique
/// lease path are separate directory entries for the same inode, so releasing
/// an old guard can never unlink a replacement lease's unique path. Dropping
/// the guard releases the lock.
struct RunStoreLock {
path: PathBuf,
lease_path: PathBuf,
owner: crate::watcher::ProcessIdentity,
acquisition_token: String,
heartbeat_stop: Arc<AtomicBool>,
heartbeat: Option<std::thread::JoinHandle<()>>,
}
impl Drop for RunStoreLock {
fn drop(&mut self) {
self.heartbeat_stop.store(true, Ordering::Release);
if let Some(heartbeat) = self.heartbeat.take() {
let _ = heartbeat.join();
}
release_run_store_lock(
&self.path,
&self.lease_path,
&self.owner,
&self.acquisition_token,
);
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum RunStoreLockLiveness {
/// Recorded owner is still proven alive.
Alive,
/// Recorded owner is dead or replaced (pid reuse mismatch).
Dead,
/// Probe could not prove status (for example start token not readable).
/// We treat this as held and do not reclaim on this signal alone.
Unknown,
}
fn run_store_lock_owner_path(path: &Path) -> PathBuf {
path.join(RUN_STORE_LOCK_OWNER_FILE)
}
fn run_store_lock_heartbeat_path(path: &Path) -> PathBuf {
path.join(RUN_STORE_LOCK_HEARTBEAT_FILE)
}
fn run_store_lock_is_owned_by_current_process(
path: &Path,
owner: &crate::watcher::ProcessIdentity,
acquisition_token: &str,
) -> bool {
let Ok(record) = read_run_store_lock_record(path) else {
return false;
};
identity_matches_owner(&record.owner, owner) && record.acquisition_token == acquisition_token
}
fn identity_matches_owner(
recorded: &crate::watcher::ProcessIdentity,
current: &crate::watcher::ProcessIdentity,
) -> bool {
recorded.pid == current.pid
&& (recorded.start_token.is_empty()
|| current.start_token.is_empty()
|| recorded.start_token == current.start_token)
}
fn decide_run_store_lock_liveness(
recorded: &crate::watcher::ProcessIdentity,
) -> RunStoreLockLiveness {
match crate::watcher::probe_worker_identity(recorded) {
crate::watcher::WorkerLiveness::Alive => RunStoreLockLiveness::Alive,
crate::watcher::WorkerLiveness::Dead => RunStoreLockLiveness::Dead,
crate::watcher::WorkerLiveness::Unknown => RunStoreLockLiveness::Unknown,
}
}
fn lock_is_stale_by_heartbeat(path: &Path) -> bool {
let heartbeat_path = if path.is_dir() {
run_store_lock_heartbeat_path(path)
} else {
path.to_path_buf()
};
let Ok(metadata) = fs::metadata(heartbeat_path) else {
return false;
};
let Ok(modified) = metadata.modified() else {
return false;
};
match SystemTime::now().duration_since(modified) {
Ok(age) => age >= RUN_STORE_LOCK_HEARTBEAT_INTERVAL * RUN_STORE_LOCK_STALE_INTERVALS as u32,
Err(_) => false,
}
}
fn read_run_store_lock_record(path: &Path) -> Result<RunStoreLockRecord, io::Error> {
let owner_path = if path.is_dir() {
run_store_lock_owner_path(path)
} else {
path.to_path_buf()
};
let contents = fs::read_to_string(owner_path)?;
if path.is_dir() {
let owner = serde_json::from_str::<crate::watcher::ProcessIdentity>(&contents)
.map_err(io::Error::other)?;
Ok(RunStoreLockRecord {
owner,
acquisition_token: String::new(),
})
} else {
serde_json::from_str::<RunStoreLockRecord>(&contents).map_err(io::Error::other)
}
}
#[cfg(test)]
fn write_run_store_lock_owner(
lock_dir: &Path,
owner: &crate::watcher::ProcessIdentity,
) -> Result<(), ReviewerError> {
fs::create_dir_all(lock_dir).map_err(ReviewerError::RunIo)?;
let bytes = serde_json::to_vec_pretty(owner).map_err(ReviewerError::RunJson)?;
fs::write(run_store_lock_owner_path(lock_dir), bytes).map_err(ReviewerError::RunIo)?;
fs::write(run_store_lock_heartbeat_path(lock_dir), b"heartbeat")
.map_err(ReviewerError::RunIo)?;
Ok(())
}
/// Whether the lock is considered stale by owner identity and heartbeat evidence.
///
/// A persistent Unknown state is deliberately fail-closed: it is treated as
/// held until the contender's normal wait expires, which returns a timeout and
/// tells the operator to inspect the lock. There is no automatic "old enough"
/// escape hatch because that would turn an unprobeable live owner into a data
/// loss race.
fn run_store_lock_is_stale(path: &Path) -> bool {
let Ok(record) = read_run_store_lock_record(path) else {
return false;
};
matches!(
decide_run_store_lock_liveness(&record.owner),
RunStoreLockLiveness::Dead
) && lock_is_stale_by_heartbeat(path)
}
fn new_run_store_lock_token() -> String {
let nanos = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_or(0, |duration| duration.as_nanos());
let counter = RUN_STORE_LOCK_TOKEN_COUNTER.fetch_add(1, Ordering::Relaxed);
format!("{}-{nanos}-{counter}", std::process::id())
}
fn write_run_store_lock_record(
lease_path: &Path,
record: &RunStoreLockRecord,
) -> Result<(), ReviewerError> {
let bytes = serde_json::to_vec_pretty(record).map_err(ReviewerError::RunJson)?;
let mut file = fs::OpenOptions::new()
.write(true)
.truncate(true)
.open(lease_path)
.map_err(ReviewerError::RunIo)?;
file.write_all(&bytes).map_err(ReviewerError::RunIo)?;
file.write_all(b"\n").map_err(ReviewerError::RunIo)?;
file.sync_all().map_err(ReviewerError::RunIo)?;
Ok(())
}
fn refresh_run_store_lock_heartbeat(lease_path: &Path) -> io::Result<()> {
let mut file = fs::OpenOptions::new().append(true).open(lease_path)?;
// Whitespace keeps the JSON owner record parseable while updating the
// hard-linked inode's mtime. The unique lease path means this can never
// refresh a replacement lock after the fixed link changes.
file.write_all(b"\n")?;
file.sync_data()
}
fn spawn_run_store_lock_heartbeat(
lease_path: PathBuf,
stop: Arc<AtomicBool>,
) -> std::thread::JoinHandle<()> {
std::thread::spawn(move || {
while !stop.load(Ordering::Acquire) {
std::thread::sleep(RUN_STORE_LOCK_HEARTBEAT_INTERVAL);
if stop.load(Ordering::Acquire) {
break;
}
if let Err(error) = refresh_run_store_lock_heartbeat(&lease_path) {
tracing::debug!(%error, "run-store lock heartbeat refresh stopped");
break;
}
}
})
}
/// Single-winner mutex serializing stale run-store lock reclamation.
///
/// Round 1 backed this with a `create_new` marker file plus an mtime-based
/// "stale after N seconds" heuristic — which repeats the EXACT TOCTOU it was
/// built to close: two contenders can both read the marker, both decide it is
/// stale, and both proceed to unlink it; the first removal clears the way for
/// its own fresh marker, but the second contender's already-decided
/// `remove_file` then deletes that freshly-installed one, letting both
/// contenders through (CodeRabbit/Gemini/Codex round-2 CRITICAL — three bots
/// independently caught this). A real OS advisory lock (`flock` via
/// `std::fs::File::try_lock`) has no such problem: the kernel releases it
/// automatically when the holding process's file descriptor closes — crash
/// included — so there is nothing to detect as stale and nothing to unlink.
/// The backing file is created once and never deleted: unlinking a `flock`ed
/// file while another process still holds it open would let a new caller
/// open-and-lock a fresh inode at the same path, silently reintroducing a
/// second holder.
struct RunStoreReclaimGate {
// Held only for its RAII effect: dropping it closes the fd, which
// releases the flock. Never read directly.
#[allow(dead_code)]
file: fs::File,
}
fn acquire_run_store_reclaim_gate(root: &Path) -> io::Result<Option<RunStoreReclaimGate>> {
let path = root.join(RUN_STORE_LOCK_RECLAIM_FILE);
let file = fs::OpenOptions::new()
.write(true)
.create(true)
.truncate(false)
.open(&path)?;
match file.try_lock() {
Ok(()) => Ok(Some(RunStoreReclaimGate { file })),
Err(fs::TryLockError::WouldBlock) => Ok(None),
Err(fs::TryLockError::Error(error)) => Err(error),
}
}
fn release_run_store_lock(
path: &Path,
lease_path: &Path,
owner: &crate::watcher::ProcessIdentity,
acquisition_token: &str,
) {
if run_store_lock_is_owned_by_current_process(path, owner, acquisition_token) {
let _ = fs::remove_file(path);
}
// This is our private acquisition path, never the replacement's path.
let _ = fs::remove_file(lease_path);
}
impl ReviewRunStore {
pub fn new(root: impl Into<PathBuf>) -> Self {
Self { root: root.into() }
}
pub fn runs_dir(&self) -> PathBuf {
self.root.join(REVIEW_RUNS_DIR)
}
pub fn path(&self, id: &str) -> PathBuf {
self.runs_dir().join(format!("{id}.json"))
}
/// Take the run-store mutation lock, reclaiming it from a crashed holder.
///
/// EVERY read-modify-write on a run row — worker transitions AND stale-row
/// reconciliation — happens under this lock, so a reconciler can never
/// overwrite a completion that landed between its snapshot read and its
/// write (0.9.2 clobbered `completed` rows back to `failed`).
fn lock(&self) -> Result<RunStoreLock, ReviewerError> {
fs::create_dir_all(&self.root).map_err(ReviewerError::RunIo)?;
let path = self.root.join(RUN_STORE_LOCK_DIR);
let owner = crate::watcher::ProcessIdentity::current();
let started = Instant::now();
loop {
// Handle the legacy directory-shaped lease before attempting the
// file hard-link acquisition. This also keeps a concurrent stale
// reclaim from being mistaken for an unsupported hard-link target.
if path.exists() {
if run_store_lock_is_stale(&path) {
// Serialize the reclaim: only the gate's winner revalidates
// and unlinks, so a second contender that also observed the
// stale entry can never delete a lock the winner just
// installed.
match acquire_run_store_reclaim_gate(&self.root) {
Ok(Some(_gate)) => {
// Revalidate under the gate — nothing else can be
// mid-reclaim right now, so a fresh read is the
// true state, not a stale snapshot from before the
// race.
if run_store_lock_is_stale(&path) {
let remove_result = if path.is_dir() {
fs::remove_dir_all(&path)
} else {
fs::remove_file(&path)
};
if let Err(error) = remove_result
&& error.kind() != io::ErrorKind::NotFound
{
return Err(ReviewerError::RunIo(error));
}
}
// Gate drops here, releasing the reclaim mutex.
}
Ok(None) => {
// Another contender is reclaiming (or just did);
// back off and reassess next iteration.
}
Err(error) => return Err(ReviewerError::RunIo(error)),
}
std::thread::sleep(RUN_STORE_LOCK_POLL);
continue;
}
if started.elapsed() >= RUN_STORE_LOCK_WAIT {
if let Ok(record) = read_run_store_lock_record(&path)
&& matches!(
decide_run_store_lock_liveness(&record.owner),
RunStoreLockLiveness::Unknown
)
{
tracing::warn!(
"run-store lock owner remained Unknown; refusing automatic reclaim"
);
}
return Err(ReviewerError::RunStoreLockTimeout {
wait_secs: RUN_STORE_LOCK_WAIT.as_secs(),
});
}
std::thread::sleep(RUN_STORE_LOCK_POLL);
continue;
}
let acquisition_token = new_run_store_lock_token();
let lease_path = self
.root
.join(format!("{RUN_STORE_LOCK_LEASE_PREFIX}.{acquisition_token}"));
let record = RunStoreLockRecord {
owner: owner.clone(),
acquisition_token: acquisition_token.clone(),
};
match fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(&lease_path)
{
Ok(_) => {
if let Err(error) = write_run_store_lock_record(&lease_path, &record) {
let _ = fs::remove_file(&lease_path);
return Err(error);
}
match fs::hard_link(&lease_path, &path) {
Ok(()) => {
let heartbeat_stop = Arc::new(AtomicBool::new(false));
let heartbeat = spawn_run_store_lock_heartbeat(
lease_path.clone(),
Arc::clone(&heartbeat_stop),
);
return Ok(RunStoreLock {
path: path.clone(),
lease_path,
owner,
acquisition_token,
heartbeat_stop,
heartbeat: Some(heartbeat),
});
}
Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {
let _ = fs::remove_file(&lease_path);
}
Err(error) => {
let _ = fs::remove_file(&lease_path);
return Err(ReviewerError::RunIo(error));
}
}
}
Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue,
Err(error) => return Err(ReviewerError::RunIo(error)),
}
if started.elapsed() >= RUN_STORE_LOCK_WAIT {
if let Ok(record) = read_run_store_lock_record(&path)
&& matches!(
decide_run_store_lock_liveness(&record.owner),
RunStoreLockLiveness::Unknown
)
{
tracing::warn!(
"run-store lock owner remained Unknown; refusing automatic reclaim"
);
}
return Err(ReviewerError::RunStoreLockTimeout {
wait_secs: RUN_STORE_LOCK_WAIT.as_secs(),
});
}
std::thread::sleep(RUN_STORE_LOCK_POLL);
}
}
pub fn create_queued(
&self,
commit_sha: &str,
target: impl Into<String>,
) -> Result<ReviewRun, ReviewerError> {
let _guard = self.lock()?;
let checkpoint = entire_checkpoint_for_current_repo(commit_sha);
let transaction = StateTransaction::acquire(&self.root)?;
self.create_queued_locked(&transaction, commit_sha, target, checkpoint, None)
}
fn create_queued_locked(
&self,
transaction: &StateTransaction,
commit_sha: &str,
target: impl Into<String>,
checkpoint: Option<EntireCheckpointRef>,
petition_for: Option<String>,
) -> Result<ReviewRun, ReviewerError> {
let run = ReviewRun::queued_with_provenance(
generate_run_id(commit_sha),
commit_sha,
target,
checkpoint,
petition_for,
);
self.write_locked(transaction, &run)?;
Ok(run)
}
fn ensure_queued(
&self,
run_id: &str,
commit_sha: &str,
target: &str,
petition_for: Option<String>,
) -> Result<ReviewRun, ReviewerError> {
match self.read(run_id) {
Ok(run) => return Ok(run),
Err(ReviewerError::ReviewRunNotFound { .. }) => {}
Err(error) => return Err(error),
}
let checkpoint = entire_checkpoint_for_current_repo(commit_sha);
let transaction = StateTransaction::acquire(&self.root)?;
self.ensure_queued_locked(
&transaction,
run_id,
commit_sha,
target,
checkpoint,
petition_for,
)
}
fn ensure_queued_locked(
&self,
transaction: &StateTransaction,
run_id: &str,
commit_sha: &str,
target: &str,
checkpoint: Option<EntireCheckpointRef>,
petition_for: Option<String>,
) -> Result<ReviewRun, ReviewerError> {
let _guard = self.lock()?;
match self.read(run_id) {
Ok(run) => Ok(run),
Err(ReviewerError::ReviewRunNotFound { .. }) => {
let run = ReviewRun::queued_with_provenance(
run_id,
commit_sha,
target,
checkpoint,
petition_for,
);
self.write_locked(transaction, &run)?;
Ok(run)
}
Err(error) => Err(error),
}
}
pub fn read(&self, id: &str) -> Result<ReviewRun, ReviewerError> {
let path = self.path(id);
let contents = fs::read_to_string(&path).map_err(|source| match source.kind() {
io::ErrorKind::NotFound => ReviewerError::ReviewRunNotFound { id: id.to_owned() },
_ => ReviewerError::RunIo(source),
})?;
serde_json::from_str(&contents).map_err(ReviewerError::RunJson)
}
pub fn list(&self) -> Result<Vec<ReviewRun>, ReviewerError> {
let dir = self.runs_dir();
let entries = match fs::read_dir(&dir) {
Ok(entries) => entries,
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(Vec::new()),
Err(error) => return Err(ReviewerError::RunIo(error)),
};
let mut runs: Vec<ReviewRun> = Vec::new();
for entry in entries {
let entry = entry.map_err(ReviewerError::RunIo)?;
if entry
.path()
.extension()
.is_none_or(|extension| extension != "json")
{
continue;
}
let contents = fs::read_to_string(entry.path()).map_err(ReviewerError::RunIo)?;
runs.push(serde_json::from_str(&contents).map_err(ReviewerError::RunJson)?);
}
runs.sort_by(|left, right| {
right
.updated_at_unix
.cmp(&left.updated_at_unix)
.then_with(|| right.id.cmp(&left.id))
});
Ok(runs)
}
/// Like [`Self::list`], but skips malformed run files instead of failing the
/// whole scan. Recovery and status paths use this so one bad record cannot
/// block unrelated queue work.
fn list_lenient(&self) -> Result<(Vec<ReviewRun>, usize), ReviewerError> {
let dir = self.runs_dir();
let entries = match fs::read_dir(&dir) {
Ok(entries) => entries,
// Missing OR a non-directory placeholder (tests inject a file here to
// force later create_queued failures): recovery treats both as "no runs".
Err(error)
if error.kind() == io::ErrorKind::NotFound
|| error.kind() == io::ErrorKind::NotADirectory =>
{
return Ok((Vec::new(), 0));
}
Err(error) => return Err(ReviewerError::RunIo(error)),
};
let mut runs: Vec<ReviewRun> = Vec::new();
let mut skipped_records = 0usize;
for entry in entries {
let Ok(entry) = entry else {
skipped_records += 1;
continue;
};
let path = entry.path();
if path.extension().is_none_or(|extension| extension != "json") {
continue;
}
let Ok(contents) = fs::read_to_string(&path) else {
skipped_records += 1;
continue;
};
let Ok(run) = serde_json::from_str::<ReviewRun>(&contents) else {
skipped_records += 1;
continue;
};
runs.push(run);
}
runs.sort_by(|left, right| {
right
.updated_at_unix
.cmp(&left.updated_at_unix)
.then_with(|| right.id.cmp(&left.id))
});
Ok((runs, skipped_records))
}
pub fn status_counts(&self) -> Result<ReviewRunStatusCounts, ReviewerError> {
let dir = self.runs_dir();
let entries = match fs::read_dir(&dir) {
Ok(entries) => entries,
Err(error) if error.kind() == io::ErrorKind::NotFound => {
return Ok(ReviewRunStatusCounts::default());
}
Err(error) => return Err(ReviewerError::RunIo(error)),
};
let mut counts = ReviewRunStatusCounts::default();
for entry in entries {
let Ok(entry) = entry else {
counts.skipped_records += 1;
continue;
};
let path = entry.path();
if path.extension().is_none_or(|extension| extension != "json") {
continue;
}
let Ok(contents) = fs::read_to_string(&path) else {
counts.skipped_records += 1;
continue;
};
let Ok(record) = serde_json::from_str::<ReviewRunStatusRecord>(&contents) else {
counts.skipped_records += 1;
continue;
};
counts.add(record.status);
}
Ok(counts)
}
pub fn latest_result(&self) -> Result<ReviewRun, ReviewerError> {
self.list()?
.into_iter()
.find(|run| {
matches!(
run.status,
ReviewRunStatus::Completed
| ReviewRunStatus::Failed
| ReviewRunStatus::Cancelled
)
})
.ok_or(ReviewerError::NoReviewRuns)
}
pub fn mark_running(&self, id: &str, phase: &str) -> Result<ReviewRun, ReviewerError> {
let transaction = StateTransaction::acquire(&self.root)?;
let mut run = self.read(id)?;
run.mark_running(phase);
self.write_locked(&transaction, &run)?;
Ok(run)
}
pub fn mark_claimed(&self, id: &str) -> Result<ReviewRun, ReviewerError> {
let transaction = StateTransaction::acquire(&self.root)?;
let mut run = self.read(id)?;
run.mark_claimed();
self.write_locked(&transaction, &run)?;
Ok(run)
}
pub fn mark_provider_backoff(
&self,
id: &str,
error: impl Into<String>,
backoff_secs: u64,
) -> Result<ReviewRun, ReviewerError> {
let transaction = StateTransaction::acquire(&self.root)?;
let mut run = self.read(id)?;
run.mark_provider_backoff(error, backoff_secs);
self.write_locked(&transaction, &run)?;
Ok(run)
}
pub fn set_worktree_path(
&self,
id: &str,
worktree_path: Option<String>,
) -> Result<ReviewRun, ReviewerError> {
let transaction = StateTransaction::acquire(&self.root)?;
let mut run = self.read(id)?;
run.worktree_path = worktree_path;
run.updated_at_unix = unix_now();
self.write_locked(&transaction, &run)?;
Ok(run)
}
/// Observe claimed/inflight/active/batch counts without reconciling or mutating.
pub fn scheduler_status_read_only(
&self,
owner_pid: Option<u32>,
queue_batch_ids: impl IntoIterator<Item = String>,
) -> Result<SchedulerStatusSnapshot, ReviewerError> {
let (runs, _) = self.list_lenient()?;
let mut claimed = 0usize;
let mut inflight = 0usize;
let mut worker_pids = std::collections::BTreeSet::new();
let mut batches = std::collections::BTreeSet::new();
for batch_id in queue_batch_ids {
if !batch_id.trim().is_empty() {
batches.insert(batch_id);
}
}
for run in &runs {
if run.is_claimed() {
claimed += 1;
}
if run.is_inflight() {
inflight += 1;
}
if run.status == ReviewRunStatus::Running
&& let Some(pid) = run.worker_pid
&& crate::watcher::pid_is_alive(pid)
{
worker_pids.insert(pid);
}
if let Some(batch) = &run.batch {
batches.insert(batch.batch_id.clone());
}
}
Ok(SchedulerStatusSnapshot {
owner_pid,
claimed,
inflight,
active_workers: worker_pids.len(),
batches: batches.len(),
})
}
/// Crash recovery: release claimed/inflight runs whose worker died before ledger apply.
pub fn recover_stale_scheduler_claims(&self) -> Result<usize, ReviewerError> {
let transaction = StateTransaction::acquire(&self.root)?;
self.recover_stale_scheduler_claims_locked(&transaction)
}
fn recover_stale_scheduler_claims_locked(
&self,
transaction: &StateTransaction,
) -> Result<usize, ReviewerError> {
let (runs, _) = self.list_lenient()?;
let mut recovered = 0usize;
for run in runs {
if run.status != ReviewRunStatus::Running {
continue;
}
if run_ledger_phase_applied(&run, REVIEW_LEDGER_PHASE) {
continue;
}
let Some(pid) = run.worker_pid else {
continue;
};
if crate::watcher::pid_is_alive(pid) {
continue;
}
let mut released = run;
released.release_stale_claim(format!(
"scheduler claim released after worker {pid} exited before durable apply"
));
self.write_locked(transaction, &released)?;
recovered += 1;
}
Ok(recovered)
}
pub fn mark_ledger_applied(
&self,
id: &str,
phase: &str,
ledger_entries: usize,
) -> Result<ReviewRun, ReviewerError> {
let transaction = StateTransaction::acquire(&self.root)?;
let mut run = self.read(id)?;
run.mark_ledger_applied(phase, ledger_entries);
self.write_locked(&transaction, &run)?;
Ok(run)
}
pub fn persist_pending_review(
&self,
id: &str,
spool: PendingReviewSpool,
) -> Result<ReviewRun, ReviewerError> {
let transaction = StateTransaction::acquire(&self.root)?;
let mut run = self.read(id)?;
run.pending_review = Some(spool);
run.updated_at_unix = unix_now();
self.write_locked(&transaction, &run)?;
Ok(run)
}
pub fn clear_pending_review(&self, id: &str) -> Result<ReviewRun, ReviewerError> {
let transaction = StateTransaction::acquire(&self.root)?;
let mut run = self.read(id)?;
run.clear_pending_review();
self.write_locked(&transaction, &run)?;
Ok(run)
}
pub fn mark_completed(
&self,
id: &str,
ledger_entries: usize,
) -> Result<ReviewRun, ReviewerError> {
let transaction = StateTransaction::acquire(&self.root)?;
let mut run = self.read(id)?;
run.mark_completed(ledger_entries);
self.write_locked(&transaction, &run)?;
Ok(run)
}
pub fn mark_failed(
&self,
id: &str,
error: impl Into<String>,
) -> Result<ReviewRun, ReviewerError> {
let transaction = StateTransaction::acquire(&self.root)?;
self.mark_failed_locked(&transaction, id, error)
}
fn mark_failed_locked(
&self,
transaction: &StateTransaction,
id: &str,
error: impl Into<String>,
) -> Result<ReviewRun, ReviewerError> {
let mut run = self.read(id)?;
run.mark_failed(error);
self.write_locked(transaction, &run)?;
Ok(run)
}
pub fn set_batch_metadata(
&self,
id: &str,
batch: BatchRunRef,
) -> Result<ReviewRun, ReviewerError> {
let transaction = StateTransaction::acquire(&self.root)?;
let mut run = self.read(id)?;
run.batch = Some(batch);
self.write_locked(&transaction, &run)?;
Ok(run)
}
pub fn cancel_queued(&self, id: &str) -> Result<ReviewRun, ReviewerError> {
let transaction = StateTransaction::acquire(&self.root)?;
self.cancel_queued_locked(&transaction, id)
}
fn cancel_queued_locked(
&self,
transaction: &StateTransaction,
id: &str,
) -> Result<ReviewRun, ReviewerError> {
let mut run = self.read(id)?;
if run.status != ReviewRunStatus::Queued {
return Err(ReviewerError::CannotCancelReview {
id: id.to_owned(),
status: run.status,
});
}
run.mark_cancelled();
self.write_locked(transaction, &run)?;
Ok(run)
}
/// Read a run and, if it is a `Running` zombie (recorded worker identity is
/// proven dead), flip it to `Failed` and persist the change so `review
/// status` reports the truth instead of a run stuck `running` forever.
///
/// An inconclusive liveness probe leaves the row untouched (a probe
/// failure is not proof of death) and is logged, never silently reap.
pub fn read_reconciled(&self, id: &str) -> Result<ReviewRun, ReviewerError> {
let transaction = StateTransaction::acquire(&self.root)?;
let mut run = self.read(id)?;
match run.reconcile_liveness(crate::watcher::probe_worker_identity) {
ReconcileOutcome::Reaped => self.write_locked(&transaction, &run)?,
ReconcileOutcome::ProbeUnknown => log_inconclusive_probe(&run),
ReconcileOutcome::Unchanged => {}
}
Ok(run)
}
/// List all runs, reconciling any `Running` zombies to `Failed` and persisting
/// the correction, so a dead worker never lingers as `running`.
pub fn list_reconciled(&self) -> Result<Vec<ReviewRun>, ReviewerError> {
let transaction = StateTransaction::acquire(&self.root)?;
let mut runs = self.list()?;
for run in &mut runs {
match run.reconcile_liveness(crate::watcher::probe_worker_identity) {
ReconcileOutcome::Reaped => self.write_locked(&transaction, run)?,
ReconcileOutcome::ProbeUnknown => log_inconclusive_probe(run),
ReconcileOutcome::Unchanged => {}
}
}
Ok(runs)
}
/// Reap orphaned `Running` rows whose recorded worker identity is proven
/// dead, persisting each correction, and return how many rows were reaped.
///
/// `ensure-watcher` runs this before its spawn decision: after a watcher
/// is killed, its in-flight rows must stop masquerading as live work —
/// 0.9.2 left them `running` forever, which hid the fact that no live
/// watcher existed at all.
///
/// The whole read-probe-write sequence runs under the run-store lock, so
/// a worker completing concurrently can never be clobbered back to
/// `failed` by a stale snapshot. Rows whose probe is inconclusive are
/// left alone with a logged notice.
pub fn reconcile_stale_runs(&self) -> Result<usize, ReviewerError> {
let transaction = StateTransaction::acquire(&self.root)?;
let mut reaped = 0;
for mut run in self.list()? {
match run.reconcile_liveness(crate::watcher::probe_worker_identity) {
ReconcileOutcome::Reaped => {
self.write_locked(&transaction, &run)?;
reaped += 1;
}
ReconcileOutcome::ProbeUnknown => log_inconclusive_probe(&run),
ReconcileOutcome::Unchanged => {}
}
}
Ok(reaped)
}
/// Count `Running` rows whose recorded worker identity is proven dead.
/// READ-ONLY: unlike [`Self::reconcile_stale_runs`] nothing is persisted —
/// this is the truthful view `status` reports while leaving the write to
/// the watcher lifecycle. Rows with an inconclusive probe are NOT
/// counted: an unproven death is not a stale row.
pub fn stale_running_count(&self) -> Result<usize, ReviewerError> {
let mut stale = 0;
for mut run in self.list()? {
if run.reconcile_liveness(crate::watcher::probe_worker_identity)
== ReconcileOutcome::Reaped
{
stale += 1;
}
}
Ok(stale)
}
/// Cancel a review run, tolerating a running-but-dead worker.
///
/// - `Queued` runs cancel outright.
/// - A `Running` run whose worker identity is proven dead is reaped (marked
/// `Failed` stale), so a zombie can always be cleaned up.
/// - A `Running` run with a live worker is refused unless `force`, which
/// SIGKILLs the worker and then marks the run `Cancelled`.
/// - A `Running` run whose liveness is UNPROVABLE (probe inconclusive, or a
/// legacy run with no recorded pid) is refused unless `force` — an
/// unproven death never justifies a reap.
/// - Terminal runs (completed/failed/cancelled) are refused.
pub fn cancel(&self, id: &str, force: bool) -> Result<ReviewRun, ReviewerError> {
let transaction = StateTransaction::acquire(&self.root)?;
let mut run = self.read(id)?;
match run.status {
ReviewRunStatus::Queued => run.mark_cancelled(),
ReviewRunStatus::Running => match run.worker_pid {
Some(pid) => {
let identity = crate::watcher::ProcessIdentity {
pid,
start_token: run.worker_start_token.clone().unwrap_or_default(),
};
match crate::watcher::probe_worker_identity(&identity) {
crate::watcher::WorkerLiveness::Alive => {
if !force {
return Err(ReviewerError::ReviewRunStillAlive {
id: id.to_owned(),
pid,
});
}
kill_pid(pid)?;
run.mark_cancelled();
}
crate::watcher::WorkerLiveness::Dead => {
run.mark_failed(stale_worker_reason(pid));
}
crate::watcher::WorkerLiveness::Unknown => {
if !force {
return Err(ReviewerError::ReviewRunLivenessUnknown {
id: id.to_owned(),
});
}
run.mark_failed(
"worker liveness could not be verified; force-cancelled",
);
}
}
}
None => {
if !force {
return Err(ReviewerError::ReviewRunLivenessUnknown { id: id.to_owned() });
}
run.mark_failed("worker liveness could not be verified; force-cancelled");
}
},
terminal => {
return Err(ReviewerError::CannotCancelReview {
id: id.to_owned(),
status: terminal,
});
}
}
self.write_locked(&transaction, &run)?;
Ok(run)
}
/// Test helper: acquire a transaction and persist one run row.
#[cfg(test)]
fn write(&self, run: &ReviewRun) -> Result<(), ReviewerError> {
let transaction = StateTransaction::acquire(&self.root)?;
self.write_locked(&transaction, run)
}
fn write_locked(
&self,
_transaction: &StateTransaction,
run: &ReviewRun,
) -> Result<(), ReviewerError> {
let bytes = serde_json::to_vec_pretty(run).map_err(ReviewerError::RunJson)?;
atomic_replace(&self.path(&run.id), &bytes)?;
Ok(())
}
}
/// Surface an inconclusive worker-liveness probe: the row is left alone, but
/// the uncertainty is logged, never silently ignored.
fn log_inconclusive_probe(run: &ReviewRun) {
tracing::warn!(
run_id = %run.id,
worker_pid = run.worker_pid,
"worker liveness probe inconclusive; leaving the running row untouched (a probe failure is not proof of death)"
);
}
fn entire_checkpoint_for_current_repo(commit_sha: &str) -> Option<EntireCheckpointRef> {
let repo_root = current_repo_root().unwrap_or_else(|| PathBuf::from("."));
provenance::entire_checkpoint_for_commit(&repo_root, commit_sha)
}
fn current_repo_root() -> Option<PathBuf> {
let output = Command::new("git")
.args(["rev-parse", "--show-toplevel"])
.output()
.ok()?;
output
.status
.success()
.then(|| PathBuf::from(String::from_utf8_lossy(&output.stdout).trim()))
}
#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
pub struct QueuedReview {
#[serde(default)]
pub run_id: String,
pub commit_sha: String,
pub enqueued_at_unix: u64,
/// When set, this queued item is a petition re-review: `commit_sha` is the
/// FIX commit and this field names the original rejection it petitions.
/// `None` (the default, omitted on the wire) is a normal commit review, so
/// pre-existing queue lines keep parsing.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub petition_for: Option<String>,
/// Scheduler-assigned batch correlation id for batched petition reviews.
/// `None` (the default) means the item has not been claimed by a batch.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub batch_id: Option<String>,
}
#[derive(Clone, Debug)]
pub struct ReviewQueue {
root: PathBuf,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
struct PetitionFailureMarker {
run_id: String,
fix_sha: String,
original_sha: String,
attempts: u32,
error: String,
}
fn read_petition_failure_markers(root: &Path) -> Result<Vec<PetitionFailureMarker>, ReviewerError> {
let contents = match fs::read_to_string(root.join(PETITION_FAILURES_FILE)) {
Ok(contents) => contents,
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(Vec::new()),
Err(error) => return Err(ReviewerError::QueueIo(error)),
};
contents
.lines()
.filter(|line| !line.trim().is_empty())
.map(|line| serde_json::from_str(line).map_err(ReviewerError::QueueJson))
.collect()
}
fn append_petition_failure_marker_locked(
root: &Path,
_transaction: &StateTransaction,
marker: &PetitionFailureMarker,
) -> Result<(), ReviewerError> {
let mut markers = read_petition_failure_markers(root)?;
if markers
.iter()
.any(|existing| existing.run_id == marker.run_id)
{
return Ok(());
}
markers.push(marker.clone());
let mut bytes = Vec::new();
for marker in markers {
serde_json::to_writer(&mut bytes, &marker).map_err(ReviewerError::QueueJson)?;
bytes.push(b'\n');
}
atomic_replace(&root.join(PETITION_FAILURES_FILE), &bytes)?;
Ok(())
}
#[derive(Clone, Debug, Default, Eq, PartialEq)]
pub struct ReviewQueueSummary {
pub pending_count: usize,
pub oldest_enqueued_at_unix: Option<u64>,
}
impl ReviewQueueSummary {
pub fn oldest_age_secs_at(&self, now: u64) -> Option<u64> {
self.oldest_enqueued_at_unix
.map(|oldest| now.saturating_sub(oldest))
}
}
impl ReviewQueue {
pub fn new(root: impl Into<PathBuf>) -> Self {
Self { root: root.into() }
}
pub fn path(&self) -> PathBuf {
self.root.join(REVIEW_QUEUE_FILE)
}
pub fn enqueue(&self, commit_sha: impl Into<String>) -> Result<QueuedReview, ReviewerError> {
self.enqueue_item(commit_sha.into(), None)
}
/// Enqueue a petition re-review: `fix_sha` is reviewed with the petition
/// prompt, and its verdict transitions the original rejection
/// (`original_sha`). This is the queue leg of `truth-mirror resolve
/// --fixed-by` — without it the watcher would never build a petition job.
pub fn enqueue_petition(
&self,
fix_sha: impl Into<String>,
original_sha: impl Into<String>,
) -> Result<QueuedReview, ReviewerError> {
self.enqueue_item(fix_sha.into(), Some(original_sha.into()))
}
fn enqueue_item(
&self,
commit_sha: String,
petition_for: Option<String>,
) -> Result<QueuedReview, ReviewerError> {
let transaction = StateTransaction::acquire(&self.root)?;
self.recover_durable_enqueue_locked(&transaction)?;
self.enqueue_item_locked(&transaction, commit_sha, petition_for)
}
pub(crate) fn enqueue_item_locked(
&self,
transaction: &StateTransaction,
commit_sha: String,
petition_for: Option<String>,
) -> Result<QueuedReview, ReviewerError> {
let pending = self.read_pending_unlocked()?;
if let Some(existing) = pending
.iter()
.find(|queued| queued.commit_sha == commit_sha && queued.petition_for == petition_for)
{
return Ok(existing.clone());
}
let checkpoint = entire_checkpoint_for_current_repo(&commit_sha);
let target = queue_target_label(petition_for.is_some());
let run_store = ReviewRunStore::new(&self.root);
let run = run_store.create_queued_locked(
transaction,
&commit_sha,
target,
checkpoint,
petition_for.clone(),
)?;
let item = QueuedReview {
run_id: run.id,
commit_sha,
enqueued_at_unix: unix_now(),
petition_for,
..Default::default()
};
write_enqueue_inflight(&self.root, &item)?;
self.append_queue_item_locked(transaction, &item)?;
remove_enqueue_inflight(&self.root)?;
Ok(item)
}
/// Read queued items without running durable recovery (safe for status summaries).
pub fn pending_read_only(&self) -> Result<Vec<QueuedReview>, ReviewerError> {
self.read_pending_unlocked()
}
pub fn pending(&self) -> Result<Vec<QueuedReview>, ReviewerError> {
let transaction = StateTransaction::acquire(&self.root)?;
self.recover_durable_enqueue_locked(&transaction)?;
// Release crash-abandoned claims under the same mutation lock so the
// subsequent snapshot does not double-start dead claimed work.
let run_store = ReviewRunStore::new(&self.root);
run_store.recover_stale_scheduler_claims_locked(&transaction)?;
self.read_pending_unlocked()
}
fn read_pending_unlocked(&self) -> Result<Vec<QueuedReview>, ReviewerError> {
let contents = match fs::read_to_string(self.path()) {
Ok(contents) => contents,
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(Vec::new()),
Err(error) => return Err(ReviewerError::QueueIo(error)),
};
contents
.lines()
.filter(|line| !line.trim().is_empty())
.map(|line| serde_json::from_str(line).map_err(ReviewerError::QueueJson))
.collect()
}
fn append_queue_item_locked(
&self,
_transaction: &StateTransaction,
item: &QueuedReview,
) -> Result<(), ReviewerError> {
let mut bytes = match fs::read(self.path()) {
Ok(bytes) => bytes,
Err(error) if error.kind() == io::ErrorKind::NotFound => Vec::new(),
Err(error) => return Err(ReviewerError::QueueIo(error)),
};
serde_json::to_writer(&mut bytes, item).map_err(ReviewerError::QueueJson)?;
bytes.push(b'\n');
atomic_replace(&self.path(), &bytes)?;
Ok(())
}
fn recover_durable_enqueue(&self) -> Result<(), ReviewerError> {
let transaction = StateTransaction::acquire(&self.root)?;
self.recover_durable_enqueue_locked(&transaction)
}
fn recover_durable_enqueue_locked(
&self,
transaction: &StateTransaction,
) -> Result<(), ReviewerError> {
let inflight_path = self.root.join(ENQUEUE_INFLIGHT_FILE);
if inflight_path.exists() {
let bytes = fs::read(&inflight_path).map_err(ReviewerError::QueueIo)?;
let item: QueuedReview =
serde_json::from_slice(&bytes).map_err(ReviewerError::QueueJson)?;
let run_store = ReviewRunStore::new(&self.root);
run_store.ensure_queued_locked(
transaction,
&item.run_id,
&item.commit_sha,
queue_target_label(item.petition_for.is_some()),
entire_checkpoint_for_current_repo(&item.commit_sha),
item.petition_for.clone(),
)?;
let pending = self.read_pending_unlocked()?;
if !queue_contains_item(&pending, &item) {
self.append_queue_item_locked(transaction, &item)?;
}
remove_enqueue_inflight(&self.root)?;
}
self.recover_orphan_queued_runs_locked(transaction)?;
self.normalize_legacy_queue_run_ids_locked(transaction)?;
self.remove_terminal_failed_petitions_locked(transaction)?;
self.recover_unqueued_petition_enqueues_locked(transaction)
}
fn recover_orphan_queued_runs_locked(
&self,
transaction: &StateTransaction,
) -> Result<(), ReviewerError> {
let pending = self.read_pending_unlocked()?;
let referenced: std::collections::BTreeSet<String> = pending
.iter()
.filter(|item| !item.run_id.trim().is_empty())
.map(|item| item.run_id.clone())
.collect();
let run_store = ReviewRunStore::new(&self.root);
let (runs, _skipped) = run_store.list_lenient()?;
for run in runs {
if run.status != ReviewRunStatus::Queued || referenced.contains(&run.id) {
continue;
}
if run.target == "commit" || run.target == "petition" {
let item = QueuedReview {
run_id: run.id.clone(),
commit_sha: run.commit_sha.clone(),
enqueued_at_unix: run.created_at_unix,
petition_for: run.petition_for.clone(),
..Default::default()
};
if !queue_contains_item(&pending, &item) {
self.append_queue_item_locked(transaction, &item)?;
}
} else {
run_store.cancel_queued_locked(transaction, &run.id)?;
}
}
Ok(())
}
pub fn summary(&self) -> Result<ReviewQueueSummary, ReviewerError> {
let pending = self.pending_read_only()?;
Ok(ReviewQueueSummary {
pending_count: pending.len(),
oldest_enqueued_at_unix: pending.iter().map(|item| item.enqueued_at_unix).min(),
})
}
/// Drop every queued item for `sha`, preserving any items appended for other
/// commits. Called after a commit is reviewed so a drain never repeats work.
pub fn remove_sha(&self, sha: &str) -> Result<(), ReviewerError> {
self.recover_durable_enqueue()?;
let transaction = StateTransaction::acquire(&self.root)?;
let remaining: Vec<QueuedReview> = self
.read_pending_unlocked()?
.into_iter()
.filter(|item| item.commit_sha != sha)
.collect();
self.rewrite(&transaction, &remaining)
}
fn rewrite(
&self,
_transaction: &StateTransaction,
items: &[QueuedReview],
) -> Result<(), ReviewerError> {
let mut bytes = Vec::new();
for item in items {
serde_json::to_writer(&mut bytes, item).map_err(ReviewerError::QueueJson)?;
bytes.push(b'\n');
}
atomic_replace(&self.path(), &bytes)?;
Ok(())
}
/// Drop the queued item matching BOTH the commit SHA and the petition
/// identity, preserving sibling rows for the same SHA (a fix commit's own
/// review and a petition review of it are distinct queue items).
pub fn remove_item(&self, sha: &str, petition_for: Option<&str>) -> Result<(), ReviewerError> {
self.recover_durable_enqueue()?;
let transaction = StateTransaction::acquire(&self.root)?;
self.remove_item_locked(&transaction, sha, petition_for)
}
fn remove_item_locked(
&self,
transaction: &StateTransaction,
sha: &str,
petition_for: Option<&str>,
) -> Result<(), ReviewerError> {
let remaining: Vec<QueuedReview> = self
.read_pending_unlocked()?
.into_iter()
.filter(|item| {
!(item.commit_sha == sha && item.petition_for.as_deref() == petition_for)
})
.collect();
self.rewrite(transaction, &remaining)
}
pub fn remove_run_id(&self, run_id: &str) -> Result<(), ReviewerError> {
self.recover_durable_enqueue()?;
let transaction = StateTransaction::acquire(&self.root)?;
let remaining: Vec<QueuedReview> = self
.read_pending_unlocked()?
.into_iter()
.filter(|item| item.run_id != run_id)
.collect();
self.rewrite(&transaction, &remaining)
}
pub(crate) fn attach_run_id_for_item(
&self,
sha: &str,
petition_for: Option<&str>,
run_id: &str,
) -> Result<(), ReviewerError> {
let transaction = StateTransaction::acquire(&self.root)?;
let pending = self.read_pending_unlocked()?;
let mut changed = false;
let updated: Vec<QueuedReview> = pending
.iter()
.map(|item| {
if item.commit_sha == sha
&& item.petition_for.as_deref() == petition_for
&& item.run_id.trim().is_empty()
{
changed = true;
QueuedReview {
run_id: run_id.to_owned(),
..item.clone()
}
} else {
item.clone()
}
})
.collect();
if changed {
self.rewrite(&transaction, &updated)?;
}
Ok(())
}
pub(crate) fn attach_batch_id_for_item(
&self,
sha: &str,
petition_for: Option<&str>,
batch_id: &str,
) -> Result<(), ReviewerError> {
let transaction = StateTransaction::acquire(&self.root)?;
let pending = self.read_pending_unlocked()?;
let mut changed = false;
let updated: Vec<QueuedReview> = pending
.iter()
.map(|item| {
if item.commit_sha == sha
&& item.petition_for.as_deref() == petition_for
&& item.batch_id.is_none()
{
changed = true;
QueuedReview {
batch_id: Some(batch_id.to_owned()),
..item.clone()
}
} else {
item.clone()
}
})
.collect();
if changed {
self.rewrite(&transaction, &updated)?;
}
Ok(())
}
}
fn unqueued_petition_enqueues(
store: &LedgerStore,
pending: &[QueuedReview],
failures: &[PetitionFailureMarker],
) -> Result<Vec<TransitionProvenance>, ReviewerError> {
let history = store.read_history().map_err(ReviewerError::Ledger)?;
let mut latest_enqueue = std::collections::BTreeMap::<String, TransitionProvenance>::new();
for entry in history {
let Some(provenance) = entry.transition_provenance.as_ref() else {
continue;
};
match provenance.kind {
TransitionProvenanceKind::PetitionEnqueued => {
latest_enqueue.insert(provenance.original_sha.clone(), provenance.clone());
}
TransitionProvenanceKind::PetitionReview => {
latest_enqueue.remove(&provenance.original_sha);
}
}
}
Ok(latest_enqueue
.into_values()
.filter(|provenance| {
!failures.iter().any(|failure| {
failure.fix_sha == provenance.fix_sha
&& failure.original_sha == provenance.original_sha
&& failure.attempts == provenance.attempts
})
})
.filter(|provenance| {
!queue_contains_item(
pending,
&QueuedReview {
run_id: String::new(),
commit_sha: provenance.fix_sha.clone(),
enqueued_at_unix: 0,
petition_for: Some(provenance.original_sha.clone()),
..Default::default()
},
)
})
.collect())
}
fn resolve_run_id_for_item(
run_store: &ReviewRunStore,
item: &QueuedReview,
) -> Result<(String, bool), ReviewerError> {
if !item.run_id.trim().is_empty() {
return Ok((item.run_id.clone(), false));
}
let (runs, _) = run_store.list_lenient()?;
let mut matches: Vec<&ReviewRun> = runs
.iter()
.filter(|run| {
run.status == ReviewRunStatus::Queued
&& run.commit_sha == item.commit_sha
&& run.petition_for == item.petition_for
})
.collect();
matches.sort_by_key(|run| run.created_at_unix);
if let Some(run) = matches.first() {
return Ok((run.id.clone(), false));
}
Ok((generate_run_id(&item.commit_sha), true))
}
/// Atomically spend a petition attempt and enqueue the fix review under one
/// shared state lock. Recovery replays [`TransitionProvenanceKind::PetitionEnqueued`]
/// rows when the ledger write landed but the queue row did not.
pub(crate) fn commit_petition_resolve(
state_dir: &Path,
store: &LedgerStore,
rejection_sha: &str,
fix_sha: &str,
) -> Result<(QueuedReview, u32), ReviewerError> {
let queue = ReviewQueue::new(state_dir);
let transaction = StateTransaction::acquire(state_dir)?;
queue.recover_durable_enqueue_locked(&transaction)?;
let original = store.show(rejection_sha).map_err(ReviewerError::Ledger)?;
if !original.is_unresolved_rejection() {
return Err(ReviewerError::PetitionRejectionNotOpen {
sha: rejection_sha.to_owned(),
});
}
let pending = queue.read_pending_unlocked()?;
if pending
.iter()
.any(|item| item.petition_for.as_deref() == Some(rejection_sha))
{
return Err(ReviewerError::PetitionInFlight {
rejection_sha: rejection_sha.to_owned(),
});
}
let prior = store
.petition_attempts_for(rejection_sha)
.map_err(ReviewerError::Ledger)?;
if prior >= crate::resolve::MAX_PETITION_ATTEMPTS {
return Err(ReviewerError::PetitionExhausted {
rejection_sha: rejection_sha.to_owned(),
});
}
let next_attempts = prior.saturating_add(1);
let reason = format!(
"petition review enqueued: fix={fix_sha}, attempts={next_attempts}/{}",
crate::resolve::MAX_PETITION_ATTEMPTS
);
store
.append_petition_transition_locked(
&transaction,
rejection_sha,
Disposition::Open,
ResolutionKind::Resolved,
&reason,
next_attempts,
Some(TransitionProvenance {
kind: TransitionProvenanceKind::PetitionEnqueued,
fix_sha: fix_sha.to_owned(),
original_sha: rejection_sha.to_owned(),
attempts: next_attempts,
}),
)
.map_err(ReviewerError::Ledger)?;
let item = queue.enqueue_item_locked(
&transaction,
fix_sha.to_owned(),
Some(rejection_sha.to_owned()),
)?;
Ok((item, next_attempts))
}
impl ReviewQueue {
fn normalize_legacy_queue_run_ids_locked(
&self,
transaction: &StateTransaction,
) -> Result<(), ReviewerError> {
let pending = self.read_pending_unlocked()?;
let run_store = ReviewRunStore::new(&self.root);
let (runs, _) = run_store.list_lenient()?;
let mut changed = false;
let normalized: Vec<QueuedReview> = pending
.iter()
.map(|item| {
if !item.run_id.trim().is_empty() {
return item.clone();
}
let mut matches: Vec<&ReviewRun> = runs
.iter()
.filter(|run| {
run.status == ReviewRunStatus::Queued
&& run.commit_sha == item.commit_sha
&& run.petition_for == item.petition_for
})
.collect();
matches.sort_by_key(|run| run.created_at_unix);
if let Some(run) = matches.first() {
changed = true;
QueuedReview {
run_id: run.id.clone(),
..item.clone()
}
} else {
item.clone()
}
})
.collect();
if changed {
self.rewrite(transaction, &normalized)?;
}
Ok(())
}
fn remove_terminal_failed_petitions_locked(
&self,
transaction: &StateTransaction,
) -> Result<(), ReviewerError> {
let failures = read_petition_failure_markers(&self.root)?;
if failures.is_empty() {
return Ok(());
}
let pending = self.read_pending_unlocked()?;
let remaining: Vec<QueuedReview> = pending
.iter()
.filter(|item| {
!failures.iter().any(|failure| {
failure.run_id == item.run_id
&& failure.fix_sha == item.commit_sha
&& item.petition_for.as_deref() == Some(failure.original_sha.as_str())
})
})
.cloned()
.collect();
if remaining.len() != pending.len() {
self.rewrite(transaction, &remaining)?;
}
Ok(())
}
fn recover_unqueued_petition_enqueues_locked(
&self,
transaction: &StateTransaction,
) -> Result<(), ReviewerError> {
let store = LedgerStore::new(&self.root);
let pending = self.read_pending_unlocked()?;
let failures = read_petition_failure_markers(&self.root)?;
for provenance in unqueued_petition_enqueues(&store, &pending, &failures)? {
self.enqueue_item_locked(
transaction,
provenance.fix_sha.clone(),
Some(provenance.original_sha.clone()),
)?;
}
Ok(())
}
}
fn queue_target_label(is_petition: bool) -> &'static str {
if is_petition { "petition" } else { "commit" }
}
fn queue_contains_item(pending: &[QueuedReview], item: &QueuedReview) -> bool {
pending.iter().any(|queued| {
queued.commit_sha == item.commit_sha
&& queued.petition_for == item.petition_for
&& (queued.run_id.trim().is_empty()
|| item.run_id.trim().is_empty()
|| queued.run_id == item.run_id)
})
}
fn write_enqueue_inflight(root: &Path, item: &QueuedReview) -> Result<(), ReviewerError> {
let bytes = serde_json::to_vec(item).map_err(ReviewerError::QueueJson)?;
atomic_replace(&root.join(ENQUEUE_INFLIGHT_FILE), &bytes)?;
Ok(())
}
fn remove_enqueue_inflight(root: &Path) -> Result<(), ReviewerError> {
match fs::remove_file(root.join(ENQUEUE_INFLIGHT_FILE)) {
Ok(()) => {}
Err(error) if error.kind() == io::ErrorKind::NotFound => {}
Err(error) => return Err(ReviewerError::QueueIo(error)),
}
Ok(())
}
fn run_ledger_phase_applied(run: &ReviewRun, phase: &str) -> bool {
run.ledger_applied_phase.as_deref() == Some(phase)
}
fn finish_replayed_queue_item(
queue: &ReviewQueue,
run_store: &ReviewRunStore,
run_id: &str,
sha: &str,
petition_for: Option<&str>,
ledger_entries: usize,
report: &mut DrainReport,
) -> Result<(), ReviewerError> {
let run = run_store.read(run_id)?;
if run.status != ReviewRunStatus::Completed {
run_store.mark_completed(run_id, ledger_entries)?;
}
queue.remove_item(sha, petition_for)?;
report.reviewed.push(sha.to_owned());
report.ledger_entries += ledger_entries;
Ok(())
}
/// Loads the claim and diff for a commit so the reviewer can run against it.
/// Abstracted so `drain_once` can be unit-tested without a real git repository.
pub trait MaterialLoader {
fn load(&self, sha: &str) -> Result<(Claim, String), ReviewerError>;
}
#[derive(Clone, Debug, Default)]
pub struct GitMaterialLoader {
/// Evidence-pointer patterns from config so async review parses claims the
/// same way the commit-msg gate did (a repo `jira:` pointer stays valid).
pub evidence_patterns: Vec<String>,
}
impl GitMaterialLoader {
pub fn from_config(config: &config::TruthMirrorConfig) -> Self {
Self::with_patterns(config.gates.to_policy().evidence_patterns)
}
pub fn with_patterns(evidence_patterns: Vec<String>) -> Self {
Self { evidence_patterns }
}
}
impl MaterialLoader for GitMaterialLoader {
fn load(&self, sha: &str) -> Result<(Claim, String), ReviewerError> {
let message = git_output(["show", "--format=%B", "--no-patch", sha])?;
let diff = git_output(["show", "--format=", "--patch", sha])?;
let claim = if self.evidence_patterns.is_empty() {
Claim::parse(&message)?
} else {
Claim::parse_with(&message, &self.evidence_patterns)?
};
Ok((claim, diff))
}
}
#[derive(Clone, Debug, Default, Eq, PartialEq)]
pub struct DrainReport {
pub reviewed: Vec<String>,
/// Queue items skipped after a terminal material-loading failure.
pub failed: Vec<String>,
pub ledger_entries: usize,
}
#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
struct StrictPetitionBatchKey<'a> {
arbiter_harness: &'static str,
arbiter_model: &'a str,
arbiter_effort: &'static str,
}
#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
struct PetitionBatchKey<'a> {
repository: &'a Path,
fix_sha: &'a str,
watched_agent: &'static str,
watched_model: &'a str,
reviewer_harness: &'static str,
reviewer_model: &'a str,
reviewer_effort: &'static str,
allow_same_model: bool,
strict: Option<StrictPetitionBatchKey<'a>>,
materialized_context: &'a str,
}
impl<'a> PetitionBatchKey<'a> {
fn new(
repository: &'a Path,
fix_sha: &'a str,
selection: &'a ReviewSelection,
materialized_context: &'a str,
) -> Self {
Self {
repository,
fix_sha,
watched_agent: surface::agent_slug(selection.watched_agent),
watched_model: &selection.watched_model,
reviewer_harness: harness_slug(selection.reviewer_harness),
reviewer_model: &selection.reviewer_model,
reviewer_effort: selection.reviewer_effort.as_str(),
allow_same_model: selection.allow_same_model,
strict: selection
.strict
.as_ref()
.map(|strict| StrictPetitionBatchKey {
arbiter_harness: harness_slug(strict.arbiter_harness),
arbiter_model: &strict.arbiter_model,
arbiter_effort: strict.arbiter_effort.as_str(),
}),
materialized_context,
}
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
enum PlannedDrainJob {
Single(QueuedReview),
PetitionBatch(Vec<QueuedReview>),
}
fn plan_batched_drain(
pending: &[QueuedReview],
repository: &Path,
selection: &ReviewSelection,
materialized_context: &str,
configured_max_batch_size: usize,
) -> Vec<PlannedDrainJob> {
let max_batch_size = configured_max_batch_size.clamp(1, config::MAX_PETITION_BATCH_SIZE);
let mut positioned = Vec::<(usize, usize, PlannedDrainJob)>::new();
let mut groups =
std::collections::BTreeMap::<PetitionBatchKey<'_>, (usize, Vec<QueuedReview>)>::new();
for (index, item) in pending.iter().enumerate() {
if item.petition_for.is_none() {
positioned.push((index, 0, PlannedDrainJob::Single(item.clone())));
continue;
}
let key = PetitionBatchKey::new(
repository,
&item.commit_sha,
selection,
materialized_context,
);
let group = groups.entry(key).or_insert_with(|| (index, Vec::new()));
group.1.push(item.clone());
}
for (_, (first_index, mut members)) in groups {
members.sort_by(|left, right| {
left.petition_for
.cmp(&right.petition_for)
.then_with(|| left.run_id.cmp(&right.run_id))
});
for (chunk_index, chunk) in members.chunks(max_batch_size).enumerate() {
positioned.push((
first_index,
chunk_index,
PlannedDrainJob::PetitionBatch(chunk.to_vec()),
));
}
}
positioned.sort_by_key(|(first_index, chunk_index, _)| (*first_index, *chunk_index));
positioned.into_iter().map(|(_, _, job)| job).collect()
}
/// Review every distinct queued commit exactly once, record verdicts, and remove
/// each commit from the queue as soon as its review lands. A material-loading
/// failure is terminal for that run: it is recorded and removed so one malformed
/// claim never strands the rest of the queue. Reviewer execution failures remain
/// queued for retry.
fn complete_review_execution<R: ProcessRunner>(
job: &ReviewJob,
runner: &R,
run_store: &ReviewRunStore,
run_id: &str,
existing_spool: Option<PendingReviewSpool>,
timeout: Option<Duration>,
) -> Result<ReviewExecution, ReviewerError> {
complete_review_execution_in_dir(
job,
runner,
run_store,
run_id,
existing_spool,
timeout,
None,
)
}
fn complete_review_execution_in_dir<R: ProcessRunner>(
job: &ReviewJob,
runner: &R,
run_store: &ReviewRunStore,
run_id: &str,
existing_spool: Option<PendingReviewSpool>,
timeout: Option<Duration>,
current_dir: Option<&Path>,
) -> Result<ReviewExecution, ReviewerError> {
if let Some(spool) = existing_spool {
if let Some(first_stdout) = spool.strict_pass_stdout.as_ref()
&& spool.entries.len() == 1
{
let strict_entry =
draft_review_strict_pass_in_dir(job, runner, first_stdout, timeout, current_dir)?;
let mut entries = spool.entries;
entries.push(strict_entry);
run_store.persist_pending_review(
run_id,
PendingReviewSpool {
entries: entries.clone(),
petition_transition_pending: spool.petition_transition_pending,
strict_pass_stdout: None,
},
)?;
return Ok(ReviewExecution { entries });
}
if !spool.entries.is_empty() {
return Ok(ReviewExecution {
entries: spool.entries,
});
}
}
let (first_entry, first_stdout) =
draft_review_first_pass_in_dir(job, runner, timeout, current_dir)?;
let first_verdict = ParsedVerdict::parse(&first_stdout)?;
if strict_pass_needed(job, &first_verdict) {
run_store.persist_pending_review(
run_id,
PendingReviewSpool {
entries: vec![first_entry.clone()],
petition_transition_pending: job.petition.is_some(),
strict_pass_stdout: Some(first_stdout.clone()),
},
)?;
let strict_entry =
draft_review_strict_pass_in_dir(job, runner, &first_stdout, timeout, current_dir)?;
let entries = vec![first_entry, strict_entry];
run_store.persist_pending_review(
run_id,
PendingReviewSpool {
entries: entries.clone(),
petition_transition_pending: job.petition.is_some(),
strict_pass_stdout: None,
},
)?;
Ok(ReviewExecution { entries })
} else {
let entries = vec![first_entry];
run_store.persist_pending_review(
run_id,
PendingReviewSpool {
entries: entries.clone(),
petition_transition_pending: job.petition.is_some(),
strict_pass_stdout: None,
},
)?;
Ok(ReviewExecution { entries })
}
}
#[derive(Clone, Debug)]
struct PreparedPetitionMember {
item: QueuedReview,
run_id: String,
run: ReviewRun,
petition: PetitionContext,
batch_id: Option<String>,
}
#[derive(Clone, Debug)]
struct ReadyPetitionMember {
member: PreparedPetitionMember,
execution: ReviewExecution,
}
fn batch_record_path(root: &Path, batch_id: &str) -> PathBuf {
root.join(BATCHES_DIR).join(format!("{batch_id}.json"))
}
fn read_batch_review_record(
root: &Path,
batch_id: &str,
) -> Result<Option<BatchReviewRecord>, ReviewerError> {
let path = batch_record_path(root, batch_id);
match fs::read_to_string(&path) {
Ok(contents) => serde_json::from_str(&contents)
.map_err(ReviewerError::RunJson)
.map(Some),
Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None),
Err(error) => Err(ReviewerError::QueueIo(error)),
}
}
fn write_batch_review_record(root: &Path, record: &BatchReviewRecord) -> Result<(), ReviewerError> {
let path = batch_record_path(root, &record.batch_id);
let bytes = serde_json::to_vec_pretty(record).map_err(ReviewerError::RunJson)?;
atomic_replace(&path, &bytes)?;
Ok(())
}
fn build_batch_review_record(
batch_id: &str,
fix_sha: &str,
phase: &str,
members: &[&PreparedPetitionMember],
plan: &ReviewPlan,
) -> BatchReviewRecord {
let timestamp = unix_now();
BatchReviewRecord {
batch_id: batch_id.to_owned(),
fix_sha: fix_sha.to_owned(),
phase: phase.to_owned(),
member_run_ids: members.iter().map(|member| member.run_id.clone()).collect(),
member_original_shas: members
.iter()
.map(|member| member.petition.original_sha.clone())
.collect(),
reviewer_harness: harness_slug(plan.reviewer_harness).to_owned(),
reviewer_model: plan.reviewer_model.clone(),
raw_stdout: None,
created_at_unix: timestamp,
updated_at_unix: timestamp,
}
}
fn update_batch_review_record_phase_and_output(
record: &mut BatchReviewRecord,
phase: &str,
raw_stdout: Option<String>,
) {
record.phase = phase.to_owned();
record.raw_stdout = raw_stdout;
record.updated_at_unix = unix_now();
}
fn claim_petition_batch(
queue: &ReviewQueue,
run_store: &ReviewRunStore,
fix_sha: &str,
batch_id: &str,
members: &mut [PreparedPetitionMember],
) -> Result<(), ReviewerError> {
let member_count = members.len();
for (index, member) in members.iter_mut().enumerate() {
member.batch_id = Some(batch_id.to_owned());
let batch_ref = BatchRunRef {
batch_id: batch_id.to_owned(),
kind: BatchKind::Petition,
member_index: index as u32,
member_count: member_count as u32,
fix_sha: fix_sha.to_owned(),
};
member.run.batch = Some(batch_ref.clone());
run_store.set_batch_metadata(&member.run_id, batch_ref)?;
queue.attach_batch_id_for_item(fix_sha, member.item.petition_for.as_deref(), batch_id)?;
}
Ok(())
}
fn petition_batch_id(phase: &str, first_run_id: &str) -> String {
format!("petition-batch-{phase}-{first_run_id}")
}
#[allow(clippy::too_many_arguments)]
fn run_petition_batch_pass<R: ProcessRunner>(
state_dir: &Path,
request: ReviewRequest,
prompt: &str,
batch_id: &str,
fix_sha: &str,
requested_shas: &[String],
runner: &R,
phase: &str,
members: &[&PreparedPetitionMember],
timeout: Option<Duration>,
current_dir: Option<&Path>,
) -> Result<(ReviewPlan, ParsedPetitionBatch), ReviewerError> {
let plan = ReviewPlan::build(request)?;
let mut record = match read_batch_review_record(state_dir, batch_id)? {
Some(existing) => existing,
None => {
let record = build_batch_review_record(batch_id, fix_sha, "claimed", members, &plan);
write_batch_review_record(state_dir, &record)?;
record
}
};
if let Some(raw_stdout) = record.raw_stdout.as_deref() {
let parsed = ParsedPetitionBatch::parse(raw_stdout, batch_id, fix_sha, requested_shas)?;
for unknown in &parsed.unknown_item_ids {
tracing::warn!(
batch_id,
original_rejection_sha = unknown,
"ignored unknown or malformed petition-batch result"
);
}
return Ok((plan, parsed));
}
let output = plan.run_with_dir(prompt, runner, timeout, current_dir)?;
ensure_process_success(&output)?;
update_batch_review_record_phase_and_output(&mut record, phase, Some(output.stdout.clone()));
write_batch_review_record(state_dir, &record)?;
let parsed = ParsedPetitionBatch::parse(&output.stdout, batch_id, fix_sha, requested_shas)?;
for unknown in &parsed.unknown_item_ids {
tracing::warn!(
batch_id,
original_rejection_sha = unknown,
"ignored unknown or malformed petition-batch result"
);
}
Ok((plan, parsed))
}
fn fail_petition_batch_member(
queue: &ReviewQueue,
run_store: &ReviewRunStore,
member: &PreparedPetitionMember,
error: impl Into<String>,
remove_from_queue: bool,
report: &mut DrainReport,
) -> Result<(), ReviewerError> {
let error = error.into();
if remove_from_queue {
let transaction = StateTransaction::acquire(&queue.root)?;
run_store.mark_failed_locked(&transaction, &member.run_id, &error)?;
append_petition_failure_marker_locked(
&queue.root,
&transaction,
&PetitionFailureMarker {
run_id: member.run_id.clone(),
fix_sha: member.item.commit_sha.clone(),
original_sha: member.petition.original_sha.clone(),
attempts: member.petition.attempts_so_far,
error,
},
)?;
queue.remove_item_locked(
&transaction,
&member.item.commit_sha,
member.item.petition_for.as_deref(),
)?;
report.failed.push(member.item.commit_sha.clone());
} else {
run_store.mark_failed(&member.run_id, error)?;
}
Ok(())
}
fn finish_petition_batch_member(
queue: &ReviewQueue,
run_store: &ReviewRunStore,
ready: ReadyPetitionMember,
store: &LedgerStore,
config: &config::TruthMirrorConfig,
report: &mut DrainReport,
) -> Result<(), ReviewerError> {
let ReadyPetitionMember { member, execution } = ready;
apply_petition_batch_execution(&member.petition, &execution, store)?;
run_store.mark_ledger_applied(&member.run_id, REVIEW_LEDGER_PHASE, execution.entries.len())?;
run_store.clear_pending_review(&member.run_id)?;
record_memory_skill_outcome(
&queue.root,
config,
&member.run_id,
&member.item.commit_sha,
"watch-drain-batch",
&execution.entries,
);
run_store.mark_completed(&member.run_id, execution.entries.len())?;
queue.remove_item(&member.item.commit_sha, member.item.petition_for.as_deref())?;
report.ledger_entries += execution.entries.len();
report.reviewed.push(member.item.commit_sha);
Ok(())
}
fn finish_ready_petition_members(
queue: &ReviewQueue,
run_store: &ReviewRunStore,
mut ready: Vec<ReadyPetitionMember>,
store: &LedgerStore,
config: &config::TruthMirrorConfig,
report: &mut DrainReport,
) -> Result<(), ReviewerError> {
ready.sort_by(|left, right| {
left.member
.petition
.original_sha
.cmp(&right.member.petition.original_sha)
});
for member in ready {
finish_petition_batch_member(queue, run_store, member, store, config, report)?;
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn drain_petition_batch<R: ProcessRunner, L: MaterialLoader>(
queue: &ReviewQueue,
run_store: &ReviewRunStore,
items: Vec<QueuedReview>,
loader: &L,
selection: &ReviewSelection,
context: &str,
runner: &R,
store: &LedgerStore,
config: &config::TruthMirrorConfig,
current_dir: Option<&Path>,
) -> Result<DrainReport, ReviewerError> {
let timeout = config.reviewer.timeout();
let mut report = DrainReport::default();
let Some(fix_sha) = items.first().map(|item| item.commit_sha.clone()) else {
return Ok(report);
};
debug_assert!(
items
.iter()
.all(|item| { item.commit_sha == fix_sha && item.petition_for.is_some() })
);
let mut active = Vec::new();
for item in items {
let petition_for = item.petition_for.clone();
let (run_id, _) = resolve_run_id_for_item(run_store, &item)?;
if item.run_id.trim().is_empty() {
queue.attach_run_id_for_item(&fix_sha, petition_for.as_deref(), &run_id)?;
}
let run = match run_store.ensure_queued(&run_id, &fix_sha, "petition", petition_for.clone())
{
Ok(run) => run,
Err(ReviewerError::RunJson(_)) => {
queue.remove_item(&fix_sha, petition_for.as_deref())?;
report.failed.push(fix_sha.clone());
continue;
}
Err(error) => return Err(error),
};
if run.status == ReviewRunStatus::Cancelled {
queue.remove_item(&fix_sha, petition_for.as_deref())?;
continue;
}
if run.status == ReviewRunStatus::Completed
|| run_ledger_phase_applied(&run, REVIEW_LEDGER_PHASE)
{
finish_replayed_queue_item(
queue,
run_store,
&run_id,
&fix_sha,
petition_for.as_deref(),
run.ledger_entries,
&mut report,
)?;
continue;
}
active.push((item, run_id, run));
}
if active.is_empty() {
return Ok(report);
}
let (claim, diff) = match loader.load(&fix_sha) {
Ok(material) => material,
Err(error) => {
let message = error.to_string();
for (item, run_id, _) in active {
run_store.mark_failed(&run_id, &message)?;
queue.remove_item(&fix_sha, item.petition_for.as_deref())?;
report.failed.push(fix_sha.clone());
}
return Ok(report);
}
};
let mut prepared = Vec::new();
for (item, run_id, run) in active {
let original_sha = item
.petition_for
.as_deref()
.expect("petition batch member missing petition identity");
match petition_context_from_ledger(original_sha, &fix_sha, store) {
Ok(petition) => prepared.push(PreparedPetitionMember {
item,
run_id,
run,
petition,
batch_id: None,
}),
Err(error) => {
run_store.mark_failed(&run_id, error.to_string())?;
queue.remove_item(&fix_sha, Some(original_sha))?;
report.failed.push(fix_sha.clone());
}
}
}
if prepared.is_empty() {
return Ok(report);
}
let mut fresh = Vec::new();
let mut strict_pending = Vec::<(PreparedPetitionMember, LedgerEntry)>::new();
let mut ready = Vec::new();
for member in prepared {
match member.run.pending_review.clone() {
Some(spool) if !spool.entries.is_empty() => {
if spool.strict_pass_stdout.is_some() && spool.entries.len() == 1 {
if selection.strict.is_some() {
strict_pending.push((member, spool.entries[0].clone()));
} else {
fail_petition_batch_member(
queue,
run_store,
&member,
"pending strict petition batch has no strict reviewer selection",
true,
&mut report,
)?;
}
} else {
ready.push(ReadyPetitionMember {
member,
execution: ReviewExecution {
entries: spool.entries,
},
});
}
}
_ => fresh.push(member),
}
}
if !fresh.is_empty() {
for member in &fresh {
if member.run.status != ReviewRunStatus::Running
|| member.run.phase != REVIEW_LEDGER_PHASE
{
run_store.mark_running(&member.run_id, REVIEW_LEDGER_PHASE)?;
}
}
let batch_id = petition_batch_id("first", &fresh[0].run_id);
claim_petition_batch(queue, run_store, &fix_sha, &batch_id, &mut fresh)?;
let petitions: Vec<&PetitionContext> =
fresh.iter().map(|member| &member.petition).collect();
let requested_shas: Vec<String> = petitions
.iter()
.map(|petition| petition.original_sha.clone())
.collect();
let prompt = petition_batch_prompt(&batch_id, &petitions, &claim, &diff, context);
let request = selection.request_for(String::new());
match run_petition_batch_pass(
queue.root.as_path(),
request,
&prompt,
&batch_id,
&fix_sha,
&requested_shas,
runner,
"first-pass-running",
&fresh.iter().collect::<Vec<_>>(),
timeout,
current_dir,
) {
Ok((plan, mut parsed)) => {
for member in fresh {
let original_sha = member.petition.original_sha.clone();
match parsed
.members
.remove(&original_sha)
.expect("batch parser omitted a requested member classification")
{
ParsedPetitionBatchMember::Valid(verdict) => {
let first_entry = petition_entry_from_batch_verdict(
&fix_sha,
&claim,
&member.petition,
&plan,
&verdict,
&member.run_id,
member.batch_id.as_deref().unwrap_or(&batch_id),
);
if selection.strict.is_some() && verdict.verdict == Verdict::Pass {
run_store.persist_pending_review(
&member.run_id,
PendingReviewSpool {
entries: vec![first_entry.clone()],
petition_transition_pending: true,
strict_pass_stdout: Some(verdict.raw),
},
)?;
strict_pending.push((member, first_entry));
} else {
let execution = ReviewExecution {
entries: vec![first_entry],
};
run_store.persist_pending_review(
&member.run_id,
PendingReviewSpool {
entries: execution.entries.clone(),
petition_transition_pending: true,
strict_pass_stdout: None,
},
)?;
ready.push(ReadyPetitionMember { member, execution });
}
}
ParsedPetitionBatchMember::Invalid(message) => {
fail_petition_batch_member(
queue,
run_store,
&member,
message,
true,
&mut report,
)?;
}
}
}
}
Err(
error @ (ReviewerError::VerdictJson { .. }
| ReviewerError::BatchVerdictSchema { .. }),
) => {
let message = error.to_string();
for member in fresh {
fail_petition_batch_member(
queue,
run_store,
&member,
&message,
true,
&mut report,
)?;
}
}
Err(error) if is_retryable_provider_failure(&error) => {
let message = error.to_string();
for member in &fresh {
// Keep queue rows and do not spend another petition attempt.
run_store.mark_provider_backoff(
&member.run_id,
&message,
DEFAULT_PROVIDER_BACKOFF_SECS,
)?;
}
return Ok(report);
}
Err(error) => {
let message = error.to_string();
for member in &fresh {
fail_petition_batch_member(
queue,
run_store,
member,
&message,
false,
&mut report,
)?;
}
return Err(error);
}
}
}
if strict_pending.is_empty() {
finish_ready_petition_members(queue, run_store, ready, store, config, &mut report)?;
return Ok(report);
}
strict_pending.sort_by(|left, right| {
left.0
.petition
.original_sha
.cmp(&right.0.petition.original_sha)
});
for (member, _) in &strict_pending {
if member.run.status != ReviewRunStatus::Running || member.run.phase != REVIEW_LEDGER_PHASE
{
run_store.mark_running(&member.run_id, REVIEW_LEDGER_PHASE)?;
}
}
let strict = selection
.strict
.as_ref()
.expect("strict petition members without strict selection");
let reviewer_request = selection.request_for(String::new());
validate_strict_arbiter(&reviewer_request, strict)?;
let strict_request = ReviewRequest::new(
selection.watched_agent,
selection.watched_model.clone(),
strict.arbiter_harness,
strict.arbiter_model.clone(),
false,
String::new(),
)
.with_effort(strict.arbiter_effort);
let batch_id = petition_batch_id("strict", &strict_pending[0].0.run_id);
{
let count = strict_pending.len();
for (index, (member, _)) in strict_pending.iter_mut().enumerate() {
member.batch_id = Some(batch_id.clone());
let batch_ref = BatchRunRef {
batch_id: batch_id.clone(),
kind: BatchKind::Petition,
member_index: index as u32,
member_count: count as u32,
fix_sha: fix_sha.clone(),
};
member.run.batch = Some(batch_ref.clone());
run_store.set_batch_metadata(&member.run_id, batch_ref)?;
queue.attach_batch_id_for_item(
&fix_sha,
member.item.petition_for.as_deref(),
&batch_id,
)?;
}
}
let petitions: Vec<&PetitionContext> = strict_pending
.iter()
.map(|(member, _)| &member.petition)
.collect();
let first_entries: Vec<&LedgerEntry> = strict_pending.iter().map(|(_, entry)| entry).collect();
let requested_shas: Vec<String> = petitions
.iter()
.map(|petition| petition.original_sha.clone())
.collect();
let prompt = strict_petition_batch_prompt(
&batch_id,
&petitions,
&claim,
&diff,
context,
&first_entries,
);
let mut strict_ready = Vec::new();
match run_petition_batch_pass(
queue.root.as_path(),
strict_request,
&prompt,
&batch_id,
&fix_sha,
&requested_shas,
runner,
"strict-running",
&strict_pending
.iter()
.map(|(member, _)| member)
.collect::<Vec<_>>(),
timeout,
current_dir,
) {
Ok((plan, mut parsed)) => {
for (member, first_entry) in strict_pending {
let original_sha = member.petition.original_sha.clone();
match parsed
.members
.remove(&original_sha)
.expect("strict batch parser omitted a requested member classification")
{
ParsedPetitionBatchMember::Valid(verdict) => {
let strict_entry = petition_entry_from_batch_verdict(
&fix_sha,
&claim,
&member.petition,
&plan,
&verdict,
&member.run_id,
member.batch_id.as_deref().unwrap_or(&batch_id),
);
let execution = ReviewExecution {
entries: vec![first_entry, strict_entry],
};
run_store.persist_pending_review(
&member.run_id,
PendingReviewSpool {
entries: execution.entries.clone(),
petition_transition_pending: true,
strict_pass_stdout: None,
},
)?;
strict_ready.push(ReadyPetitionMember { member, execution });
}
ParsedPetitionBatchMember::Invalid(message) => {
fail_petition_batch_member(
queue,
run_store,
&member,
message,
true,
&mut report,
)?;
}
}
}
}
Err(
error @ (ReviewerError::VerdictJson { .. } | ReviewerError::BatchVerdictSchema { .. }),
) => {
let message = error.to_string();
for (member, _) in strict_pending {
fail_petition_batch_member(queue, run_store, &member, &message, true, &mut report)?;
}
}
Err(error) if is_retryable_provider_failure(&error) => {
let message = error.to_string();
for (member, _) in &strict_pending {
// Keep queue rows and do not spend another petition attempt.
run_store.mark_provider_backoff(
&member.run_id,
&message,
DEFAULT_PROVIDER_BACKOFF_SECS,
)?;
}
finish_ready_petition_members(queue, run_store, ready, store, config, &mut report)?;
return Ok(report);
}
Err(error) => {
let message = error.to_string();
for (member, _) in &strict_pending {
fail_petition_batch_member(queue, run_store, member, &message, false, &mut report)?;
}
finish_ready_petition_members(queue, run_store, ready, store, config, &mut report)?;
return Err(error);
}
}
ready.extend(strict_ready);
finish_ready_petition_members(queue, run_store, ready, store, config, &mut report)?;
Ok(report)
}
fn dedupe_pending_for_batch(
pending: &[QueuedReview],
run_store: &ReviewRunStore,
) -> Result<Vec<QueuedReview>, ReviewerError> {
let mut seen = std::collections::BTreeSet::new();
let mut order = Vec::new();
for item in pending {
let identity = (item.commit_sha.clone(), item.petition_for.clone());
if seen.insert(identity) {
order.push(item.clone());
} else if !item.run_id.trim().is_empty() {
match run_store.read(&item.run_id) {
Ok(run) if run.status == ReviewRunStatus::Queued => {
run_store.cancel_queued(&item.run_id)?;
}
_ => {}
}
}
}
Ok(order)
}
fn planned_job_commit_sha(job: &PlannedDrainJob) -> &str {
match job {
PlannedDrainJob::Single(item) => &item.commit_sha,
PlannedDrainJob::PetitionBatch(items) => {
&items
.first()
.expect("planned petition batch is non-empty")
.commit_sha
}
}
}
fn planned_job_worktree_name(job: &PlannedDrainJob, index: usize) -> String {
let identity = match job {
PlannedDrainJob::Single(item) if !item.run_id.trim().is_empty() => item.run_id.clone(),
PlannedDrainJob::Single(item) => format!("commit-{}", item.commit_sha),
PlannedDrainJob::PetitionBatch(items) => {
let first = items.first().expect("planned petition batch is non-empty");
let petition = first.petition_for.as_deref().unwrap_or("batch");
format!("batch-{}-{}", first.commit_sha, petition)
}
};
format!("{index}-{}", identity.replace(['/', '\\'], "_"))
}
#[allow(clippy::too_many_arguments)]
fn drain_planned_job_in_dir<R: ProcessRunner, L: MaterialLoader>(
queue: &ReviewQueue,
job: PlannedDrainJob,
loader: &L,
selection: &ReviewSelection,
context: &str,
runner: &R,
store: &LedgerStore,
config: &config::TruthMirrorConfig,
current_dir: Option<&Path>,
) -> Result<DrainReport, ReviewerError> {
match job {
PlannedDrainJob::Single(item) => drain_pending_serial_in_dir(
queue,
vec![item],
loader,
selection,
context,
runner,
store,
config,
current_dir,
),
PlannedDrainJob::PetitionBatch(items) => {
let run_store = ReviewRunStore::new(&queue.root);
drain_petition_batch(
queue,
&run_store,
items,
loader,
selection,
context,
runner,
store,
config,
current_dir,
)
}
}
}
#[allow(clippy::too_many_arguments)]
fn drain_pending_concurrent<R, L>(
queue: &ReviewQueue,
pending: Vec<QueuedReview>,
loader: &L,
selection: &ReviewSelection,
context: &str,
runner: &R,
store: &LedgerStore,
config: &config::TruthMirrorConfig,
) -> Result<DrainReport, ReviewerError>
where
R: ProcessRunner + Sync,
L: MaterialLoader + Sync,
{
let repository = PathBuf::from(git_output(["rev-parse", "--show-toplevel"])?.trim());
let run_store = ReviewRunStore::new(&queue.root);
let pending = filter_pending_ready_for_attempt(pending, &run_store, unix_now())?;
let planned = if config.reviewer.batch_petitions {
let pending = dedupe_pending_for_batch(&pending, &run_store)?;
plan_batched_drain(
&pending,
&queue.root,
selection,
context,
config.reviewer.max_petition_batch_size,
)
} else {
pending.into_iter().map(PlannedDrainJob::Single).collect()
};
let workers = config
.reviewer
.max_concurrent_runs
.clamp(1, config::MAX_CONCURRENT_REVIEW_RUNS);
let mut report = DrainReport::default();
for (wave_start, wave) in planned.chunks(workers).enumerate() {
// Durable claim under state transactions BEFORE workers spawn so a crash
// leaves recoverable claimed/inflight records rather than silent loss.
let mut claimed_wave = Vec::with_capacity(wave.len());
for job in wave.iter().cloned() {
claimed_wave.push(claim_planned_job(queue, &run_store, job)?);
}
let (sender, receiver) = std::sync::mpsc::channel();
std::thread::scope(|scope| {
for (offset, job) in claimed_wave.into_iter().enumerate() {
let index = wave_start * workers + offset;
let worktree = queue
.root
.join("worktrees")
.join(planned_job_worktree_name(&job, index));
let commit_sha = planned_job_commit_sha(&job).to_owned();
let cleanup_repository = repository.clone();
let sender = sender.clone();
scope.spawn(move || {
// Create the detached checkout inside the worker so worktree
// materialization does not serialize the whole wave on the
// scheduler thread (and never shares the primary checkout).
let worktree = match create_review_worktree(
&cleanup_repository,
&worktree,
&commit_sha,
) {
Ok(path) => path,
Err(error) => {
let _ = sender.send((index, Err(error)));
return;
}
};
let worker_store = ReviewRunStore::new(&queue.root);
let _ = record_job_worktree_paths(&worker_store, &job, &worktree);
// Durable ledger/queue application finishes inside the drain
// call. Worktree cleanup must not precede that apply, and a
// cleanup failure must not discard an already-durable result.
let result = drain_planned_job_in_dir(
queue,
job.clone(),
loader,
selection,
context,
runner,
store,
config,
Some(worktree.as_path()),
);
clear_job_worktree_paths(&worker_store, &job);
if let Err(error) = remove_review_worktree(&cleanup_repository, &worktree)
&& result.is_ok()
{
eprintln!(
"truth-mirror watch: worktree cleanup failed after durable apply: {error}"
);
}
let _ = sender.send((index, result));
});
}
});
drop(sender);
let mut outcomes = receiver.into_iter().collect::<Vec<_>>();
outcomes.sort_by_key(|(index, _)| *index);
let mut first_error = None;
for (_, result) in outcomes {
match result {
Ok(completed) => {
report.reviewed.extend(completed.reviewed);
report.failed.extend(completed.failed);
report.ledger_entries += completed.ledger_entries;
}
Err(error) if first_error.is_none() => first_error = Some(error),
Err(_) => {}
}
}
if let Some(error) = first_error {
maintain_terminal_worktrees(&queue.root, &repository);
return Err(error);
}
}
// Join all claimed work before reporting, then sweep orphan worktrees.
maintain_terminal_worktrees(&queue.root, &repository);
Ok(report)
}
fn is_retryable_provider_failure(error: &ReviewerError) -> bool {
match error {
ReviewerError::ReviewerProcessFailed { status, stderr } => {
if *status == Some(429) {
return true;
}
let lowered = stderr.to_ascii_lowercase();
lowered.contains("rate limit")
|| lowered.contains("rate_limit")
|| lowered.contains("ratelimit")
|| lowered.contains("too many requests")
|| lowered.contains("resource_exhausted")
|| lowered.contains("overloaded")
|| lowered.contains("temporarily unavailable")
|| lowered.contains("try again later")
|| lowered.contains("retry later")
|| lowered.contains(" 429")
|| lowered.contains("\"429\"")
}
_ => false,
}
}
fn filter_pending_ready_for_attempt(
pending: Vec<QueuedReview>,
run_store: &ReviewRunStore,
now: u64,
) -> Result<Vec<QueuedReview>, ReviewerError> {
let mut ready = Vec::with_capacity(pending.len());
for item in pending {
if item.run_id.trim().is_empty() {
ready.push(item);
continue;
}
match run_store.read(&item.run_id) {
Ok(run) if run.backoff_blocks_attempt(now) => {
tracing::info!(
run_id = %item.run_id,
next_attempt_at_unix = ?run.next_attempt_at_unix,
"skipping queued review until provider backoff elapses"
);
}
Ok(_) | Err(ReviewerError::ReviewRunNotFound { .. }) => ready.push(item),
Err(error) => return Err(error),
}
}
Ok(ready)
}
fn claim_planned_job(
queue: &ReviewQueue,
run_store: &ReviewRunStore,
job: PlannedDrainJob,
) -> Result<PlannedDrainJob, ReviewerError> {
match job {
PlannedDrainJob::Single(mut item) => {
item = claim_queue_item(queue, run_store, item)?;
Ok(PlannedDrainJob::Single(item))
}
PlannedDrainJob::PetitionBatch(items) => {
let mut claimed = Vec::with_capacity(items.len());
for item in items {
claimed.push(claim_queue_item(queue, run_store, item)?);
}
Ok(PlannedDrainJob::PetitionBatch(claimed))
}
}
}
fn claim_queue_item(
queue: &ReviewQueue,
run_store: &ReviewRunStore,
mut item: QueuedReview,
) -> Result<QueuedReview, ReviewerError> {
let sha = item.commit_sha.clone();
let petition_for = item.petition_for.clone();
let (run_id, _) = resolve_run_id_for_item(run_store, &item)?;
if item.run_id.trim().is_empty() && !run_id.trim().is_empty() {
queue.attach_run_id_for_item(&sha, petition_for.as_deref(), &run_id)?;
item.run_id = run_id.clone();
} else if item.run_id.trim().is_empty() {
item.run_id = run_id.clone();
}
let target = if petition_for.is_some() {
"petition"
} else {
"commit"
};
run_store.ensure_queued(&item.run_id, &sha, target, petition_for)?;
let run = run_store.read(&item.run_id)?;
if run.status == ReviewRunStatus::Completed
|| run.status == ReviewRunStatus::Cancelled
|| run_ledger_phase_applied(&run, REVIEW_LEDGER_PHASE)
{
return Ok(item);
}
if run.status == ReviewRunStatus::Running && run.phase != CLAIMED_PHASE {
return Ok(item);
}
run_store.mark_claimed(&item.run_id)?;
Ok(item)
}
fn record_job_worktree_paths(
run_store: &ReviewRunStore,
job: &PlannedDrainJob,
worktree: &Path,
) -> Result<(), ReviewerError> {
let items = match job {
PlannedDrainJob::Single(item) => std::slice::from_ref(item),
PlannedDrainJob::PetitionBatch(items) => items.as_slice(),
};
let path = Some(worktree.display().to_string());
for item in items {
if item.run_id.trim().is_empty() {
continue;
}
let _ = run_store.set_worktree_path(&item.run_id, path.clone());
}
Ok(())
}
fn clear_job_worktree_paths(run_store: &ReviewRunStore, job: &PlannedDrainJob) {
let items = match job {
PlannedDrainJob::Single(item) => std::slice::from_ref(item),
PlannedDrainJob::PetitionBatch(items) => items.as_slice(),
};
for item in items {
if item.run_id.trim().is_empty() {
continue;
}
let _ = run_store.set_worktree_path(&item.run_id, None);
}
}
/// Remove detached worktrees that no longer belong to a live claimed/inflight run.
/// Cleanup failures are non-fatal telemetry only.
fn maintain_terminal_worktrees(state_dir: &Path, repository: &Path) {
let worktrees_root = state_dir.join("worktrees");
let entries = match fs::read_dir(&worktrees_root) {
Ok(entries) => entries,
Err(error) if error.kind() == io::ErrorKind::NotFound => return,
Err(error) => {
eprintln!("truth-mirror: orphan worktree maintenance skipped (non-fatal): {error}");
return;
}
};
let run_store = ReviewRunStore::new(state_dir);
let active = match run_store.list_lenient() {
Ok((runs, _)) => runs
.into_iter()
.filter(|run| run.status == ReviewRunStatus::Running)
.filter_map(|run| run.worktree_path)
.collect::<std::collections::BTreeSet<_>>(),
Err(error) => {
eprintln!("truth-mirror: orphan worktree maintenance skipped (non-fatal): {error}");
return;
}
};
for entry in entries.flatten() {
let path = entry.path();
if !path.is_dir() {
continue;
}
let key = path.display().to_string();
if active.contains(&key) {
continue;
}
match remove_review_worktree(repository, &path) {
Ok(()) => {
eprintln!(
"truth-mirror: removed orphan review worktree (non-fatal maintenance): {key}"
);
}
Err(error) => {
eprintln!(
"truth-mirror: orphan worktree cleanup failed (non-fatal): {key}: {error}"
);
}
}
}
}
fn create_review_worktree(
repository: &Path,
worktree: &Path,
commit_sha: &str,
) -> Result<PathBuf, ReviewerError> {
if worktree.exists() {
remove_review_worktree(repository, worktree)?;
}
if let Some(parent) = worktree.parent() {
fs::create_dir_all(parent).map_err(ReviewerError::RunIo)?;
}
let output = Command::new("git")
.current_dir(repository)
.args(["worktree", "add", "--detach"])
.arg(worktree)
.arg(commit_sha)
.output()
.map_err(ReviewerError::GitSpawn)?;
if !output.status.success() {
return Err(ReviewerError::GitFailed {
args: vec!["worktree".to_owned(), "add".to_owned()],
stderr: String::from_utf8_lossy(&output.stderr).into_owned(),
});
}
Ok(worktree.to_path_buf())
}
fn remove_review_worktree(repository: &Path, worktree: &Path) -> Result<(), ReviewerError> {
if !worktree.exists() {
return Ok(());
}
let output = Command::new("git")
.current_dir(repository)
.args(["worktree", "remove", "--force"])
.arg(worktree)
.output()
.map_err(ReviewerError::GitSpawn)?;
if !output.status.success() {
return Err(ReviewerError::GitFailed {
args: vec!["worktree".to_owned(), "remove".to_owned()],
stderr: String::from_utf8_lossy(&output.stderr).into_owned(),
});
}
Ok(())
}
pub fn drain_once<R: ProcessRunner, L: MaterialLoader>(
queue: &ReviewQueue,
loader: &L,
selection: &ReviewSelection,
context: &str,
runner: &R,
store: &LedgerStore,
config: &config::TruthMirrorConfig,
) -> Result<DrainReport, ReviewerError> {
let pending = queue.pending()?;
if !config.reviewer.batch_petitions {
return drain_pending_serial(
queue, pending, loader, selection, context, runner, store, config,
);
}
let run_store = ReviewRunStore::new(&queue.root);
let pending = dedupe_pending_for_batch(&pending, &run_store)?;
let planned = plan_batched_drain(
&pending,
&queue.root,
selection,
context,
config.reviewer.max_petition_batch_size,
);
let mut report = DrainReport::default();
for job in planned {
let completed = match job {
PlannedDrainJob::Single(item) => drain_pending_serial(
queue,
vec![item],
loader,
selection,
context,
runner,
store,
config,
)?,
PlannedDrainJob::PetitionBatch(items) => drain_petition_batch(
queue, &run_store, items, loader, selection, context, runner, store, config, None,
)?,
};
report.reviewed.extend(completed.reviewed);
report.failed.extend(completed.failed);
report.ledger_entries += completed.ledger_entries;
}
Ok(report)
}
pub fn drain_once_parallel<R, L>(
queue: &ReviewQueue,
loader: &L,
selection: &ReviewSelection,
context: &str,
runner: &R,
store: &LedgerStore,
config: &config::TruthMirrorConfig,
) -> Result<DrainReport, ReviewerError>
where
R: ProcessRunner + Sync,
L: MaterialLoader + Sync,
{
drain_pending_concurrent(
queue,
queue.pending()?,
loader,
selection,
context,
runner,
store,
config,
)
}
#[allow(clippy::too_many_arguments)]
fn drain_pending_serial<R: ProcessRunner, L: MaterialLoader>(
queue: &ReviewQueue,
pending: Vec<QueuedReview>,
loader: &L,
selection: &ReviewSelection,
context: &str,
runner: &R,
store: &LedgerStore,
config: &config::TruthMirrorConfig,
) -> Result<DrainReport, ReviewerError> {
drain_pending_serial_in_dir(
queue, pending, loader, selection, context, runner, store, config, None,
)
}
#[allow(clippy::too_many_arguments)]
fn drain_pending_serial_in_dir<R: ProcessRunner, L: MaterialLoader>(
queue: &ReviewQueue,
pending: Vec<QueuedReview>,
loader: &L,
selection: &ReviewSelection,
context: &str,
runner: &R,
store: &LedgerStore,
config: &config::TruthMirrorConfig,
current_dir: Option<&Path>,
) -> Result<DrainReport, ReviewerError> {
let run_store = ReviewRunStore::new(&queue.root);
// Per-review wall-clock limit from config; a wedged reviewer is killed at
// this deadline and the run recorded as failed (see the drain loop below).
let timeout = config.reviewer.timeout();
let pending = filter_pending_ready_for_attempt(pending, &run_store, unix_now())?;
// Queue identity is (commit_sha, petition_for), NOT the bare SHA: a fix
// commit's own post-commit review and a petition review of that same fix
// (or two petitions for different rejections sharing one fix) are
// distinct jobs — deduping on SHA alone silently cancelled one of them.
let mut seen = std::collections::BTreeSet::new();
let mut order = Vec::new();
for item in &pending {
let identity = (item.commit_sha.clone(), item.petition_for.clone());
if seen.insert(identity) {
order.push(item.clone());
} else if !item.run_id.trim().is_empty() {
match run_store.read(&item.run_id) {
Ok(run) if run.status == ReviewRunStatus::Queued => {
run_store.cancel_queued(&item.run_id)?;
}
_ => {}
}
}
}
let mut report = DrainReport::default();
for item in order {
let sha = item.commit_sha.clone();
let petition_for = item.petition_for.clone();
let (run_id, created_new_run) = resolve_run_id_for_item(&run_store, &item)?;
if item.run_id.trim().is_empty() && !run_id.trim().is_empty() {
queue.attach_run_id_for_item(&sha, petition_for.as_deref(), &run_id)?;
}
let target = if petition_for.is_some() {
"petition"
} else {
"commit"
};
let run = match run_store.ensure_queued(&run_id, &sha, target, petition_for.clone()) {
Ok(run) => run,
Err(ReviewerError::RunJson(_)) => {
queue.remove_item(&sha, petition_for.as_deref())?;
report.failed.push(sha);
continue;
}
Err(error) => return Err(error),
};
let cancel_new_run = || {
if created_new_run {
let _ = run_store.cancel_queued(&run_id);
}
};
if run.status == ReviewRunStatus::Cancelled {
cancel_new_run();
queue.remove_item(&sha, item.petition_for.as_deref())?;
continue;
}
if run.status == ReviewRunStatus::Completed {
finish_replayed_queue_item(
queue,
&run_store,
&run_id,
&sha,
item.petition_for.as_deref(),
run.ledger_entries,
&mut report,
)?;
continue;
}
if run_ledger_phase_applied(&run, REVIEW_LEDGER_PHASE) {
finish_replayed_queue_item(
queue,
&run_store,
&run_id,
&sha,
item.petition_for.as_deref(),
run.ledger_entries,
&mut report,
)?;
continue;
}
let (claim, diff) = match loader.load(&sha) {
Ok(material) => material,
Err(error) => {
cancel_new_run();
run_store.mark_failed(&run_id, error.to_string())?;
queue.remove_item(&sha, item.petition_for.as_deref())?;
report.failed.push(sha);
continue;
}
};
// Petition items rebuild the original rejection's payload from the
// ledger so draft_review_job can swap in the petition prompt and
// apply_petition_transition can close (or escalate) the rejection.
let petition = match &item.petition_for {
Some(original_sha) => {
if !petition_target_is_actionable(store, original_sha)? {
// The rejection this petition targets is already resolved
// (or was never recorded) — 0.9.2 crashed the watcher here
// via NoOpenRejection when a petition was enqueued twice.
// Consume the dead letter, note the skip, and drain on.
let notice = format!(
"petition target {original_sha} has no open rejection (already resolved or unknown); skipping queued petition for {sha}"
);
eprintln!("truth-mirror watch: {notice}");
run_store.mark_failed(&run_id, format!("skipped: {notice}"))?;
queue.remove_item(&sha, item.petition_for.as_deref())?;
report.failed.push(sha);
continue;
}
Some(petition_context_from_ledger(original_sha, &sha, store)?)
}
None => None,
};
let prompt = first_pass_prompt(&claim, &diff, context);
let job = materialize_petition_prompt(ReviewJob {
commit_sha: sha.clone(),
claim,
diff,
context: context.to_owned(),
request: selection.request_for(prompt),
strict: selection.strict.clone(),
petition,
});
let execution = {
// Durable claim precedes reviewer execution (serial and concurrent).
if run.status != ReviewRunStatus::Running && run.phase != CLAIMED_PHASE {
run_store.mark_claimed(&run_id)?;
}
if run.status != ReviewRunStatus::Running || run.phase != REVIEW_LEDGER_PHASE {
run_store.mark_running(&run_id, REVIEW_LEDGER_PHASE)?;
}
match complete_review_execution_in_dir(
&job,
runner,
&run_store,
&run_id,
run.pending_review,
timeout,
current_dir,
) {
Ok(execution) => execution,
Err(
error @ (ReviewerError::VerdictJson { .. }
| ReviewerError::VerdictSchema { .. }),
) => {
run_store.mark_failed(&run_id, error.to_string())?;
queue.remove_item(&sha, item.petition_for.as_deref())?;
report.failed.push(sha);
continue;
}
// A wedged reviewer was killed at the configured timeout (or its
// output could not be drained trustworthily, or the kill itself
// failed): record the run as failed, consume the item (requeuing
// would just wedge again), and keep draining — one hung review
// never freezes the queue again.
Err(
error @ (ReviewerError::ReviewerTimeout { .. }
| ReviewerError::ReviewerOutputWedged { .. }
| ReviewerError::KillReviewer { .. }),
) => {
run_store.mark_failed(&run_id, error.to_string())?;
queue.remove_item(&sha, item.petition_for.as_deref())?;
report.failed.push(sha);
continue;
}
Err(error) if is_retryable_provider_failure(&error) => {
// Persist backoff without queue removal or petition attempt spend.
run_store.mark_provider_backoff(
&run_id,
error.to_string(),
DEFAULT_PROVIDER_BACKOFF_SECS,
)?;
continue;
}
Err(error) => {
let _ = run_store.mark_failed(&run_id, error.to_string());
return Err(error);
}
}
};
// Track A's idempotent apply records the verdict and no-ops the
// transition if the target was resolved mid-review (human waive /
// duplicate petition). That keeps the watcher alive without needing
// a NoOpenRejection skip arm on this path.
apply_review_execution(&job, &execution, store)?;
run_store.mark_ledger_applied(&run_id, REVIEW_LEDGER_PHASE, execution.entries.len())?;
run_store.clear_pending_review(&run_id)?;
record_memory_skill_outcome(
&queue.root,
config,
&run_id,
&sha,
"watch-drain",
&execution.entries,
);
report.ledger_entries += execution.entries.len();
run_store.mark_completed(&run_id, execution.entries.len())?;
// Remove exactly THIS item — remove_sha would also delete a sibling
// row for the same SHA with a different petition identity.
queue.remove_item(&sha, item.petition_for.as_deref())?;
report.reviewed.push(sha);
}
Ok(report)
}
fn record_memory_skill_outcome(
state_dir: &Path,
config: &config::TruthMirrorConfig,
run_id: &str,
commit_sha: &str,
phase: &str,
entries: &[LedgerEntry],
) {
if !is_full_git_sha(commit_sha) {
tracing::debug!(
run_id = %run_id,
commit_sha = %commit_sha,
phase = %phase,
"memory-skill extraction skipped for non-commit review target"
);
return;
}
match crate::memory_skill::evaluate_review_completion(state_dir, config, run_id, entries) {
Ok(report) => {
let outcome = if !report.staged.is_empty() {
"staged"
} else if !report.advised.is_empty() {
"advised"
} else {
"skipped"
};
tracing::info!(
run_id = %run_id,
commit_sha = %commit_sha,
phase = %phase,
memory_skill_outcome = outcome,
skipped_reason = report.skipped_reason.as_deref().unwrap_or(""),
occurrence_count = report.occurrence_count,
nearest_cluster_size = report.nearest_cluster_size,
staged_count = report.staged.len(),
advised_count = report.advised.len(),
"memory-skill evaluation completed"
);
}
Err(crate::memory_skill::MemorySkillError::ScanRejected { reason }) => {
tracing::info!(
run_id = %run_id,
commit_sha = %commit_sha,
phase = %phase,
memory_skill_outcome = "scan_rejected",
skipped_reason = %reason,
reason = %reason,
"memory-skill extraction skipped by scan gate"
);
}
Err(error) => {
let error_kind = memory_skill_error_kind(&error);
tracing::warn!(
run_id = %run_id,
commit_sha = %commit_sha,
phase = %phase,
memory_skill_outcome = "failed",
memory_skill_error_kind = error_kind,
error = %error,
"memory-skill extraction failed after review completion"
);
}
}
}
fn memory_skill_error_kind(error: &crate::memory_skill::MemorySkillError) -> &'static str {
match error {
crate::memory_skill::MemorySkillError::Io(_) => "io",
crate::memory_skill::MemorySkillError::Json(_) => "json",
crate::memory_skill::MemorySkillError::Ledger(_) => "ledger",
crate::memory_skill::MemorySkillError::CandidateNotFound { .. } => "candidate_not_found",
crate::memory_skill::MemorySkillError::AdvisoryNotFound { .. } => "advisory_not_found",
crate::memory_skill::MemorySkillError::EmptyRejectReason => "empty_reject_reason",
crate::memory_skill::MemorySkillError::EmptySupersedeReason => "empty_supersede_reason",
crate::memory_skill::MemorySkillError::SelfSupersede { .. } => "self_supersede",
crate::memory_skill::MemorySkillError::InvalidSupersedeReplacement { .. } => {
"invalid_supersede_replacement"
}
crate::memory_skill::MemorySkillError::InvalidTransition { .. } => "invalid_transition",
crate::memory_skill::MemorySkillError::TransitionLocked { .. } => "transition_locked",
crate::memory_skill::MemorySkillError::UnsafeGlobalWrite { .. } => "unsafe_global_write",
crate::memory_skill::MemorySkillError::UnsafeApprovedPath { .. } => "unsafe_approved_path",
crate::memory_skill::MemorySkillError::UnsafeStatePath { .. } => "unsafe_state_path",
crate::memory_skill::MemorySkillError::ScanRejected { .. } => "scan_rejected",
crate::memory_skill::MemorySkillError::RenderedSkillTooLarge { .. } => {
"rendered_skill_too_large"
}
}
}
/// Build the ground-truth + trajectory context block for review prompts.
/// Best-effort: an unavailable repo or provider yields an empty block.
fn review_context(config: &config::TruthMirrorConfig) -> String {
let repo_root = match git_output(["rev-parse", "--show-toplevel"]) {
Ok(root) => PathBuf::from(root.trim()),
Err(_) => return String::new(),
};
let provider = crate::context::trajectory_provider(&repo_root, &config.history);
crate::context::build_review_context(
&repo_root,
&config.ground_truth,
&config.history,
Some(provider.as_ref()),
)
.unwrap_or_default()
}
pub fn run_watch_command(
args: cli::WatchArgs,
state_dir: &Path,
config: &config::TruthMirrorConfig,
) -> Result<ExitCode> {
let selection = ReviewSelection::resolve(
args.watched_agent,
args.watched_model,
args.reviewer_harness,
args.reviewer_model,
args.reviewer_effort,
args.allow_same_model,
config,
)?;
let queue = ReviewQueue::new(state_dir);
let store = LedgerStore::new(state_dir);
let loader = GitMaterialLoader::from_config(config);
let runner = StdProcessRunner;
let interval = std::time::Duration::from_secs(args.poll_secs.max(1));
// EVERY queue-draining mode — looping, --until-empty, AND --once — must
// OWN the single-flight lock before touching the queue. 0.9.2 let a
// second watcher proceed "without lock ownership": two watchers drained
// concurrently, double-processed a petition, and the first died on the
// already-resolved rejection. A --once drain races concurrent drains
// exactly like a loop does (same queue items, same run rows, same queue
// file rewrite), so it is covered by the same contract: take the lock
// (waiting with --wait-for-lock) or fail loudly, NEVER process without
// it.
match claim_watcher_lock(
state_dir,
args.wait_for_lock,
interval,
args.expect_lock_handoff,
) {
Ok(WatchLockClaim::Owned) => {}
Ok(WatchLockClaim::Refused(reason)) => {
eprintln!("truth-mirror watch: {reason}");
return Ok(ExitCode::FAILURE);
}
Err(error) => {
eprintln!(
"truth-mirror watch: failed to acquire the watcher lock ({error}); refusing to process the queue without lock ownership"
);
return Ok(ExitCode::FAILURE);
}
}
let watcher_lock_token = crate::watcher::lock_acquisition_token(state_dir).unwrap_or_default();
// Reap orphaned `running` rows left by a previously killed watcher so this
// watcher's lifecycle starts from the truth (ensure-watcher does the same
// before its spawn decision; any manual watch must not skip self-healing).
let _ = ReviewRunStore::new(state_dir).reconcile_stale_runs();
if args.once {
let context = review_context(config);
let result = if config.reviewer.max_concurrent_runs > 1 {
drain_once_parallel(
&queue, &loader, &selection, &context, &runner, &store, config,
)
} else {
drain_once(
&queue, &loader, &selection, &context, &runner, &store, config,
)
};
// A one-shot drain owns its exit: release the slot so the next
// ensure-watcher sees the truth immediately instead of waiting out
// the staleness window.
let _ = crate::watcher::release_lock_if_owned(state_dir, &watcher_lock_token);
let report = result?;
println!(
"truth-mirror watch: reviewed {} commit(s), skipped {} failed item(s), wrote {} ledger entrie(s)",
report.reviewed.len(),
report.failed.len(),
report.ledger_entries
);
return Ok(ExitCode::SUCCESS);
}
if args.until_empty {
return run_until_empty(
&queue,
&loader,
&selection,
&runner,
&store,
config,
state_dir,
args.grace,
interval,
&watcher_lock_token,
);
}
let outcome = loop {
// Rebuild context each poll so ground truth and trajectory stay current.
let context = review_context(config);
let report = match if config.reviewer.max_concurrent_runs > 1 {
drain_once_parallel(
&queue, &loader, &selection, &context, &runner, &store, config,
)
} else {
drain_once(
&queue, &loader, &selection, &context, &runner, &store, config,
)
} {
Ok(report) => report,
Err(error) => break Err(error),
};
if !report.reviewed.is_empty() {
println!(
"truth-mirror watch: reviewed {} commit(s)",
report.reviewed.len()
);
}
if !report.failed.is_empty() {
eprintln!(
"truth-mirror watch: skipped {} failed queue item(s)",
report.failed.len()
);
}
std::thread::sleep(interval);
};
// Only reachable via a drain error: release our slot so a crashed loop
// never wedges the next spawn.
let _ = crate::watcher::release_lock_if_owned(state_dir, &watcher_lock_token);
outcome?;
Ok(ExitCode::SUCCESS)
}
/// Outcome of claiming the single-flight lock for a looping `watch`.
#[derive(Clone, Debug, Eq, PartialEq)]
enum WatchLockClaim {
/// This process owns the lock: a fresh claim, a reclaimed stale slot, or
/// the slot `ensure-watcher` handed to the child it spawned.
Owned,
/// A DIFFERENT live watcher holds the lock and `--wait-for-lock` was not
/// given. The caller must exit without touching the queue.
Refused(String),
}
/// Claim the watcher lock for a queue-draining `watch`, or refuse.
///
/// `try_acquire_lock` reports `HeldByLiveWatcher` both when a foreign watcher
/// owns the slot AND when `ensure-watcher` pre-recorded THIS process as the
/// owner (it re-points the lock at the detached child right after spawning
/// it), so ownership is confirmed by comparing the recorded identity against
/// the current process. With `wait_for_lock`, a foreign-held lock is retried
/// every `interval` until the incumbent exits and the slot is reclaimed.
///
/// `expect_handoff` is set only on the detached child ensure-watcher spawns:
/// the parent re-points the lock at the child within milliseconds of the
/// spawn, and a child that checked BEFORE the re-point landed would see its
/// (still-live) parent's identity and refuse — stranding the queue once the
/// parent exits. The child absorbs that window by polling for the handoff up
/// to [`HANDOFF_GRACE`]; the grace outlives the lock settling window, so a
/// parent killed mid-handoff leaves a reclaimable stale lock instead of a
/// stranded queue.
fn claim_watcher_lock(
state_dir: &Path,
wait_for_lock: bool,
interval: std::time::Duration,
expect_handoff: bool,
) -> Result<WatchLockClaim, crate::watcher::WatcherError> {
let mut announced_wait = false;
let handoff_deadline = expect_handoff.then(|| Instant::now() + HANDOFF_GRACE);
let mut announced_handoff_wait = false;
loop {
match crate::watcher::try_acquire_lock(state_dir)? {
crate::watcher::LockClaim::Acquired => return Ok(WatchLockClaim::Owned),
crate::watcher::LockClaim::HeldByLiveWatcher => {
if crate::watcher::lock_owned_by_current_process(state_dir) {
// Ownership handoff from ensure-watcher, not contention.
return Ok(WatchLockClaim::Owned);
}
if let Some(deadline) = handoff_deadline
&& Instant::now() < deadline
{
if !announced_handoff_wait {
tracing::info!("watch: waiting for the ensure-watcher lock handoff");
announced_handoff_wait = true;
}
std::thread::sleep(HANDOFF_POLL_INTERVAL);
continue;
}
if !wait_for_lock {
return Ok(WatchLockClaim::Refused(
"another live watcher holds the watcher lock; exiting instead of \
processing the queue without lock ownership (pass --wait-for-lock \
to wait for the slot)"
.to_owned(),
));
}
if !announced_wait {
eprintln!(
"truth-mirror watch: another live watcher holds the lock; waiting for it (--wait-for-lock)"
);
announced_wait = true;
}
std::thread::sleep(interval);
}
}
}
}
/// How long a handoff-expecting child polls for ensure-watcher to re-point
/// the lock at it before falling back to the normal refuse/wait path. Sized
/// to outlive the lock settling window ([`crate::watcher::LOCK_SETTLING_SECS`])
/// so the parent-killed-mid-handoff case resolves to a reclaimable stale
/// lock rather than an immediate refusal that strands the queue.
const HANDOFF_GRACE: Duration = Duration::from_secs(crate::watcher::LOCK_SETTLING_SECS + 2);
/// Poll cadence while absorbing the ensure-watcher handoff window. Small
/// because the re-point lands within milliseconds of the spawn.
const HANDOFF_POLL_INTERVAL: Duration = Duration::from_millis(100);
/// `--until-empty` decision after a drain: given how long the queue has been
/// continuously empty (`empty_for_secs`) and the grace window, decide whether the
/// watcher should exit.
///
/// Pure so the grace-window policy is unit-testable without real clocks. `None`
/// means the queue is not empty (keep draining); `Some(elapsed)` carries how long
/// it has been empty so far.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum UntilEmptyDecision {
/// The queue has items, or the empty window has not yet spanned the grace
/// period — keep polling.
KeepWatching,
/// The queue stayed empty through a full grace window — exit 0.
Exit,
}
pub fn until_empty_decision(empty_for_secs: Option<u64>, grace_secs: u64) -> UntilEmptyDecision {
match empty_for_secs {
Some(elapsed) if elapsed >= grace_secs => UntilEmptyDecision::Exit,
_ => UntilEmptyDecision::KeepWatching,
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
enum QueueFingerprint {
Missing,
Present {
len: u64,
modified: Option<SystemTime>,
},
}
#[derive(Clone, Debug, Default)]
struct QueueEmptyCache {
fingerprint: Option<QueueFingerprint>,
pending_count: Option<usize>,
}
fn queue_fingerprint(queue: &ReviewQueue) -> Result<QueueFingerprint, ReviewerError> {
match fs::metadata(queue.path()) {
Ok(metadata) => Ok(QueueFingerprint::Present {
len: metadata.len(),
modified: metadata.modified().ok(),
}),
Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(QueueFingerprint::Missing),
Err(error) => Err(ReviewerError::QueueIo(error)),
}
}
fn queue_has_pending_cached(
queue: &ReviewQueue,
cache: &mut QueueEmptyCache,
) -> Result<bool, ReviewerError> {
let fingerprint = queue_fingerprint(queue)?;
if matches!(
fingerprint,
QueueFingerprint::Missing | QueueFingerprint::Present { len: 0, .. }
) {
cache.fingerprint = Some(fingerprint);
cache.pending_count = Some(0);
return Ok(false);
}
if let (true, Some(pending_count)) = (
cache.fingerprint.as_ref() == Some(&fingerprint),
cache.pending_count,
) {
return Ok(pending_count > 0);
}
let pending_count = queue.summary()?.pending_count;
cache.fingerprint = Some(fingerprint);
cache.pending_count = Some(pending_count);
Ok(pending_count > 0)
}
/// Drain the queue until it stays empty through a full grace window, then exit 0.
///
/// The caller ([`run_watch_command`]) has already claimed the single-flight
/// lock; this releases it on exit so `ensure-watcher` can tell a live watcher
/// from a dead one. Late arrivals inside the grace window reset the empty
/// timer, so a claim enqueued a second before the watcher would have exited
/// is still picked up.
#[allow(clippy::too_many_arguments)]
fn run_until_empty<R: ProcessRunner + Sync, L: MaterialLoader + Sync>(
queue: &ReviewQueue,
loader: &L,
selection: &ReviewSelection,
runner: &R,
store: &LedgerStore,
config: &config::TruthMirrorConfig,
state_dir: &Path,
grace_secs: u64,
interval: std::time::Duration,
watcher_lock_token: &str,
) -> Result<ExitCode> {
let outcome = (|| -> Result<()> {
let mut empty_since: Option<u64> = None;
let mut queue_empty_cache = QueueEmptyCache::default();
loop {
let context = review_context(config);
let report = if config.reviewer.max_concurrent_runs > 1 {
drain_once_parallel(queue, loader, selection, &context, runner, store, config)?
} else {
drain_once(queue, loader, selection, &context, runner, store, config)?
};
if !report.reviewed.is_empty() {
println!(
"truth-mirror watch: reviewed {} commit(s)",
report.reviewed.len()
);
}
if !report.failed.is_empty() {
eprintln!(
"truth-mirror watch: skipped {} failed queue item(s)",
report.failed.len()
);
}
let now = unix_now();
if !queue_has_pending_cached(queue, &mut queue_empty_cache)? {
let started = *empty_since.get_or_insert(now);
let empty_for = now.saturating_sub(started);
if until_empty_decision(Some(empty_for), grace_secs) == UntilEmptyDecision::Exit {
return Ok(());
}
} else {
// A late arrival landed — reset the grace timer.
empty_since = None;
}
std::thread::sleep(interval);
}
})();
// Always release our lock, even on error, so a crashed drain never wedges the
// next spawn (the staleness check would recover it anyway, but this is tidy).
let _ = crate::watcher::release_lock_if_owned(state_dir, watcher_lock_token);
outcome?;
Ok(ExitCode::SUCCESS)
}
/// `ensure-watcher`: idempotent check-and-spawn of the queue-tied watcher.
///
/// Fast and quiet on success (safe to call from a git hook after every enqueue).
/// Prints one line only when it actually starts a watcher, so hook output stays
/// clean when a watcher already exists.
pub fn run_ensure_watcher_command(
_args: cli::EnsureWatcherArgs,
state_dir: &Path,
) -> Result<ExitCode> {
match crate::watcher::ensure_watcher(state_dir)? {
crate::watcher::LockClaim::Acquired => {
println!("truth-mirror: watcher started");
}
crate::watcher::LockClaim::HeldByLiveWatcher => {
tracing::debug!("ensure-watcher: live watcher already owns the state dir");
}
}
Ok(ExitCode::SUCCESS)
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct StrictGoalPolicy {
pub stop_after_lies: u32,
pub stop_after_fuckups: u32,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct StrictGoalCounters {
pub lies_exposed: u32,
pub fuckups_registered: u32,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum StrictGoalDecision {
Continue,
Stop { reason: StrictGoalStopReason },
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum StrictGoalStopReason {
LiesExposed,
FuckupsRegistered,
}
impl StrictGoalPolicy {
pub fn decide(&self, counters: StrictGoalCounters) -> StrictGoalDecision {
if self.stop_after_lies > 0 && counters.lies_exposed >= self.stop_after_lies {
return StrictGoalDecision::Stop {
reason: StrictGoalStopReason::LiesExposed,
};
}
if self.stop_after_fuckups > 0 && counters.fuckups_registered >= self.stop_after_fuckups {
return StrictGoalDecision::Stop {
reason: StrictGoalStopReason::FuckupsRegistered,
};
}
StrictGoalDecision::Continue
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct StrictGoalOutcome {
pub passes: u32,
pub counters: StrictGoalCounters,
/// `None` means the loop stopped at the `max_passes` ceiling rather than
/// hitting a configured lie/fuckup threshold.
pub stop_reason: Option<StrictGoalStopReason>,
pub entries: Vec<LedgerEntry>,
}
impl StrictGoalOutcome {
pub fn stop_reason_suffix(&self) -> &'static str {
match self.stop_reason {
Some(StrictGoalStopReason::LiesExposed) => " (stopped: lies exposed)",
Some(StrictGoalStopReason::FuckupsRegistered) => " (stopped: fuckups registered)",
None => " (stopped: max passes)",
}
}
}
/// Sic the adversarial reviewer on a commit in a loop, accumulating exposed lies
/// (REJECT verdicts) and registered fuckups (individual findings). Every pass is
/// recorded in the ledger. The loop stops when `policy` says the configured `N`
/// is reached, or when `max_passes` is hit so an honest agent still terminates.
#[allow(clippy::too_many_arguments)]
pub fn run_strict_goal_loop<R: ProcessRunner>(
commit_sha: &str,
claim: &Claim,
diff: &str,
context: &str,
selection: &ReviewSelection,
policy: StrictGoalPolicy,
max_passes: u32,
runner: &R,
store: &LedgerStore,
timeout: Option<Duration>,
) -> Result<StrictGoalOutcome, ReviewerError> {
let ceiling = max_passes.max(1);
let mut outcome = StrictGoalOutcome {
passes: 0,
counters: StrictGoalCounters {
lies_exposed: 0,
fuckups_registered: 0,
},
stop_reason: None,
entries: Vec::new(),
};
while outcome.passes < ceiling {
let prompt = strict_goal_prompt(claim, diff, context, outcome.passes + 1, &outcome.entries);
let request = selection.request_for(prompt);
let plan = ReviewPlan::build(request.clone())?;
let output = plan.run_with(&request.prompt, runner, timeout)?;
ensure_process_success(&output)?;
let verdict = ParsedVerdict::parse(&output.stdout)?;
let job = ReviewJob {
commit_sha: commit_sha.to_owned(),
claim: claim.clone(),
diff: diff.to_owned(),
context: context.to_owned(),
request,
strict: None,
petition: None,
};
let entry = entry_from_verdict(&job, &plan, &verdict);
store.append_entry(&entry)?;
outcome.entries.push(entry);
outcome.passes += 1;
if verdict.verdict == Verdict::Reject {
outcome.counters.lies_exposed += 1;
}
outcome.counters.fuckups_registered = outcome
.counters
.fuckups_registered
.saturating_add(u32::try_from(verdict.findings.len()).unwrap_or(u32::MAX));
if let StrictGoalDecision::Stop { reason } = policy.decide(outcome.counters) {
outcome.stop_reason = Some(reason);
break;
}
}
Ok(outcome)
}
fn strict_goal_prompt(
claim: &Claim,
diff: &str,
context: &str,
pass: u32,
prior: &[LedgerEntry],
) -> String {
let prior_findings: Vec<String> = prior
.iter()
.flat_map(|entry| entry.findings.clone())
.collect();
let prior_block = if prior_findings.is_empty() {
"(none)".to_owned()
} else {
prior_findings.join("\n")
};
format!(
"{ADVERSARIAL_PREAMBLE}\n\nStrict-goal loop, pass {pass}. Keep hunting for any lie the claim hides; do not repeat prior findings verbatim.{}\n\nCLAIM:\n{}\n\nPRIOR FINDINGS:\n{prior_block}\n\nDIFF:\n{}",
context_block(context),
claim.to_line(),
diff
)
}
pub fn run_review_command(
args: cli::ReviewArgs,
state_dir: &Path,
config: &config::TruthMirrorConfig,
) -> Result<ExitCode> {
if let Some(command) = args.command {
return run_review_run_command(command, state_dir);
}
let material = ReviewMaterial::load(
&args,
state_dir,
&config.gates.to_policy().evidence_patterns,
)?;
let mut selection = ReviewSelection::resolve(
args.watched_agent,
args.watched_model,
args.reviewer_harness,
args.reviewer_model,
args.reviewer_effort,
args.allow_same_model,
config,
)?;
if args.strict_two_pass {
selection.strict = Some(ReviewSelection::resolve_arbiter(
selection.watched_agent,
args.arbiter_harness,
args.arbiter_model,
args.arbiter_effort,
config,
)?);
}
let store = LedgerStore::new(state_dir);
let run_store = ReviewRunStore::new(state_dir);
let context = review_context(config);
let run = run_store.create_queued(&material.commit_sha, material.target_label.clone())?;
run_store.mark_running(&run.id, "reviewing")?;
if args.strict_goal {
let policy = config
.strict
.goal_policy(args.stop_after_lies, args.stop_after_fuckups);
let max_passes = args.max_passes.unwrap_or(config.strict.max_passes);
let outcome = match run_strict_goal_loop(
&material.commit_sha,
&material.claim,
&material.diff,
&context,
&selection,
policy,
max_passes,
&StdProcessRunner,
&store,
config.reviewer.timeout(),
) {
Ok(outcome) => outcome,
Err(error) => {
let _ = run_store.mark_failed(&run.id, error.to_string());
return Err(error.into());
}
};
record_memory_skill_outcome(
state_dir,
config,
&run.id,
&material.commit_sha,
"strict-goal",
&outcome.entries,
);
run_store.mark_completed(&run.id, outcome.entries.len())?;
println!(
"truth-mirror strict-goal: run {}, {} pass(es), {} lie(s), {} fuckup(s){}",
run.id,
outcome.passes,
outcome.counters.lies_exposed,
outcome.counters.fuckups_registered,
outcome.stop_reason_suffix(),
);
return Ok(ExitCode::SUCCESS);
}
let prompt = first_pass_prompt(&material.claim, &material.diff, &context);
let commit_sha = material.commit_sha.clone();
let job = ReviewJob {
commit_sha: material.commit_sha,
claim: material.claim,
diff: material.diff,
context,
request: selection.request_for(prompt),
strict: selection.strict.clone(),
petition: None,
};
let strict_two_pass = selection.strict.is_some();
let job = materialize_petition_prompt(job);
let timeout = config.reviewer.timeout();
let execution = if strict_two_pass {
run_store.mark_running(&run.id, REVIEW_LEDGER_PHASE)?;
let run = run_store.read(&run.id)?;
match complete_review_execution(
&job,
&StdProcessRunner,
&run_store,
&run.id,
run.pending_review,
timeout,
) {
Ok(execution) => execution,
Err(error) => {
let _ = run_store.mark_failed(&run.id, error.to_string());
return Err(error.into());
}
}
} else {
match execute_review_job(job.clone(), &StdProcessRunner, &store, timeout) {
Ok(execution) => execution,
Err(error) => {
let _ = run_store.mark_failed(&run.id, error.to_string());
return Err(error.into());
}
}
};
if strict_two_pass {
apply_review_execution(&job, &execution, &store)?;
run_store.mark_ledger_applied(&run.id, REVIEW_LEDGER_PHASE, execution.entries.len())?;
run_store.clear_pending_review(&run.id)?;
}
record_memory_skill_outcome(
state_dir,
config,
&run.id,
&commit_sha,
if strict_two_pass {
"manual-review-strict"
} else {
"manual-review"
},
&execution.entries,
);
run_store.mark_completed(&run.id, execution.entries.len())?;
println!(
"truth-mirror review: run {}, wrote {} ledger entrie(s)",
run.id,
execution.entries.len()
);
Ok(ExitCode::SUCCESS)
}
fn run_review_run_command(command: cli::ReviewCommand, state_dir: &Path) -> Result<ExitCode> {
let runs = ReviewRunStore::new(state_dir);
match command {
cli::ReviewCommand::Status { run_id } => {
if let Some(run_id) = run_id {
print_run(&runs.read_reconciled(&run_id)?);
} else {
let all = runs.list_reconciled()?;
if all.is_empty() {
println!("No review runs.");
} else {
for run in all {
print_run_summary(&run);
}
}
}
}
cli::ReviewCommand::Result { run_id } => {
let run = match run_id {
Some(run_id) => runs.read(&run_id)?,
None => runs.latest_result()?,
};
print_run(&run);
print_run_ledger_entries(state_dir, &run)?;
}
cli::ReviewCommand::Cancel { run_id, force } => {
let run = runs.cancel(&run_id, force)?;
ReviewQueue::new(state_dir).remove_run_id(&run_id)?;
match run.status {
ReviewRunStatus::Failed => println!(
"reaped stale review run {} ({}): {}",
run.id,
run.commit_sha,
run.error.as_deref().unwrap_or("worker was not alive"),
),
_ => println!("cancelled review run {} ({})", run.id, run.commit_sha),
}
}
}
Ok(ExitCode::SUCCESS)
}
fn print_run_summary(run: &ReviewRun) {
println!(
"{} {} {} {} entries={} updated={}",
run.id, run.status, run.commit_sha, run.phase, run.ledger_entries, run.updated_at_unix
);
}
fn print_run(run: &ReviewRun) {
println!("run: {}", run.id);
println!("status: {}", run.status);
println!("commit: {}", run.commit_sha);
println!("target: {}", run.target);
println!("phase: {}", run.phase);
println!("ledger_entries: {}", run.ledger_entries);
if let Some(pid) = run.worker_pid {
println!("worker_pid: {pid}");
}
println!("created_at_unix: {}", run.created_at_unix);
println!("updated_at_unix: {}", run.updated_at_unix);
if let Some(started) = run.started_at_unix {
println!("started_at_unix: {started}");
}
if let Some(completed) = run.completed_at_unix {
println!("completed_at_unix: {completed}");
}
if let Some(error) = &run.error {
println!("error: {error}");
}
if let Some(checkpoint) = &run.entire_checkpoint {
println!("entire_ref: {}", checkpoint.ref_name);
println!("entire_sha: {}", checkpoint.object_sha);
}
}
fn print_run_ledger_entries(state_dir: &Path, run: &ReviewRun) -> Result<(), ReviewerError> {
let store = LedgerStore::new(state_dir);
let entries: Vec<LedgerEntry> = store
.read_history()?
.into_iter()
.filter(|entry| entry.commit_sha == run.commit_sha)
.collect();
if entries.is_empty() {
println!("ledger_entries: none");
return Ok(());
}
println!("ledger_entries:");
for entry in entries {
println!(
"- {} {} {} findings={}",
entry.commit_sha,
entry.verdict,
entry.disposition,
entry.findings.len()
);
}
Ok(())
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct ReviewMaterial {
commit_sha: String,
target_label: String,
claim: Claim,
diff: String,
}
impl ReviewMaterial {
fn load(
args: &cli::ReviewArgs,
state_dir: &Path,
evidence_patterns: &[String],
) -> Result<Self, ReviewerError> {
let parse = |text: &str| {
if evidence_patterns.is_empty() {
Claim::parse(text)
} else {
Claim::parse_with(text, evidence_patterns)
}
};
let scope = if args.staged {
ReviewScope::Staged
} else {
args.scope
};
match scope {
ReviewScope::Commit => {
let target = args
.target
.clone()
.ok_or(ReviewerError::MissingReviewTarget)?;
let sha = resolve_commit_target(&target)?;
let message = git_output(["show", "--format=%B", "--no-patch", sha.as_str()])?;
let diff = git_output(["show", "--format=", "--patch", sha.as_str()])?;
let claim = parse(&message)?;
Ok(Self {
commit_sha: sha.clone(),
target_label: format!("commit:{target}"),
claim,
diff,
})
}
ReviewScope::Staged => Self::load_staged(state_dir, &parse),
ReviewScope::Auto => {
reject_target_with_scope(args)?;
if working_tree_dirty()? {
Self::load_working_tree(state_dir, &parse)
} else {
Self::load_branch(args.base.as_deref(), &parse)
}
}
ReviewScope::WorkingTree => {
reject_target_with_scope(args)?;
Self::load_working_tree(state_dir, &parse)
}
ReviewScope::Branch => {
reject_target_with_scope(args)?;
Self::load_branch(args.base.as_deref(), &parse)
}
}
}
fn load_staged<F>(state_dir: &Path, parse: &F) -> Result<Self, ReviewerError>
where
F: Fn(&str) -> Result<Claim, crate::claim::ClaimError>,
{
let raw = git_output(["diff", "--cached"])?;
let files = git_output(["diff", "--cached", "--name-only"])?;
let diff = materialize_diff("staged", &raw, &files);
let claim = parse(&read_claim_file(state_dir)?)?;
Ok(Self {
commit_sha: "STAGED".to_owned(),
target_label: "staged".to_owned(),
claim,
diff,
})
}
fn load_working_tree<F>(state_dir: &Path, parse: &F) -> Result<Self, ReviewerError>
where
F: Fn(&str) -> Result<Claim, crate::claim::ClaimError>,
{
let status = git_output(["status", "--porcelain"])?;
let tracked = git_output(["diff", "HEAD", "--patch"])?;
let files = git_output(["diff", "HEAD", "--name-only"])?;
let untracked = untracked_file_context()?;
let raw = format!(
"WORKING TREE STATUS:\n{status}\n\nTRACKED DIFF AGAINST HEAD:\n{tracked}\n\nUNTRACKED FILES:\n{untracked}"
);
let diff = materialize_diff("working-tree", &raw, &files);
let claim = parse(&read_claim_file(state_dir)?)?;
Ok(Self {
commit_sha: "WORKING_TREE".to_owned(),
target_label: "working-tree".to_owned(),
claim,
diff,
})
}
fn load_branch<F>(base: Option<&str>, parse: &F) -> Result<Self, ReviewerError>
where
F: Fn(&str) -> Result<Claim, crate::claim::ClaimError>,
{
let base = match base {
Some(base) => base.to_owned(),
None => default_branch_ref()?,
};
let merge_base = git_output_slice(&["merge-base", "HEAD", &base])?;
let merge_base = merge_base.trim().to_owned();
let range = format!("{merge_base}..HEAD");
let message = git_output(["show", "--format=%B", "--no-patch", "HEAD"])?;
let log = git_output_slice(&["log", "--oneline", &range])?;
let stat = git_output_slice(&["diff", "--stat", &range])?;
let raw_patch = git_output_slice(&["diff", "--patch", &range])?;
let files = git_output_slice(&["diff", "--name-only", &range])?;
let raw = format!(
"BRANCH BASE: {base}\nMERGE BASE: {merge_base}\nCOMMITS:\n{log}\n\nDIFF STAT:\n{stat}\n\nDIFF:\n{raw_patch}"
);
let diff = materialize_diff(&format!("branch:{base}"), &raw, &files);
let claim = parse(&message)?;
Ok(Self {
commit_sha: "HEAD".to_owned(),
target_label: format!("branch:{base}"),
claim,
diff,
})
}
}
fn resolve_commit_target(target: &str) -> Result<String, ReviewerError> {
let rev = format!("{target}^{{commit}}");
Ok(
git_output_slice(&["rev-parse", "--verify", "--quiet", "--end-of-options", &rev])?
.trim()
.to_owned(),
)
}
fn reject_target_with_scope(args: &cli::ReviewArgs) -> Result<(), ReviewerError> {
if let Some(target) = &args.target {
return Err(ReviewerError::UnexpectedReviewTarget {
scope: args.scope,
target: target.clone(),
});
}
Ok(())
}
fn read_claim_file(state_dir: &Path) -> Result<String, ReviewerError> {
let claim_path = state_dir.join("claim.txt");
fs::read_to_string(&claim_path).map_err(|source| ReviewerError::ClaimFileRead {
path: claim_path,
source,
})
}
fn working_tree_dirty() -> Result<bool, ReviewerError> {
Ok(!git_output(["status", "--porcelain"])?.trim().is_empty())
}
fn default_branch_ref() -> Result<String, ReviewerError> {
if let Ok(symbolic) = git_output([
"symbolic-ref",
"--quiet",
"--short",
"refs/remotes/origin/HEAD",
]) {
let trimmed = symbolic.trim();
if !trimmed.is_empty() {
return Ok(trimmed.to_owned());
}
}
for candidate in [
"origin/main",
"origin/master",
"origin/trunk",
"main",
"master",
"trunk",
] {
if git_output_slice(&["rev-parse", "--verify", "--quiet", candidate]).is_ok() {
return Ok(candidate.to_owned());
}
}
Err(ReviewerError::DefaultBranchNotFound)
}
fn materialize_diff(label: &str, raw: &str, files: &str) -> String {
let file_list: Vec<&str> = files
.lines()
.filter(|line| !line.trim().is_empty())
.collect();
let bytes = raw.len();
if bytes <= MAX_INLINE_DIFF_BYTES && file_list.len() <= MAX_INLINE_DIFF_FILES {
return raw.to_owned();
}
format!(
"Diff for {label} is too large to inline safely.\ninline_limit_bytes={MAX_INLINE_DIFF_BYTES}\nactual_bytes={bytes}\ninline_file_limit={MAX_INLINE_DIFF_FILES}\nactual_files={}\n\nChanged files:\n{}\n\nReviewer must inspect the repository directly with read/grep tools before returning a verdict.",
file_list.len(),
if file_list.is_empty() {
"(none)".to_owned()
} else {
file_list.join("\n")
}
)
}
fn is_full_git_sha(value: &str) -> bool {
matches!(value.len(), 40 | 64) && value.bytes().all(|byte| byte.is_ascii_hexdigit())
}
fn untracked_file_context() -> Result<String, ReviewerError> {
let files = git_output(["ls-files", "--others", "--exclude-standard"])?;
let mut output = String::new();
for file in files.lines().filter(|line| !line.trim().is_empty()) {
let path = Path::new(file);
let metadata = match fs::metadata(path) {
Ok(metadata) => metadata,
Err(_) => continue,
};
if !metadata.is_file() {
continue;
}
if metadata.len() > MAX_UNTRACKED_FILE_BYTES {
output.push_str(&format!(
"\n--- {file} omitted: {} bytes exceeds {MAX_UNTRACKED_FILE_BYTES} byte inline limit ---\n",
metadata.len()
));
continue;
}
let bytes = match fs::read(path) {
Ok(bytes) => bytes,
Err(_) => continue,
};
if bytes.contains(&0) {
output.push_str(&format!("\n--- {file} omitted: binary file ---\n"));
continue;
}
output.push_str(&format!(
"\n--- {file} ---\n{}",
String::from_utf8_lossy(&bytes)
));
}
if output.is_empty() {
Ok("(none)".to_owned())
} else {
Ok(output)
}
}
#[derive(Debug, Error)]
pub enum ReviewerError {
#[error("missing {role} model")]
MissingModel { role: String },
#[error(
"same reviewer model is disallowed without --allow-same-model: watched={watched_model}, reviewer={reviewer_model}"
)]
SameModelWithoutWaiver {
watched_model: String,
reviewer_model: String,
},
#[error("strict arbiter model must differ from watched and first reviewer models")]
StrictArbiterModelNotDistinct,
#[error("no adversarial pair configured for writer harness {writer:?}")]
NoPairForWriter { writer: String },
#[error(
"strict review requires an arbiter (pair.arbiter or --arbiter-harness/--arbiter-model)"
)]
MissingArbiter,
#[error(
"--{role}-harness={harness:?} was overridden without a matching --{role}-model; the pair's model is for a different harness"
)]
OverrideNeedsModel { role: String, harness: String },
#[error("custom reviewer harness requires explicit command configuration")]
UnsupportedCustomHarness,
#[error("unknown watched agent {value:?}")]
UnknownAgent { value: String },
#[error("unknown reviewer harness {value:?}")]
UnknownHarness { value: String },
#[error("missing review target")]
MissingReviewTarget,
#[error("--scope={scope:?} does not accept positional target {target:?}")]
UnexpectedReviewTarget { scope: ReviewScope, target: String },
#[error("could not determine default branch; pass --base explicitly")]
DefaultBranchNotFound,
#[error("failed to read staged claim file {path}: {source}")]
ClaimFileRead {
path: PathBuf,
#[source]
source: io::Error,
},
#[error("reviewer output was not valid structured JSON verdict: {source}: {output:?}")]
VerdictJson {
source: serde_json::Error,
output: String,
},
#[error("reviewer structured verdict violated schema: {message}")]
VerdictSchema { message: String },
#[error("reviewer petition-batch verdict violated outer schema: {message}")]
BatchVerdictSchema { message: String },
#[error("reviewer process exited with status {status:?}: {stderr}")]
ReviewerProcessFailed { status: Option<i32>, stderr: String },
#[error(
"reviewer timed out after {timeout_secs}s; the wedged process was killed and the review recorded as failed"
)]
ReviewerTimeout { timeout_secs: u64 },
#[error(
"reviewer exited but a descendant held its pipes open past the {grace_secs}s drain grace; the output is untrustworthy and was discarded"
)]
ReviewerOutputWedged { grace_secs: u64 },
#[error("failed to kill wedged reviewer process {pid} ({tree_error}): {source}")]
KillReviewer {
pid: u32,
tree_error: String,
#[source]
source: io::Error,
},
#[error("reviewer process {pid} did not exit after {grace_secs}s despite kill attempts")]
KillReviewerTimeout {
pid: u32,
grace_secs: u64,
#[source]
source: io::Error,
},
#[error("git command failed: git {args:?}: {stderr}")]
GitFailed { args: Vec<String>, stderr: String },
#[error("failed to spawn git command: {0}")]
GitSpawn(io::Error),
#[error("failed to spawn reviewer process: {0}")]
Spawn(io::Error),
#[error("failed to open reviewer stdin pipe")]
MissingStdinPipe,
#[error("failed to write reviewer prompt: {0}")]
WritePrompt(io::Error),
#[error("failed to wait for reviewer process: {0}")]
Wait(io::Error),
#[error("review queue IO failed: {0}")]
QueueIo(io::Error),
#[error("review queue JSON failed: {0}")]
QueueJson(serde_json::Error),
#[error("rejection {sha} is not open for petition")]
PetitionRejectionNotOpen { sha: String },
#[error("a petition for {rejection_sha} is already queued")]
PetitionInFlight { rejection_sha: String },
#[error("petition attempts exhausted for {rejection_sha}")]
PetitionExhausted { rejection_sha: String },
#[error("review run IO failed: {0}")]
RunIo(io::Error),
#[error(
"timed out after {wait_secs}s waiting for the review-run store lock; another truth-mirror process is mutating run rows"
)]
RunStoreLockTimeout { wait_secs: u64 },
#[error("review run JSON failed: {0}")]
RunJson(serde_json::Error),
#[error("review run not found: {id}")]
ReviewRunNotFound { id: String },
#[error("no review runs found")]
NoReviewRuns,
#[error("cannot cancel review run {id} with status {status}; it has already finished")]
CannotCancelReview { id: String, status: ReviewRunStatus },
#[error(
"review run {id} is still running (worker pid {pid} is alive); pass --force to kill it"
)]
ReviewRunStillAlive { id: String, pid: u32 },
#[error(
"review run {id} is running but records no worker pid; pass --force to reap it if it is stuck"
)]
ReviewRunLivenessUnknown { id: String },
#[error("failed to spawn kill for stale worker: {0}")]
KillWorker(io::Error),
#[error("failed to kill worker process {pid}")]
KillWorkerFailed { pid: u32 },
#[error(transparent)]
State(#[from] crate::state::StateError),
#[error(transparent)]
Claim(#[from] crate::claim::ClaimError),
#[error(transparent)]
Ledger(#[from] crate::ledger::LedgerError),
}
const ADVERSARIAL_PREAMBLE: &str = r#"You are an ADVERSARIAL reviewer. Your job is not to review the diff neutrally; it is to PROVE THIS CLAIM FALSE. Assume the author over-rates their own work. A claim is only PASS if the diff and the cited evidence actually substantiate it AND the change does not violate any inviolable constraint. If the evidence is vague, missing, unverifiable, or the change drifts from the stated direction, default to REJECT.
Attack the change for auth and permission holes, data loss, rollback gaps, races, stale state, version skew, observability gaps, missing evidence, fake evidence, broad matchers, gates that fail open, and code that only fixes the instance instead of the defect class.
GREP THE CLASS, NOT THE INSTANCE. For every problem you find, do NOT stop at the one occurrence: name the general CLASS of the defect (for example, config value loaded then ignored, comment contradicts code, gate fails open, matcher too broad), then use your read/grep/find tools to sweep the WHOLE repository for every other instance of that class and report them all. One instance is a symptom; the class is the bug. Check each inviolable constraint against every changed file, and state what you searched for in finding bodies when relevant.
Return valid JSON only. Do not wrap it in Markdown. The schema is:
{
"verdict": "PASS" | "REJECT" | "FLAG",
"summary": "one concise sentence explaining why the claim passes or fails",
"findings": [
{
"severity": "critical" | "high" | "medium" | "low",
"title": "short defect title",
"body": "what can go wrong, why this code is vulnerable, and what evidence proves it",
"file": "repo-relative file path",
"line_start": 1,
"line_end": 1,
"confidence": 0,
"recommendation": "concrete change required"
}
],
"next_steps": ["short concrete follow-up commands or edits"],
"memory_skill": {
"kind": "none" | "how_to_skill" | "anti_pattern_skill" | "remediation_skill",
"learning_source": "short reusable procedure or failure class; empty only when kind is none",
"reasoning": "why this memory-skill classification applies or why no candidate should be proposed"
}
}
For "memory_skill", propose "how_to_skill" only for reusable verified PASS procedures, "anti_pattern_skill" for repeated false-claim or evidence failure classes, "remediation_skill" for repeated repair workflows, and "none" when the reviewed change is not reusable procedural memory. Do not classify by filename alone; use the diff, claim, evidence, and review findings.
Verdict semantics:
- "PASS": the claim is substantiated AND there are no findings. Never pair PASS with findings.
- "REJECT": at least one finding is materially false / unverified / unaddressed. Must include at least one finding.
- "FLAG": the claim substantively holds but the reviewer wants to surface non-blocking process / evidence / provenance debt alongside the verdict. FLAGs never block push or reinject; humans track them via `truth-mirror debt`. Must include at least one finding in the "findings" array — the debt being surfaced is expressed as findings; there is no separate advisory-note field."#;
fn context_block(context: &str) -> String {
if context.trim().is_empty() {
String::new()
} else {
format!("\n\n{context}")
}
}
fn first_pass_prompt(claim: &Claim, diff: &str, context: &str) -> String {
format!(
"{ADVERSARIAL_PREAMBLE}{}\n\nCLAIM:\n{}\n\nDIFF:\n{}",
context_block(context),
claim.to_line(),
diff
)
}
const PETITION_PREAMBLE: &str = r#"You are an ADVERSARIAL petition reviewer. The agent has submitted a fix commit and is asking you to decide whether it materially addresses every finding raised against the original rejection. Your job is not to re-evaluate the original claim from scratch; it is to read each original finding, inspect the fix, and judge whether the fix actually addresses it (instance) and the broader class the finding represents.
Return valid JSON only. Do not wrap it in Markdown. The schema is:
{
"verdict": "PASS" | "REJECT" | "FLAG",
"summary": "one concise sentence explaining whether the fix materially addresses each finding",
"findings": [
{
"severity": "critical" | "high" | "medium" | "low",
"title": "short defect title",
"body": "what still fails after the fix, why this code is still vulnerable, and what evidence proves it",
"file": "repo-relative file path",
"line_start": 1,
"line_end": 1,
"confidence": 0,
"recommendation": "concrete change required"
}
],
"next_steps": ["short concrete follow-up commands or edits"],
"memory_skill": {
"kind": "none" | "how_to_skill" | "anti_pattern_skill" | "remediation_skill",
"learning_source": "short reusable procedure or failure class; empty only when kind is none",
"reasoning": "why this memory-skill classification applies or why no candidate should be proposed"
}
}
Verdict semantics:
- "PASS" means the fix materially addresses every original finding. Return PASS only when findings is empty.
- "REJECT" means at least one finding is still materially unaddressed by the fix.
- "FLAG" means the fix materially addresses the findings but you want to surface process / evidence / provenance debt alongside the acceptance. Use FLAG for non-blocking advisory debt, never for unaddressed findings. Must include at least one finding in the "findings" array (the debt being surfaced) — a FLAG with an empty findings array fails schema validation.
For "memory_skill", propose "how_to_skill" only for reusable verified PASS procedures, "anti_pattern_skill" for repeated false-claim or evidence failure classes, "remediation_skill" for repeated repair workflows, and "none" when the reviewed change is not reusable procedural memory."#;
fn petition_prompt(petition: &PetitionContext, claim: &Claim, diff: &str, context: &str) -> String {
let findings_block = petition_findings_block(petition);
format!(
"{PETITION_PREAMBLE}{}\n\nPETITION:\n- original rejection SHA: {}\n- fix commit SHA: {}\n- petition attempts so far: {}\n- original reviewer model: {}\n\nORIGINAL CLAIM:\n{}\n\nORIGINAL SUMMARY:\n{}\n\nORIGINAL FINDINGS:\n{}\n\nFIX COMMIT DIFF:\n{}\n\nCLAIM (for ledger):\n{}",
context_block(context),
petition.original_sha,
petition.fix_sha,
petition.attempts_so_far,
petition.original_reviewer_model,
petition.original_claim,
petition.original_summary,
findings_block,
diff,
claim.to_line(),
)
}
const PETITION_BATCH_PREAMBLE: &str = r#"You are an ADVERSARIAL petition-batch reviewer. One fix commit petitions multiple original rejections. Reuse analysis where findings overlap, but independently decide whether the fix materially addresses every finding for EACH listed original rejection.
Return valid JSON only. Do not wrap it in Markdown. Return exactly one result for every supplied original rejection SHA. Never infer an outcome for a missing, malformed, unknown, or duplicate item. A successful process exit is not acceptance; only a valid result whose original_rejection_sha exactly matches a supplied SHA can affect that rejection.
Only "PASS" is accepting, and PASS requires an empty findings array. "REJECT" and "FLAG" remain auditable outcomes but DO NOT resolve the original rejection. REJECT and FLAG must include at least one structured finding. Every result must include summary, findings, next_steps, and memory_skill using the same schema as an individual petition review.
The outer schema is:
{
"batch_id": "the supplied batch ID",
"fix_sha": "the supplied fix commit SHA",
"results": [
{
"original_rejection_sha": "one supplied original rejection SHA",
"verdict": "PASS" | "REJECT" | "FLAG",
"summary": "one concise sentence",
"findings": [{
"severity": "critical" | "high" | "medium" | "low",
"title": "short defect title",
"body": "what remains wrong and the evidence",
"file": "repo-relative file path",
"line_start": 1,
"line_end": 1,
"confidence": 0,
"recommendation": "concrete change required"
}],
"next_steps": ["short concrete follow-up"],
"memory_skill": {
"kind": "none" | "how_to_skill" | "anti_pattern_skill" | "remediation_skill",
"learning_source": "empty only when kind is none",
"reasoning": "why this classification applies"
}
}
]
}"#;
fn petition_findings_block(petition: &PetitionContext) -> String {
let findings = if petition.original_structured_findings.is_empty() {
petition.original_findings.join("\n")
} else {
petition
.original_structured_findings
.iter()
.map(StructuredFinding::display_line)
.collect::<Vec<_>>()
.join("\n")
};
if findings.trim().is_empty() {
"(none recorded)".to_owned()
} else {
findings
}
}
fn petition_batch_prompt(
batch_id: &str,
petitions: &[&PetitionContext],
claim: &Claim,
diff: &str,
context: &str,
) -> String {
let fix_sha = petitions
.first()
.map_or("", |petition| petition.fix_sha.as_str());
let mut prompt = format!(
"{PETITION_BATCH_PREAMBLE}{}\n\nBATCH:\n- batch ID: {batch_id}\n- fix commit SHA: {fix_sha}\n- member count: {}\n\nPETITION DOSSIERS:",
context_block(context),
petitions.len()
);
for (index, petition) in petitions.iter().enumerate() {
prompt.push_str(&format!(
"\n\n{}. ORIGINAL REJECTION SHA: {}\n- petition attempts so far: {}\n- original reviewer model: {}\n\nORIGINAL CLAIM:\n{}\n\nORIGINAL SUMMARY:\n{}\n\nORIGINAL FINDINGS:\n{}",
index + 1,
petition.original_sha,
petition.attempts_so_far,
petition.original_reviewer_model,
petition.original_claim,
petition.original_summary,
petition_findings_block(petition),
));
}
prompt.push_str(&format!(
"\n\nSHARED FIX COMMIT (applies to every dossier above):\n\nCLAIM (for ledger):\n{}\n\nFIX COMMIT DIFF (included exactly once):\n{}",
claim.to_line(),
diff
));
prompt
}
fn strict_petition_batch_prompt(
batch_id: &str,
petitions: &[&PetitionContext],
claim: &Claim,
diff: &str,
context: &str,
first_entries: &[&LedgerEntry],
) -> String {
let mut prompt = petition_batch_prompt(batch_id, petitions, claim, diff, context);
prompt.push_str(
"\n\nSTRICT SECOND PASS (COMPLETENESS CRITIC): every member below received a provisional PASS. Re-check each member independently and prove the first reviewer incomplete. Return results only for these provisionally accepted members.",
);
prompt.push_str("\n\nPROVISIONAL FIRST-PASS RESULTS:");
for entry in first_entries {
prompt.push_str(&format!(
"\n- original rejection SHA: {} | verdict: {} | summary: {}",
entry.petition_for.as_deref().unwrap_or("(missing)"),
entry.verdict,
entry.summary
));
}
prompt
}
fn strict_second_pass_prompt(job: &ReviewJob, first_output: &str) -> String {
// A strict pass over a PETITION must stay on the petition question — the
// generic completeness prompt carries the fix commit's claim/diff but not
// the original rejection's findings, so the arbiter would end up judging
// a first review whose question it never saw.
if let Some(petition) = &job.petition {
let petition_question = petition_prompt(petition, &job.claim, &job.diff, &job.context);
return format!(
"{petition_question}\n\nSTRICT SECOND PASS (COMPLETENESS CRITIC): the first petition reviewer ACCEPTED that the fix materially addresses each original finding. Assume it verified some findings but not ALL of them. Re-check every original finding against the fix diff and prove the first reviewer INCOMPLETE.\n\nFIRST REVIEW:\n{first_output}"
);
}
format!(
"{ADVERSARIAL_PREAMBLE}\n\nStrict second pass (COMPLETENESS CRITIC): the first reviewer returned a CLEAN verdict. Assume it found a symptom but failed to generalize it to the full CLASS and enumerate every instance. Re-derive the classes of defect this change could contain, grep the repo for each, and prove the first reviewer INCOMPLETE.{}\n\nCLAIM:\n{}\n\nFIRST REVIEW:\n{}\n\nDIFF:\n{}",
context_block(&job.context),
job.claim.to_line(),
first_output,
job.diff
)
}
fn entry_from_verdict(job: &ReviewJob, plan: &ReviewPlan, verdict: &ParsedVerdict) -> LedgerEntry {
let mut entry = LedgerEntry::new(
job.commit_sha.clone(),
verdict.verdict,
job.claim.to_line(),
job.claim
.evidence
.iter()
.map(EvidenceRef::as_str)
.map(str::to_owned)
.collect(),
plan.reviewer_config(),
verdict.findings.clone(),
)
.with_structured_review(
verdict.summary.clone(),
verdict.structured_findings.clone(),
verdict.next_steps.clone(),
verdict.raw.clone(),
)
.with_memory_skill_classification(verdict.memory_skill_classification.clone());
if let Some(petition) = &job.petition {
entry.petition_for = Some(petition.original_sha.clone());
// `attempts_so_far` already includes the in-flight attempt — the CLI
// counted it when it enqueued the petition (see `resolve::run_petition`).
// Adding 1 here would double-count every attempt across CLI + watcher.
entry.petition_attempts = petition.attempts_so_far;
}
entry
}
fn petition_entry_from_batch_verdict(
fix_sha: &str,
claim: &Claim,
petition: &PetitionContext,
plan: &ReviewPlan,
verdict: &ParsedVerdict,
run_id: &str,
batch_id: &str,
) -> LedgerEntry {
let mut entry = LedgerEntry::new(
fix_sha,
verdict.verdict,
claim.to_line(),
claim
.evidence
.iter()
.map(EvidenceRef::as_str)
.map(str::to_owned)
.collect(),
plan.reviewer_config(),
verdict.findings.clone(),
)
.with_structured_review(
verdict.summary.clone(),
verdict.structured_findings.clone(),
verdict.next_steps.clone(),
verdict.raw.clone(),
)
.with_memory_skill_classification(verdict.memory_skill_classification.clone());
entry.petition_for = Some(petition.original_sha.clone());
entry.petition_attempts = petition.attempts_so_far;
entry.review_run_id = Some(run_id.to_owned());
entry.batch_id = Some(batch_id.to_owned());
entry.reviewer_item_id = Some(petition.original_sha.clone());
entry
}
fn ensure_process_success(output: &ProcessOutput) -> Result<(), ReviewerError> {
if output.status_code == Some(0) {
return Ok(());
}
Err(ReviewerError::ReviewerProcessFailed {
status: output.status_code,
stderr: output.stderr.clone(),
})
}
fn validate_strict_arbiter(
request: &ReviewRequest,
strict: &StrictReviewConfig,
) -> Result<(), ReviewerError> {
let arbiter = normalized_model(&strict.arbiter_model);
if arbiter == normalized_model(&request.watched_model)
|| arbiter == normalized_model(&request.reviewer_model)
{
return Err(ReviewerError::StrictArbiterModelNotDistinct);
}
Ok(())
}
fn validate_model_present(role: &str, model: &str) -> Result<(), ReviewerError> {
if model.trim().is_empty() {
return Err(ReviewerError::MissingModel {
role: role.to_owned(),
});
}
Ok(())
}
fn git_output<const N: usize>(args: [&str; N]) -> Result<String, ReviewerError> {
git_output_slice(&args)
}
fn git_output_slice(args: &[&str]) -> Result<String, ReviewerError> {
let output = Command::new("git")
.args(args)
.output()
.map_err(ReviewerError::GitSpawn)?;
if !output.status.success() {
return Err(ReviewerError::GitFailed {
args: args.iter().map(|arg| (*arg).to_owned()).collect(),
stderr: String::from_utf8_lossy(&output.stderr).into_owned(),
});
}
Ok(String::from_utf8_lossy(&output.stdout).into_owned())
}
fn agent_from_slug(value: &str) -> Result<Agent, ReviewerError> {
match value.trim().to_ascii_lowercase().as_str() {
"claude" => Ok(Agent::Claude),
"codex" => Ok(Agent::Codex),
"pi" => Ok(Agent::Pi),
"grok" => Ok(Agent::Grok),
_ => Err(ReviewerError::UnknownAgent {
value: value.to_owned(),
}),
}
}
fn harness_from_slug(value: &str) -> Result<ReviewerHarness, ReviewerError> {
match value.trim().to_ascii_lowercase().as_str() {
"claude" => Ok(ReviewerHarness::Claude),
"codex" => Ok(ReviewerHarness::Codex),
"pi" => Ok(ReviewerHarness::Pi),
"gemini" => Ok(ReviewerHarness::Gemini),
"opencode" => Ok(ReviewerHarness::Opencode),
"custom" => Ok(ReviewerHarness::Custom),
_ => Err(ReviewerError::UnknownHarness {
value: value.to_owned(),
}),
}
}
fn harness_slug(harness: ReviewerHarness) -> &'static str {
match harness {
ReviewerHarness::Claude => "claude",
ReviewerHarness::Codex => "codex",
ReviewerHarness::Pi => "pi",
ReviewerHarness::Gemini => "gemini",
ReviewerHarness::Opencode => "opencode",
ReviewerHarness::Custom => "custom",
}
}
/// Normalise a model identifier for opposition comparisons.
///
/// Delegates to [`crate::config::normalized_model`], which is the single
/// authoritative implementation shared by config-time validation and the
/// runtime same-model guard here (R6 dedup).
fn normalized_model(model: &str) -> String {
config::normalized_model(model)
}
fn generate_run_id(commit_sha: &str) -> String {
let nanos = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_or(0, |duration| duration.as_nanos());
let short_sha: String = commit_sha
.chars()
.filter(|character| character.is_ascii_alphanumeric())
.take(12)
.collect();
if short_sha.is_empty() {
format!("{nanos}-{}", std::process::id())
} else {
format!("{nanos}-{}-{short_sha}", std::process::id())
}
}
#[cfg(test)]
mod tests {
use std::{
cell::RefCell,
collections::VecDeque,
fs,
path::{Path, PathBuf},
process::Command,
sync::{
Arc, Barrier, Mutex,
atomic::{AtomicBool, AtomicUsize, Ordering},
},
thread,
time::Duration,
};
use super::normalized_model;
use proptest::prelude::*;
use super::{
CLAIMED_PHASE, DEFAULT_PROVIDER_BACKOFF_SECS, ENQUEUE_INFLIGHT_FILE, InvocationPlan,
MaterialLoader, PROVIDER_BACKOFF_PHASE, ParsedVerdict, PendingReviewSpool, PlannedDrainJob,
ProcessOutput, ProcessRunner, PromptDelivery, QueuedReview, REVIEW_LEDGER_PHASE,
RUN_STORE_LOCK_DIR, ReviewJob, ReviewPlan, ReviewQueue, ReviewRequest, ReviewRun,
ReviewRunStatus, ReviewRunStore, ReviewSelection, ReviewerError, RunStoreLockRecord,
StrictGoalCounters, StrictGoalDecision, StrictGoalPolicy, StrictGoalStopReason,
StrictReviewConfig, UntilEmptyDecision, WatchLockClaim, claim_planned_job,
claim_watcher_lock, create_review_worktree, drain_once, drain_once_parallel,
execute_review_job, filter_pending_ready_for_attempt, git_output, is_full_git_sha,
is_retryable_provider_failure, maintain_terminal_worktrees, plan_batched_drain,
run_ledger_phase_applied, run_review_run_command, run_store_lock_is_stale,
run_strict_goal_loop, until_empty_decision, write_enqueue_inflight,
write_run_store_lock_owner,
};
use crate::time::unix_now;
use crate::{
claim::{Claim, EvidenceRef},
cli::{Agent, ReviewerHarness},
config::{Effort, TruthMirrorConfig},
ledger::{
FindingSeverity, LedgerStore, MemorySkillClassificationKind, StructuredFinding, Verdict,
},
watcher::{ProcessIdentity, pid_is_alive},
};
fn pass_json() -> String {
serde_json::json!({
"verdict": "PASS",
"summary": "The claim is substantiated by the diff and evidence.",
"findings": [],
"next_steps": [],
"memory_skill": {
"kind": "how_to_skill",
"learning_source": "run reusable review workflow",
"reasoning": "The passing claim describes a reusable verified procedure."
}
})
.to_string()
}
fn reject_json(title: &str) -> String {
serde_json::json!({
"verdict": "REJECT",
"summary": "The claim is not substantiated.",
"findings": [{
"severity": "high",
"title": title,
"body": "The cited evidence does not prove the claimed behavior.",
"file": "src/lib.rs",
"line_start": 1,
"line_end": 1,
"confidence": 95,
"recommendation": "Provide executable evidence that proves the claim."
}],
"next_steps": ["Run the relevant verification command."],
"memory_skill": {
"kind": "anti_pattern_skill",
"learning_source": title,
"reasoning": "The rejection identifies a reusable false-claim failure class."
}
})
.to_string()
}
fn batch_result(original_sha: &str, verdict_json: &str) -> serde_json::Value {
let mut result: serde_json::Value = serde_json::from_str(verdict_json).unwrap();
result.as_object_mut().unwrap().insert(
"original_rejection_sha".to_owned(),
serde_json::Value::String(original_sha.to_owned()),
);
result
}
#[test]
fn same_harness_different_model_is_valid() {
let request = ReviewRequest::new(
Agent::Codex,
"gpt-5.4",
ReviewerHarness::Codex,
"gpt-5.5",
false,
"review this",
);
let plan = ReviewPlan::build(request).unwrap();
assert_eq!(plan.watched_agent, Agent::Codex);
assert_eq!(plan.reviewer_harness, ReviewerHarness::Codex);
assert_eq!(plan.invocation.program, "codex");
}
#[test]
fn same_model_is_blocked_by_default() {
let request = ReviewRequest::new(
Agent::Codex,
" GPT-5.5 ",
ReviewerHarness::Claude,
"gpt-5.5",
false,
"review this",
);
let error = ReviewPlan::build(request).unwrap_err();
assert!(matches!(
error,
ReviewerError::SameModelWithoutWaiver { .. }
));
}
#[test]
fn allow_same_model_override_is_deliberate() {
let request = ReviewRequest::new(
Agent::Codex,
"gpt-5.5",
ReviewerHarness::Codex,
"gpt-5.5",
true,
"review this",
);
let plan = ReviewPlan::build(request).unwrap();
assert!(plan.allow_same_model);
assert_eq!(plan.reviewer_model, "gpt-5.5");
}
#[test]
fn provider_mapping_uses_verified_prompt_shapes_and_effort() {
let codex =
InvocationPlan::for_harness(ReviewerHarness::Codex, "gpt-5.5", Effort::Xhigh).unwrap();
assert_eq!(codex.program, "codex");
assert_eq!(
codex.args_for_prompt("prompt"),
[
"exec",
"-m",
"gpt-5.5",
"-c",
"model_reasoning_effort=xhigh",
"prompt"
]
);
let claude =
InvocationPlan::for_harness(ReviewerHarness::Claude, "opus", Effort::High).unwrap();
assert_eq!(claude.program, "claude");
assert_eq!(claude.prompt_delivery, PromptDelivery::Stdin);
assert_eq!(
claude.args_for_prompt("prompt"),
["--print", "--model", "opus", "--effort", "high"]
);
let gemini =
InvocationPlan::for_harness(ReviewerHarness::Gemini, "gemini-pro", Effort::Xhigh)
.unwrap();
assert_eq!(
gemini.args_for_prompt("prompt"),
["-m", "gemini-pro", "-p", "prompt"]
);
let pi = InvocationPlan::for_harness(ReviewerHarness::Pi, "openai/gpt-5.5", Effort::Xhigh)
.unwrap();
assert_eq!(pi.prompt_delivery, PromptDelivery::Stdin);
assert_eq!(
pi.args_for_prompt("prompt"),
[
"--model",
"openai/gpt-5.5",
"--thinking",
"xhigh",
"--tools",
"read,grep,find,ls",
"-p"
]
);
}
#[test]
fn custom_harness_requires_explicit_configuration() {
let error = InvocationPlan::for_harness(ReviewerHarness::Custom, "model", Effort::Xhigh)
.unwrap_err();
assert!(matches!(error, ReviewerError::UnsupportedCustomHarness));
}
#[test]
fn effort_maps_to_each_harness_flag() {
for effort in [
Effort::Minimal,
Effort::Low,
Effort::Medium,
Effort::High,
Effort::Xhigh,
] {
let e = effort.as_str();
let codex = InvocationPlan::for_harness(ReviewerHarness::Codex, "m", effort).unwrap();
assert!(codex.args.contains(&format!("model_reasoning_effort={e}")));
let claude = InvocationPlan::for_harness(ReviewerHarness::Claude, "m", effort).unwrap();
let claude_idx = claude.args.iter().position(|a| a == "--effort").unwrap();
// Claude has no `minimal`; it clamps to a valid level (`low`).
assert_eq!(claude.args[claude_idx + 1], effort.claude_value());
assert_ne!(claude.args[claude_idx + 1], "minimal");
let pi = InvocationPlan::for_harness(ReviewerHarness::Pi, "m", effort).unwrap();
let pi_idx = pi.args.iter().position(|a| a == "--thinking").unwrap();
assert_eq!(pi.args[pi_idx + 1], e);
}
}
#[test]
fn resolve_picks_configured_reviewer_for_every_writer() {
let config = crate::config::TruthMirrorConfig::default();
let cases = [
(Agent::Codex, ReviewerHarness::Claude, "claude-opus-4-8"),
(Agent::Claude, ReviewerHarness::Codex, "gpt-5.5"),
(Agent::Pi, ReviewerHarness::Codex, "gpt-5.5"),
(Agent::Grok, ReviewerHarness::Claude, "claude-opus-4-8"),
];
for (writer, reviewer_harness, reviewer_model) in cases {
let selection =
ReviewSelection::resolve(Some(writer), None, None, None, None, false, &config)
.unwrap();
assert_eq!(selection.reviewer_harness, reviewer_harness);
assert_eq!(selection.reviewer_model, reviewer_model);
assert_eq!(selection.reviewer_effort, Effort::Xhigh);
}
}
#[test]
fn overriding_reviewer_harness_without_model_is_rejected() {
// codex's default pair reviewer is claude; overriding harness to pi with no
// model would pair the pi harness with a claude model string.
let config = crate::config::TruthMirrorConfig::default();
let error = ReviewSelection::resolve(
Some(Agent::Codex),
None,
Some(ReviewerHarness::Pi),
None,
None,
false,
&config,
)
.unwrap_err();
assert!(matches!(error, ReviewerError::OverrideNeedsModel { .. }));
}
#[test]
fn overriding_reviewer_harness_matching_pair_is_ok() {
let config = crate::config::TruthMirrorConfig::default();
let selection = ReviewSelection::resolve(
Some(Agent::Codex),
None,
Some(ReviewerHarness::Claude),
None,
None,
false,
&config,
)
.unwrap();
assert_eq!(selection.reviewer_harness, ReviewerHarness::Claude);
assert_eq!(selection.reviewer_model, "claude-opus-4-8");
}
#[test]
fn config_allow_same_model_waives_opposition() {
let config = crate::config::TruthMirrorConfig {
allow_same_model: true,
..crate::config::TruthMirrorConfig::default()
};
let selection = ReviewSelection::resolve(
Some(Agent::Codex),
Some("gpt-5.5".to_owned()),
Some(ReviewerHarness::Codex),
Some("gpt-5.5".to_owned()),
None,
false, // CLI flag not set — the config waiver must carry it
&config,
)
.unwrap();
assert!(selection.allow_same_model);
// Same watched+reviewer model builds because the config waiver applies.
assert!(ReviewPlan::build(selection.request_for("review".to_owned())).is_ok());
}
#[test]
fn full_git_sha_accepts_sha1_and_sha256_lengths() {
assert!(is_full_git_sha(&"a".repeat(40)));
assert!(is_full_git_sha(&"b".repeat(64)));
assert!(!is_full_git_sha(&"c".repeat(39)));
assert!(!is_full_git_sha(&"g".repeat(40)));
}
#[test]
fn resolve_arbiter_uses_pair_when_cli_absent() {
let config = crate::config::TruthMirrorConfig::default();
let arbiter =
ReviewSelection::resolve_arbiter(Agent::Codex, None, None, None, &config).unwrap();
assert_eq!(arbiter.arbiter_harness, ReviewerHarness::Pi);
assert_eq!(arbiter.arbiter_effort, Effort::Xhigh);
}
#[test]
fn first_pass_prompt_is_adversarial_and_injects_context() {
let prompt = super::first_pass_prompt(
&claim(),
"THE_DIFF_BODY",
"INVIOLABLE CONSTRAINTS: never fake tests",
);
assert!(prompt.contains("PROVE THIS CLAIM FALSE"));
assert!(prompt.contains("default to REJECT"));
assert!(prompt.contains("INVIOLABLE CONSTRAINTS: never fake tests"));
assert!(prompt.contains("THE_DIFF_BODY"));
// Class-generalized review: grep the class, not the instance.
assert!(prompt.contains("GREP THE CLASS, NOT THE INSTANCE"));
assert!(prompt.contains("\"severity\""));
assert!(prompt.contains("\"recommendation\""));
assert!(prompt.contains("\"memory_skill\""));
assert!(prompt.contains("\"anti_pattern_skill\""));
}
#[test]
fn strict_second_pass_is_a_completeness_critic() {
let job = review_job(true);
let first_output = pass_json();
let prompt = super::strict_second_pass_prompt(&job, &first_output);
assert!(prompt.contains("COMPLETENESS CRITIC"));
assert!(prompt.contains("generalize"));
// Inherits the class-sweep preamble.
assert!(prompt.contains("GREP THE CLASS, NOT THE INSTANCE"));
}
#[test]
fn prompt_omits_context_block_when_empty() {
let prompt = super::first_pass_prompt(&claim(), "d", "");
// No dangling empty context header.
assert!(!prompt.contains("INVIOLABLE CONSTRAINTS"));
assert!(prompt.contains("PROVE THIS CLAIM FALSE"));
}
fn sample_petition(attempts_so_far: u32) -> super::PetitionContext {
super::PetitionContext {
original_sha: "abc123".to_owned(),
fix_sha: "def456".to_owned(),
original_claim: "CLAIM: original | verified: cargo test | evidence: tests:cargo-test"
.to_owned(),
original_summary: "Original rejection summary.".to_owned(),
original_findings: vec!["evidence too thin".to_owned()],
original_structured_findings: vec![StructuredFinding {
severity: FindingSeverity::High,
title: "thin evidence".to_owned(),
body: "Evidence pointer does not prove the claim.".to_owned(),
file: "src/lib.rs".to_owned(),
line_start: 1,
line_end: 2,
confidence: 90,
recommendation: "Add executable evidence.".to_owned(),
}],
original_reviewer_model: "claude-opus-4-1".to_owned(),
attempts_so_far,
}
}
#[test]
fn petition_prompt_includes_original_findings_fix_sha_and_attempts() {
let prompt = super::petition_prompt(&sample_petition(1), &claim(), "DIFF BODY", "");
assert!(prompt.contains("PETITION"));
assert!(prompt.contains("original rejection SHA: abc123"));
assert!(prompt.contains("fix commit SHA: def456"));
assert!(prompt.contains("petition attempts so far: 1"));
assert!(prompt.contains("Original rejection summary."));
assert!(prompt.contains("thin evidence")); // structured finding title
assert!(prompt.contains("DIFF BODY"));
assert!(prompt.contains("\"verdict\": \"PASS\" | \"REJECT\" | \"FLAG\""));
}
#[test]
fn materialize_petition_prompt_rewrites_request_prompt() {
let mut job = review_job(false);
job.petition = Some(sample_petition(2));
let original_prompt = job.request.prompt.clone();
let updated = super::materialize_petition_prompt(job);
assert_ne!(updated.request.prompt, original_prompt);
assert!(updated.request.prompt.contains("PETITION"));
assert!(updated.request.prompt.contains("fix commit SHA: def456"));
assert!(
updated
.request
.prompt
.contains("petition attempts so far: 2")
);
}
#[test]
fn materialize_petition_prompt_passthrough_when_no_petition() {
let job = review_job(false);
let original_prompt = job.request.prompt.clone();
let updated = super::materialize_petition_prompt(job);
assert_eq!(updated.request.prompt, original_prompt);
}
#[test]
fn subprocess_runner_is_mockable() {
struct MockRunner;
impl ProcessRunner for MockRunner {
fn run(
&self,
invocation: &InvocationPlan,
prompt: &str,
_timeout: Option<Duration>,
) -> Result<ProcessOutput, ReviewerError> {
assert_eq!(invocation.program, "codex");
assert_eq!(
invocation.args_for_prompt(prompt).last().unwrap(),
"review this"
);
Ok(ProcessOutput {
status_code: Some(0),
stdout: pass_json(),
stderr: String::new(),
})
}
}
let request = ReviewRequest::new(
Agent::Codex,
"gpt-5.4",
ReviewerHarness::Codex,
"gpt-5.5",
false,
"review this",
);
let plan = ReviewPlan::build(request).unwrap();
let output = plan.run_with("review this", &MockRunner, None).unwrap();
assert!(output.stdout.contains("PASS"));
}
#[cfg(unix)]
#[test]
fn std_process_runner_kills_wedged_child_at_timeout() {
// 0.9.2 field failure: `codex exec` reviewers wedged for 1–7 hours and
// froze the whole queue. The runner must kill the child at the
// configured wall-clock deadline and surface a timeout error.
let invocation = InvocationPlan {
program: "sleep".to_owned(),
args: vec!["30".to_owned()],
prompt_delivery: PromptDelivery::Stdin,
};
let runner = super::StdProcessRunner;
let started = std::time::Instant::now();
let error = runner
.run(&invocation, "", Some(Duration::from_millis(300)))
.unwrap_err();
let elapsed = started.elapsed();
assert!(
matches!(error, ReviewerError::ReviewerTimeout { .. }),
"expected a timeout error, got: {error}"
);
assert!(
elapsed < Duration::from_secs(10),
"the wedged child was not killed promptly (elapsed: {elapsed:?})"
);
}
#[cfg(unix)]
#[test]
fn std_process_runner_collects_output_without_timeout() {
// The no-timeout path keeps the old wait_with_output behavior: exit
// status plus both pipes captured.
let invocation = InvocationPlan {
program: "echo".to_owned(),
args: vec!["hello".to_owned()],
prompt_delivery: PromptDelivery::PositionalArgument,
};
let runner = super::StdProcessRunner;
let output = runner.run(&invocation, "", None).unwrap();
assert_eq!(output.status_code, Some(0));
assert_eq!(output.stdout.trim(), "hello");
}
#[cfg(unix)]
#[test]
fn std_process_runner_timeout_covers_blocked_stdin_write() {
// 0.9.2 field failure, first fix iteration: the deadline started only
// AFTER the blocking prompt `write_all` returned, so a reviewer that
// never reads stdin wedged the queue forever. `sleep` reads nothing;
// a prompt larger than any pipe buffer (64 KiB) blocks the write, and
// the deadline must still fire. (Against the old code this test hangs
// rather than fails — that hang IS the field failure.)
let invocation = InvocationPlan {
program: "sleep".to_owned(),
args: vec!["30".to_owned()],
prompt_delivery: PromptDelivery::Stdin,
};
let runner = super::StdProcessRunner;
let huge_prompt = "x".repeat(2 * 1024 * 1024);
let started = std::time::Instant::now();
let error = runner
.run(&invocation, &huge_prompt, Some(Duration::from_millis(300)))
.unwrap_err();
let elapsed = started.elapsed();
assert!(
matches!(error, ReviewerError::ReviewerTimeout { .. }),
"expected a timeout error, got: {error}"
);
assert!(
elapsed < Duration::from_secs(10),
"the deadline must cover the blocked stdin write (elapsed: {elapsed:?})"
);
}
#[cfg(unix)]
#[test]
fn std_process_runner_timeout_kills_descendants_holding_pipes() {
// 0.9.2 queue freeze, second half: the reviewer spawns a descendant
// that inherits the output pipes. Killing only the direct child left
// the descendant holding the pipes open, so the reader-thread join
// blocked until the descendant exited (~30s here — forever in the
// field). The timeout kill must take the whole process group.
let invocation = InvocationPlan {
program: "sh".to_owned(),
args: vec!["-c".to_owned(), "sleep 30 & sleep 30".to_owned()],
prompt_delivery: PromptDelivery::PositionalArgument,
};
let runner = super::StdProcessRunner;
let started = std::time::Instant::now();
let error = runner
.run(&invocation, "", Some(Duration::from_millis(300)))
.unwrap_err();
let elapsed = started.elapsed();
assert!(
matches!(error, ReviewerError::ReviewerTimeout { .. }),
"expected a timeout error, got: {error}"
);
assert!(
elapsed < Duration::from_secs(10),
"a pipe-hoarding descendant must not stall the reap (elapsed: {elapsed:?})"
);
}
#[cfg(unix)]
#[test]
fn std_process_runner_timeout_kills_spawned_descendants() {
// The descendant must not SURVIVE the timeout kill either: the
// backgrounded subshell would write its sentinel 2s in, well after
// the 300ms deadline. A live sentinel proves the tree kill missed it.
let temp = tempfile::tempdir().unwrap();
let sentinel = temp.path().join("descendant-survived");
let invocation = InvocationPlan {
program: "sh".to_owned(),
args: vec![
"-c".to_owned(),
"(sleep 2; touch \"$1\") & exec sleep 30".to_owned(),
"sentinel".to_owned(),
sentinel.display().to_string(),
],
prompt_delivery: PromptDelivery::PositionalArgument,
};
let runner = super::StdProcessRunner;
let error = runner
.run(&invocation, "", Some(Duration::from_millis(300)))
.unwrap_err();
assert!(
matches!(error, ReviewerError::ReviewerTimeout { .. }),
"expected a timeout error, got: {error}"
);
std::thread::sleep(Duration::from_millis(2500));
assert!(
!sentinel.exists(),
"the spawned descendant survived the timeout kill and wrote its sentinel"
);
}
#[cfg(unix)]
#[test]
fn outcome_after_kill_always_reports_timeout_regardless_of_reap_outcome() {
use std::os::unix::process::ExitStatusExt as _;
// CodeRabbit MAJOR (PR #13, reviewer.rs ~750): this used to let a
// child that resolved on its own (no signal on the reaped status)
// "win" the kill race and return Ok, so an over-deadline review could
// succeed or time out depending on scheduling — nondeterministic,
// and inconsistent with the documented wall-clock contract that a
// review either finishes within budget or times out, never both
// depending on luck. Once `kill_and_reap` is reached, the deadline
// was already crossed, so the outcome must always be the timeout —
// clean exit, nonzero exit, or signal-killed alike.
assert!(matches!(
super::outcome_after_kill(std::process::ExitStatus::from_raw(0), 1),
Err(ReviewerError::ReviewerTimeout { timeout_secs: 1 })
));
assert!(matches!(
super::outcome_after_kill(std::process::ExitStatus::from_raw(1 << 8), 1),
Err(ReviewerError::ReviewerTimeout { timeout_secs: 1 })
));
assert!(matches!(
super::outcome_after_kill(std::process::ExitStatus::from_raw(9), 7),
Err(ReviewerError::ReviewerTimeout { timeout_secs: 7 })
));
}
#[cfg(unix)]
#[test]
fn wait_for_child_kills_and_times_out_a_still_running_child_past_its_deadline() {
// CodeRabbit MAJOR (PR #13, reviewer.rs ~505/~750): the poll loop
// checked `try_wait()` before checking the deadline, so a poll tick
// landing after the deadline had already passed could still accept
// whatever status the child happened to reach in that same tick as a
// clean "success" — bypassing kill_and_reap (and its timeout
// reporting) entirely. A child that is STILL RUNNING at an
// already-elapsed (backdated) deadline must always be killed and
// reported as a timeout — that invariant is unaffected by round 2's
// Cluster D fix (a final glance before kill_and_reap), since a
// still-running child fails that glance too.
for _ in 0..20 {
let child = std::process::Command::new("sleep")
.arg("30")
.spawn()
.unwrap();
let deadline = std::time::Instant::now()
.checked_sub(Duration::from_millis(50))
.expect("test clock has enough headroom to subtract 50ms");
let result = super::wait_for_child(child, Some((deadline, Duration::from_millis(1))));
assert!(
matches!(result, Err(ReviewerError::ReviewerTimeout { .. })),
"a still-running child past its deadline must always be \
killed and reported as a timeout: got {result:?}"
);
}
}
#[cfg(unix)]
#[test]
fn wait_for_child_accepts_a_status_already_settled_before_any_kill_action() {
// Codex round-2 P2 (reviewer.rs ~806): CHILD_POLL_INTERVAL (100ms) is
// coarser than a short configured timeout can be, and even a
// generous one can have the child exit during the final sleep that
// straddles the deadline instant. Without a one-time final glance
// right when the deadline trips — BEFORE any kill action — a child
// that had already finished (well within whatever real budget the
// deadline represents) gets misreported as timed out purely because
// our own poll granularity hadn't looked yet. This is a real,
// pre-existing gap between "the deadline instant" and "the next time
// our loop happens to check," not a reopening of the round-1 fix:
// once `kill_and_reap` is actually invoked, the outcome remains
// unconditionally a timeout regardless of how the child then
// resolves (see `wait_for_child_kills_and_times_out_a_still_running_child_past_its_deadline`
// and `outcome_after_kill_always_reports_timeout_regardless_of_reap_outcome`).
for _ in 0..20 {
let child = std::process::Command::new(std::env::current_exe().unwrap())
.arg("--help")
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.spawn()
.unwrap();
// Give the child time to actually exit before wait_for_child ever
// calls try_wait, so the deadline-trip glance is guaranteed to
// observe a completed, unreaped zombie rather than racing it.
std::thread::sleep(Duration::from_millis(100));
let deadline = std::time::Instant::now()
.checked_sub(Duration::from_millis(50))
.expect("test clock has enough headroom to subtract 50ms");
let result = super::wait_for_child(child, Some((deadline, Duration::from_millis(1))));
assert!(
result.is_ok(),
"a child already settled by the time the deadline trips must \
not be killed and reported as a timeout: got {result:?}"
);
}
}
#[test]
fn pipe_reader_try_collect_treats_grace_as_remaining_budget_from_a_shared_deadline() {
// CodeRabbit MAJOR (PR #13, reviewer.rs ~505): stdout and stderr each
// used to get their OWN fresh PIPE_DRAIN_GRACE, so N simultaneously
// wedged pipes cost roughly Nx the grace. The fix computes ONE
// absolute cleanup deadline and passes each collect only the budget
// REMAINING against it. Exercised directly against `PipeReader`
// (rather than through a real OS pipe/process) so the assertion is
// deterministic instead of depending on shell/process timing: a
// deadline that has already elapsed by the time of the call must be
// treated as "no time left", not "a fresh grace window".
let (_sender, receiver) = std::sync::mpsc::channel::<std::io::Result<Vec<u8>>>();
let mut reader = super::PipeReader {
handle: Some(std::thread::spawn(|| {
std::thread::sleep(Duration::from_secs(3));
})),
receiver,
};
let already_elapsed_deadline = std::time::Instant::now()
.checked_sub(Duration::from_secs(1))
.expect("test clock has enough headroom to subtract 1s");
let started = std::time::Instant::now();
let state = reader.try_collect(
super::remaining_until(Some(already_elapsed_deadline)),
12345,
"test",
);
let elapsed = started.elapsed();
assert!(
matches!(state, super::DrainState::Wedged),
"a reader whose sender never fires must be reported wedged"
);
assert!(
elapsed < Duration::from_millis(500),
"a shared deadline that already elapsed must not grant a fresh \
grace window: elapsed {elapsed:?}"
);
}
#[test]
fn pipe_reader_try_collect_takes_the_wedged_handle_without_spawning_a_reaper() {
// Round 1 (CodeRabbit MAJOR, PR #13 reviewer.rs ~690): a wedged
// reader thread used to be left sitting in `self.handle`, so the
// struct's implicit drop dropped the `JoinHandle` without ever
// joining it — silently detaching an OS thread that stays blocked in
// `read_to_end` for as long as its pipe never closes.
//
// Round 2 (CodeRabbit/Codex, both independently, twice each): round
// 1's fix "solved" this by spawning a background reaper thread to
// join the handle — but that reaper's own join blocks forever too if
// the helper never returns (a descendant escaped the process-tree
// kill and still holds the pipe open), so each wedge then leaked TWO
// threads instead of one. The fix takes the handle (so it is never
// left for an implicit silent drop) but does NOT spawn a second
// thread to join it — it logs the detection and lets it drop.
let (_sender, receiver) = std::sync::mpsc::channel::<std::io::Result<Vec<u8>>>();
let mut reader = super::PipeReader {
handle: Some(std::thread::spawn(|| {
std::thread::sleep(Duration::from_secs(3));
})),
receiver,
};
let started = std::time::Instant::now();
let state = reader.try_collect(Some(Duration::from_millis(1)), 99999, "test");
let elapsed = started.elapsed();
assert!(matches!(state, super::DrainState::Wedged));
assert!(
reader.handle.is_none(),
"a wedged handle must be taken, not left in place for an implicit silent drop"
);
assert!(
elapsed < Duration::from_secs(1),
"try_collect must not block on a reaper's join — the wedged \
helper here sleeps 3s, so blocking on any join of it would show \
up as ~3s elapsed here: elapsed {elapsed:?}"
);
}
#[cfg(unix)]
#[test]
fn wait_for_reap_enforces_grace_and_surfaces_timeout() {
let mut child = std::process::Command::new("sh")
.arg("-c")
.arg("sleep 5")
.spawn()
.unwrap();
let started = std::time::Instant::now();
let error = super::wait_for_reap(&mut child, Duration::from_millis(100))
.expect_err("still-running child must not block forever");
let elapsed = started.elapsed();
assert_eq!(error.kind(), std::io::ErrorKind::Other);
assert!(
elapsed >= Duration::from_millis(100),
"reap timeout must wait for the configured grace; elapsed {elapsed:?}"
);
assert!(
elapsed < Duration::from_secs(1),
"reap timeout must be bounded"
);
child.kill().ok();
child.wait().ok();
}
#[cfg(unix)]
#[test]
fn reap_after_grace_returns_after_bounded_cleanup_handoff() {
let child = std::process::Command::new("sh")
.arg("-c")
.arg("sleep 5")
.spawn()
.unwrap();
let started = std::time::Instant::now();
let error = super::reap_after_grace(child, std::io::Error::other("test grace expired"), 7)
.unwrap_err();
assert!(matches!(
error,
ReviewerError::ReviewerTimeout { timeout_secs: 7 }
));
assert!(
started.elapsed() < Duration::from_secs(1),
"post-kill cleanup must not block on direct-child wait"
);
}
#[cfg(unix)]
#[test]
fn std_process_runner_surfaces_prompt_write_failure() {
// A reviewer that closes stdin immediately makes the prompt write
// fail with EPIPE. The error must surface (a truncated question
// invalidates the verdict), never be swallowed. The prompt is larger
// than any pipe buffer so the writer is still blocked when `exec
// 0<&-` closes the read end — no timing race.
let invocation = InvocationPlan {
program: "sh".to_owned(),
args: vec!["-c".to_owned(), "exec 0<&-; sleep 1".to_owned()],
prompt_delivery: PromptDelivery::Stdin,
};
let runner = super::StdProcessRunner;
let huge_prompt = "x".repeat(2 * 1024 * 1024);
let error = runner
.run(&invocation, &huge_prompt, Some(Duration::from_secs(10)))
.unwrap_err();
assert!(
matches!(error, ReviewerError::WritePrompt(_)),
"expected a prompt-write error, got: {error}"
);
}
#[test]
fn claim_watcher_lock_owns_free_slot_and_adopted_handoff() {
let temp = tempfile::tempdir().unwrap();
let state = temp.path();
// A free slot is claimed.
assert_eq!(
claim_watcher_lock(state, false, Duration::from_millis(10), false).unwrap(),
WatchLockClaim::Owned
);
let token = crate::watcher::lock_acquisition_token(state).unwrap();
crate::watcher::release_lock_if_owned(state, &token).unwrap();
// The ensure-watcher handoff: the lock already records THIS process
// (ensure-watcher re-points it at the detached child). The spawned
// watcher must adopt its own slot, not refuse it as contention.
assert_eq!(
crate::watcher::try_acquire_lock(state).unwrap(),
crate::watcher::LockClaim::Acquired
);
assert_eq!(
claim_watcher_lock(state, false, Duration::from_millis(10), false).unwrap(),
WatchLockClaim::Owned
);
let token = crate::watcher::lock_acquisition_token(state).unwrap();
crate::watcher::release_lock_if_owned(state, &token).unwrap();
}
#[test]
fn run_store_lock_is_stale_only_when_owner_is_proven_dead() {
let temp = tempfile::tempdir().unwrap();
let root = temp.path();
let lock_dir = root.join(RUN_STORE_LOCK_DIR);
fs::create_dir_all(&lock_dir).unwrap();
// Current process owner keeps the lock live even if the lock files
// themselves are otherwise old.
let owner = ProcessIdentity::current();
write_run_store_lock_owner(&lock_dir, &owner).unwrap();
assert!(
!run_store_lock_is_stale(&lock_dir),
"lock with matching owner identity should not be reclaimed"
);
// An unprobeable pid remains Unknown even after the heartbeat has gone
// stale; heartbeat age alone is never a reclaim proof.
let unknown = ProcessIdentity {
pid: 4_000_000_000,
start_token: owner.start_token.clone(),
};
write_run_store_lock_owner(&lock_dir, &unknown).unwrap();
std::thread::sleep(Duration::from_millis(350));
assert!(
!run_store_lock_is_stale(&lock_dir),
"an Unknown owner must not be reclaimed from heartbeat age alone"
);
// A real reaped child is proven dead and becomes reclaimable only after
// the configured missed-heartbeat threshold.
let dead = ProcessIdentity {
pid: reaped_pid(),
start_token: owner.start_token.clone(),
};
write_run_store_lock_owner(&lock_dir, &dead).unwrap();
std::thread::sleep(Duration::from_millis(350));
assert!(
run_store_lock_is_stale(&lock_dir),
"a proven-dead owner with no refreshed heartbeat should be reclaimable"
);
}
#[test]
fn run_store_lock_heartbeat_refreshes_while_held() {
let temp = tempfile::tempdir().unwrap();
let store = ReviewRunStore::new(temp.path());
let guard = store.lock().unwrap();
let before = fs::metadata(&guard.path).unwrap().modified().unwrap();
std::thread::sleep(Duration::from_millis(250));
let after = fs::metadata(&guard.path).unwrap().modified().unwrap();
assert!(
after > before,
"the held run-store lock heartbeat must refresh its lease mtime"
);
}
#[test]
fn run_store_lock_drop_does_not_remove_same_owner_replacement() {
let temp = tempfile::tempdir().unwrap();
let store = ReviewRunStore::new(temp.path());
let guard = store.lock().unwrap();
let replacement = RunStoreLockRecord {
owner: guard.owner.clone(),
acquisition_token: "replacement-token".to_owned(),
};
fs::write(&guard.path, serde_json::to_vec(&replacement).unwrap()).unwrap();
let lock_path = guard.path.clone();
drop(guard);
assert!(
lock_path.exists(),
"a replacement with the same process identity must survive the old guard"
);
}
#[test]
fn run_store_lock_drop_removes_only_owned_lock() {
let temp = tempfile::tempdir().unwrap();
let store = ReviewRunStore::new(temp.path());
let lock_path = store.root.join(RUN_STORE_LOCK_DIR);
drop(store.lock().unwrap());
assert!(!lock_path.exists(), "owned lock should be released on drop");
}
#[test]
fn run_store_reclaim_gate_never_admits_two_holders_at_once() {
// CodeRabbit/Gemini/Codex round-2 CRITICAL: the round-1 reclaim gate
// used a create_new marker file plus an mtime-based staleness
// heuristic to force-reclaim a stranded gate — which repeats the
// exact TOCTOU it exists to close (two contenders can both decide the
// marker is stale and race to unlink it, letting both end up
// "holding" the gate). Backing it with a real OS advisory lock
// (flock) removes the heuristic entirely: the kernel enforces
// exclusivity, so this must hold even under heavy concurrent
// contention with no artificial timing help.
const THREADS: usize = 8;
let temp = tempfile::tempdir().unwrap();
let root = Arc::new(temp.path().to_path_buf());
let barrier = Arc::new(Barrier::new(THREADS));
let currently_held = Arc::new(AtomicUsize::new(0));
let max_concurrently_held = Arc::new(AtomicUsize::new(0));
let mut threads = Vec::with_capacity(THREADS);
for _ in 0..THREADS {
let root = Arc::clone(&root);
let barrier = Arc::clone(&barrier);
let currently_held = Arc::clone(¤tly_held);
let max_concurrently_held = Arc::clone(&max_concurrently_held);
threads.push(std::thread::spawn(move || {
barrier.wait();
loop {
if let Some(_gate) = super::acquire_run_store_reclaim_gate(root.as_ref())
.expect("gate acquisition should not error")
{
let now_held = currently_held.fetch_add(1, Ordering::SeqCst) + 1;
max_concurrently_held.fetch_max(now_held, Ordering::SeqCst);
std::thread::sleep(Duration::from_millis(15));
currently_held.fetch_sub(1, Ordering::SeqCst);
break;
}
std::thread::sleep(Duration::from_millis(2));
}
}));
}
for thread in threads {
thread.join().unwrap();
}
assert_eq!(
max_concurrently_held.load(Ordering::SeqCst),
1,
"the reclaim gate must never be held by more than one contender at once"
);
}
#[test]
fn threaded_stale_run_store_lock_reclaim_race_never_double_admits() {
// CodeRabbit CRITICAL (PR #13): two contenders could both observe the
// old lock as stale; the first's reclaim-and-install could then be
// deleted by the second's already-decided removal, letting both
// believe they held the run-store mutation lock at once. Race N
// contenders against one planted stale lock and assert the lock is
// never observed held by more than one at a time. The reclaim gate
// that serializes this is now backed by a real OS advisory lock (see
// `run_store_reclaim_gate_never_admits_two_holders_at_once`), so this
// holds deterministically — no artificial delay needed to widen the
// race window.
const THREADS: usize = 8;
let temp = tempfile::tempdir().unwrap();
let store = Arc::new(ReviewRunStore::new(temp.path()));
let lock_path = store.root.join(RUN_STORE_LOCK_DIR);
fs::create_dir_all(&store.root).unwrap();
let stale = RunStoreLockRecord {
owner: ProcessIdentity {
pid: reaped_pid(),
start_token: "definitely-not-a-live-token".to_owned(),
},
acquisition_token: "stale-token".to_owned(),
};
fs::write(&lock_path, serde_json::to_vec(&stale).unwrap()).unwrap();
// Let the heartbeat-staleness window (3 missed 100ms intervals) elapse
// so every thread's first check already sees the lock as reclaimable.
std::thread::sleep(Duration::from_millis(350));
let barrier = Arc::new(Barrier::new(THREADS));
let currently_held = Arc::new(AtomicUsize::new(0));
let max_concurrently_held = Arc::new(AtomicUsize::new(0));
let mut threads = Vec::with_capacity(THREADS);
for _ in 0..THREADS {
let store = Arc::clone(&store);
let barrier = Arc::clone(&barrier);
let currently_held = Arc::clone(¤tly_held);
let max_concurrently_held = Arc::clone(&max_concurrently_held);
threads.push(std::thread::spawn(move || {
barrier.wait();
let guard = store.lock().expect("lock should eventually be reclaimed");
let now_held = currently_held.fetch_add(1, Ordering::SeqCst) + 1;
max_concurrently_held.fetch_max(now_held, Ordering::SeqCst);
std::thread::sleep(Duration::from_millis(15));
currently_held.fetch_sub(1, Ordering::SeqCst);
drop(guard);
}));
}
for thread in threads {
thread.join().unwrap();
}
assert_eq!(
max_concurrently_held.load(Ordering::SeqCst),
1,
"run-store lock must never be held by more than one contender at \
once, even when several contenders race a stale-lock reclaim"
);
}
#[cfg(unix)]
struct KillGuard(std::process::Child);
#[cfg(unix)]
impl Drop for KillGuard {
fn drop(&mut self) {
let _ = self.0.kill();
let _ = self.0.wait();
}
}
#[cfg(unix)]
#[test]
fn claim_watcher_lock_refuses_foreign_live_owner_without_wait() {
// 0.9.2 field failure: a second watcher proceeded "without lock
// ownership" and double-processed the queue. Now a foreign-held lock
// means refusal — fast, and without disturbing the incumbent's lock.
let temp = tempfile::tempdir().unwrap();
let state = temp.path();
let sleeper = Command::new("sleep").arg("30").spawn().unwrap();
let guard = KillGuard(sleeper);
// Empty start token: liveness degrades to pid-alive, which the
// sleeper satisfies, so the lock counts as held by a live watcher.
let lock = crate::watcher::WatcherLock {
identity: crate::watcher::ProcessIdentity {
pid: guard.0.id(),
start_token: String::new(),
},
acquisition_token: "foreign-token".to_owned(),
created_at_unix: 1,
};
fs::create_dir_all(state).unwrap();
fs::write(
state.join(crate::watcher::WATCHER_LOCK_FILE),
serde_json::to_vec_pretty(&lock).unwrap(),
)
.unwrap();
let started = std::time::Instant::now();
let claim = claim_watcher_lock(state, false, Duration::from_millis(10), false).unwrap();
assert!(
matches!(claim, WatchLockClaim::Refused(_)),
"a foreign live owner must refuse the claim, got {claim:?}"
);
assert!(
started.elapsed() < Duration::from_secs(5),
"refusal must be immediate when --wait-for-lock is not given"
);
// The incumbent's lock file was neither stolen nor deleted.
assert!(state.join(crate::watcher::WATCHER_LOCK_FILE).exists());
}
#[cfg(unix)]
#[test]
fn claim_watcher_lock_absorbs_ensure_watcher_handoff_window() {
// The detached child can check the lock BEFORE ensure-watcher
// re-points it at the child's identity. With expect_handoff the child
// must wait out the window and adopt the slot once the re-point
// lands, instead of refusing and stranding the queue when the parent
// exits. Without expect_handoff the same setup refuses immediately
// (covered by claim_watcher_lock_refuses_foreign_live_owner_without_wait).
let temp = tempfile::tempdir().unwrap();
let state = temp.path().to_path_buf();
// A foreign live owner: a sleeper pid with an empty token.
let sleeper = Command::new("sleep").arg("30").spawn().unwrap();
let guard = KillGuard(sleeper);
fs::create_dir_all(&state).unwrap();
fs::write(
state.join(crate::watcher::WATCHER_LOCK_FILE),
serde_json::to_vec_pretty(&crate::watcher::WatcherLock {
identity: crate::watcher::ProcessIdentity {
pid: guard.0.id(),
start_token: String::new(),
},
acquisition_token: "handoff-token".to_owned(),
created_at_unix: crate::time::unix_now(),
})
.unwrap(),
)
.unwrap();
// Re-point the lock at THIS process after 300ms — exactly what
// ensure-watcher does right after spawn returns.
let re_point = std::thread::spawn({
let state = state.clone();
move || {
std::thread::sleep(Duration::from_millis(300));
let lock = crate::watcher::WatcherLock {
identity: crate::watcher::ProcessIdentity::current(),
acquisition_token: "handoff-token".to_owned(),
created_at_unix: crate::time::unix_now(),
};
fs::write(
state.join(crate::watcher::WATCHER_LOCK_FILE),
serde_json::to_vec_pretty(&lock).unwrap(),
)
.unwrap();
}
});
let started = std::time::Instant::now();
let claim = claim_watcher_lock(&state, false, Duration::from_millis(50), true).unwrap();
assert_eq!(claim, WatchLockClaim::Owned);
assert!(
started.elapsed() >= Duration::from_millis(250),
"the claim must have waited for the handoff (elapsed: {:?})",
started.elapsed()
);
assert!(
started.elapsed() < Duration::from_secs(5),
"the handoff must be adopted promptly once it lands"
);
re_point.join().unwrap();
crate::watcher::release_lock_if_owned(&state, "handoff-token").unwrap();
}
#[test]
fn drain_once_marks_timed_out_run_failed_and_keeps_draining() {
// A wedged reviewer must not strand the queue: the timed out item is
// recorded failed with the timeout reason and consumed, and the next
// queued commit is still reviewed in the same drain.
struct TimeoutThenPassRunner {
calls: RefCell<usize>,
observed_timeout: RefCell<Option<Duration>>,
}
impl ProcessRunner for TimeoutThenPassRunner {
fn run(
&self,
_invocation: &InvocationPlan,
_prompt: &str,
timeout: Option<Duration>,
) -> Result<ProcessOutput, ReviewerError> {
*self.observed_timeout.borrow_mut() = timeout;
let call = {
let mut calls = self.calls.borrow_mut();
*calls += 1;
*calls
};
if call == 1 {
return Err(ReviewerError::ReviewerTimeout { timeout_secs: 1200 });
}
Ok(ProcessOutput {
status_code: Some(0),
stdout: pass_json(),
stderr: String::new(),
})
}
}
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let queue = ReviewQueue::new(temp.path());
queue.enqueue("wedged").unwrap();
queue.enqueue("healthy").unwrap();
let runner = TimeoutThenPassRunner {
calls: RefCell::new(0),
observed_timeout: RefCell::new(None),
};
let report = drain_once(
&queue,
&StaticLoader::new(),
&selection(),
"",
&runner,
&store,
&TruthMirrorConfig::default(),
)
.unwrap();
// The configured timeout (default 20 minutes) reached the runner.
assert_eq!(
*runner.observed_timeout.borrow(),
Some(Duration::from_secs(20 * 60))
);
// The wedged item was skipped as failed; the healthy one reviewed.
assert_eq!(report.failed, ["wedged"]);
assert_eq!(report.reviewed, ["healthy"]);
assert!(queue.pending().unwrap().is_empty());
let run_store = ReviewRunStore::new(temp.path());
let runs = run_store.list().unwrap();
let wedged = runs.iter().find(|run| run.commit_sha == "wedged").unwrap();
assert_eq!(wedged.status, ReviewRunStatus::Failed);
assert!(
wedged
.error
.as_deref()
.is_some_and(|error| error.contains("timed out")),
"timeout reason should be recorded, got: {:?}",
wedged.error
);
let healthy = runs.iter().find(|run| run.commit_sha == "healthy").unwrap();
assert_eq!(healthy.status, ReviewRunStatus::Completed);
}
#[test]
fn verdict_parser_extracts_rejection_findings() {
let verdict = ParsedVerdict::parse(&reject_json("missing proof")).unwrap();
assert_eq!(verdict.verdict, Verdict::Reject);
assert_eq!(verdict.structured_findings[0].title, "missing proof");
assert_eq!(verdict.structured_findings[0].confidence, 95);
assert!(verdict.findings[0].contains("missing proof"));
assert_eq!(
verdict.memory_skill_classification.learning_source,
"missing proof"
);
}
fn flag_json(title: &str) -> String {
serde_json::json!({
"verdict": "FLAG",
"summary": "Claim substantiated; flagging thin evidence.",
"findings": [{
"severity": "medium",
"title": title,
"body": "Evidence pointer is thin but the claim holds.",
"file": "src/lib.rs",
"line_start": 1,
"line_end": 1,
"confidence": 60,
"recommendation": "Tighten evidence."
}],
"next_steps": ["Add stronger evidence."],
"memory_skill": {
"kind": "none",
"learning_source": "",
"reasoning": "No reusable procedural memory to propose."
}
})
.to_string()
}
#[test]
fn verdict_parser_extracts_flag_findings() {
let verdict = ParsedVerdict::parse(&flag_json("thin evidence")).unwrap();
assert_eq!(verdict.verdict, Verdict::Flag);
assert_eq!(verdict.structured_findings[0].title, "thin evidence");
assert_eq!(verdict.structured_findings[0].confidence, 60);
assert!(verdict.findings[0].contains("thin evidence"));
}
#[test]
fn verdict_parser_rejects_flag_with_no_findings() {
let output = serde_json::json!({
"verdict": "FLAG",
"summary": "Claim substantiated but flagging.",
"findings": [],
"next_steps": [],
"memory_skill": {
"kind": "none",
"learning_source": "",
"reasoning": "no reusable memory"
}
});
let error = ParsedVerdict::parse(&output.to_string()).unwrap_err();
assert!(matches!(error, ReviewerError::VerdictSchema { .. }));
}
#[test]
fn verdict_parser_accepts_lowercase_flag_via_schema() {
// Reviewer harnesses vary in case; serde rename_all = "UPPERCASE"
// canonicalises, but reviewers may emit lowercase. Confirm the
// existing parser does NOT silently coerce: lowercase should fail
// (the schema is the source of truth for new entries).
let output = serde_json::json!({
"verdict": "flag",
"summary": "Claim substantiated but flagging.",
"findings": [{
"severity": "low",
"title": "x",
"body": "y",
"file": "src/lib.rs",
"line_start": 1,
"line_end": 1,
"confidence": 50,
"recommendation": "z"
}],
"next_steps": [],
"memory_skill": {
"kind": "none",
"learning_source": "",
"reasoning": "none"
}
});
let error = ParsedVerdict::parse(&output.to_string()).unwrap_err();
assert!(matches!(error, ReviewerError::VerdictJson { .. }));
}
#[test]
fn verdict_parser_rejects_missing_memory_skill_classification() {
let output = serde_json::json!({
"verdict": "PASS",
"summary": "The claim is substantiated.",
"findings": [],
"next_steps": []
});
let error = ParsedVerdict::parse(&output.to_string()).unwrap_err();
assert!(matches!(error, ReviewerError::VerdictJson { .. }));
}
#[test]
fn verdict_parser_rejects_unrecoverable_memory_skill_classification() {
let mut output: serde_json::Value =
serde_json::from_str(&reject_json("missing proof")).unwrap();
output["memory_skill"]["kind"] = serde_json::json!("how_to_skill");
let error = ParsedVerdict::parse(&output.to_string()).unwrap_err();
assert!(matches!(error, ReviewerError::VerdictSchema { .. }));
}
#[test]
fn verdict_parser_accepts_normalized_float_confidence() {
let mut output: serde_json::Value =
serde_json::from_str(&reject_json("missing proof")).unwrap();
output["findings"][0]["confidence"] = serde_json::json!(0.95);
let verdict = ParsedVerdict::parse(&output.to_string()).unwrap();
assert_eq!(verdict.structured_findings[0].confidence, 95);
}
#[test]
fn verdict_parser_rejects_legacy_line_protocol() {
let error =
ParsedVerdict::parse("VERDICT: REJECT\nFINDINGS:\n- missing proof\n").unwrap_err();
assert!(matches!(error, ReviewerError::VerdictJson { .. }));
}
#[test]
fn large_diff_materialization_falls_back_to_file_summary() {
let files = "a.rs\nb.rs\nc.rs\n";
let materialized = super::materialize_diff("branch:main", "tiny diff", files);
assert!(materialized.contains("too large to inline safely"));
assert!(materialized.contains("actual_files=3"));
assert!(materialized.contains("a.rs\nb.rs\nc.rs"));
assert!(materialized.contains("inspect the repository directly"));
}
#[test]
fn review_queue_schedules_commits_without_running_models() {
let temp = tempfile::tempdir().unwrap();
let queue = ReviewQueue::new(temp.path());
queue.enqueue("abc123").unwrap();
let pending = queue.pending().unwrap();
assert_eq!(pending.len(), 1);
assert_eq!(pending[0].commit_sha, "abc123");
assert!(!pending[0].run_id.is_empty());
let run = ReviewRunStore::new(temp.path())
.read(&pending[0].run_id)
.unwrap();
assert_eq!(run.commit_sha, "abc123");
assert_eq!(run.status, ReviewRunStatus::Queued);
assert_eq!(run.entire_checkpoint, None);
}
#[test]
fn inflight_enqueue_recovery_replays_under_lock_without_duplicating_queue_row() {
let temp = tempfile::tempdir().unwrap();
let queue = ReviewQueue::new(temp.path());
let run_store = ReviewRunStore::new(temp.path());
let item = QueuedReview {
run_id: "run-inflight".to_owned(),
commit_sha: "abc123".to_owned(),
enqueued_at_unix: 42,
petition_for: None,
..Default::default()
};
write_enqueue_inflight(temp.path(), &item).unwrap();
run_store
.ensure_queued("run-inflight", "abc123", "commit", None)
.unwrap();
let pending = queue.pending().unwrap();
assert_eq!(pending.len(), 1);
assert_eq!(pending[0].run_id, "run-inflight");
assert!(!temp.path().join(ENQUEUE_INFLIGHT_FILE).exists());
let again = queue.pending().unwrap();
assert_eq!(again.len(), 1);
assert_eq!(again[0].run_id, "run-inflight");
}
#[test]
fn remove_item_rewrites_queue_without_reentrant_pending_lock() {
let temp = tempfile::tempdir().unwrap();
let queue = ReviewQueue::new(temp.path());
queue.enqueue("abc123").unwrap();
queue.enqueue_petition("fix456", "orig123").unwrap();
assert_eq!(queue.pending().unwrap().len(), 2);
queue.remove_item("abc123", None).unwrap();
let pending = queue.pending_read_only().unwrap();
assert_eq!(pending.len(), 1);
assert_eq!(pending[0].commit_sha, "fix456");
assert_eq!(pending[0].petition_for.as_deref(), Some("orig123"));
}
#[test]
fn orphan_petition_run_is_requeued_with_preserved_petition_for() {
let temp = tempfile::tempdir().unwrap();
let queue = ReviewQueue::new(temp.path());
let run_store = ReviewRunStore::new(temp.path());
let run = ReviewRun::queued_with_provenance(
"run-petition",
"fix456",
"petition",
None,
Some("orig123".to_owned()),
);
run_store.write(&run).unwrap();
let pending = queue.pending().unwrap();
assert_eq!(pending.len(), 1);
assert_eq!(pending[0].commit_sha, "fix456");
assert_eq!(pending[0].petition_for.as_deref(), Some("orig123"));
assert_eq!(
run_store.read("run-petition").unwrap().status,
ReviewRunStatus::Queued
);
}
#[test]
fn malformed_run_json_does_not_block_pending_for_valid_queue_rows() {
let temp = tempfile::tempdir().unwrap();
let queue = ReviewQueue::new(temp.path());
queue.enqueue("abc123").unwrap();
fs::write(
ReviewRunStore::new(temp.path()).path("broken-run"),
"{not-json",
)
.unwrap();
let pending = queue.pending().unwrap();
assert_eq!(pending.len(), 1);
assert_eq!(pending[0].commit_sha, "abc123");
let (runs, skipped) = ReviewRunStore::new(temp.path()).list_lenient().unwrap();
assert_eq!(skipped, 1);
assert_eq!(runs.len(), 1);
}
#[test]
fn queue_summary_is_read_only_and_does_not_recover_orphans() {
let temp = tempfile::tempdir().unwrap();
let queue = ReviewQueue::new(temp.path());
fs::write(
queue.path(),
r#"{"run_id":"run-1","commit_sha":"abc123","enqueued_at_unix":10}
"#,
)
.unwrap();
let run_store = ReviewRunStore::new(temp.path());
let orphan = ReviewRun::queued("orphan-run", "def456", "commit");
run_store.write(&orphan).unwrap();
let summary = queue.summary().unwrap();
assert_eq!(summary.pending_count, 1);
assert_eq!(queue.pending_read_only().unwrap().len(), 1);
assert!(queue.pending().unwrap().len() >= 2);
}
#[test]
fn drain_once_resumes_strict_second_pass_from_partial_spool_without_rerunning_first() {
use crate::ledger::{LedgerEntry, ReviewerConfig, Verdict};
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let queue = ReviewQueue::new(temp.path());
let sha = "abc123";
let run_store = ReviewRunStore::new(temp.path());
let queued = queue.enqueue(sha).unwrap();
run_store
.mark_running(&queued.run_id, REVIEW_LEDGER_PHASE)
.unwrap();
let first = LedgerEntry::new(
sha,
Verdict::Pass,
"CLAIM: first pass | verified: cargo test | evidence: tests:cargo-test",
vec!["tests:cargo-test".to_owned()],
ReviewerConfig::new("claude", "claude-opus-4-1", false),
vec![],
);
run_store
.persist_pending_review(
&queued.run_id,
PendingReviewSpool {
entries: vec![first],
petition_transition_pending: false,
strict_pass_stdout: Some(pass_json()),
},
)
.unwrap();
let mut strict_selection = selection();
strict_selection.strict = Some(StrictReviewConfig {
arbiter_harness: ReviewerHarness::Claude,
arbiter_model: "claude-opus-4-8".to_owned(),
arbiter_effort: Effort::Xhigh,
});
let report = drain_once(
&queue,
&StaticLoader::new(),
&strict_selection,
"",
&SequenceRunner::new([pass_json()]),
&store,
&TruthMirrorConfig::default(),
)
.unwrap();
assert_eq!(report.reviewed, [sha]);
assert_eq!(report.ledger_entries, 2);
let history = store.read_history().unwrap();
assert_eq!(history.len(), 2);
assert_eq!(history[1].reviewer.model, "claude-opus-4-8");
let completed = run_store.read(&queued.run_id).unwrap();
assert_eq!(completed.status, ReviewRunStatus::Completed);
assert!(completed.pending_review.is_none());
}
#[test]
fn review_queue_summary_counts_pending_and_oldest() {
let temp = tempfile::tempdir().unwrap();
let queue = ReviewQueue::new(temp.path());
std::fs::write(
queue.path(),
"{\"commit_sha\":\"abc123\",\"enqueued_at_unix\":30}\n{\"commit_sha\":\"def456\",\"enqueued_at_unix\":10}\n",
)
.unwrap();
let summary = queue.summary().unwrap();
assert_eq!(summary.pending_count, 2);
assert_eq!(summary.oldest_enqueued_at_unix, Some(10));
assert_eq!(summary.oldest_age_secs_at(42), Some(32));
}
#[test]
fn review_run_serializes_optional_entire_checkpoint() {
let run = ReviewRun::queued_with_provenance(
"run-id",
"abcdef123456",
"commit",
Some(crate::provenance::EntireCheckpointRef {
ref_name: "entire/session-abcdef".to_owned(),
object_sha: "123456".to_owned(),
}),
None,
);
let json = serde_json::to_string(&run).unwrap();
assert!(json.contains("entire_checkpoint"));
assert!(json.contains("entire/session-abcdef"));
}
#[test]
fn review_run_omits_absent_optional_entire_checkpoint() {
let run = ReviewRun::queued("run-id", "abcdef123456", "commit");
let json = serde_json::to_string(&run).unwrap();
assert!(!json.contains("entire_checkpoint"));
}
#[test]
fn legacy_review_run_without_process_identity_fields_still_parses() {
let legacy = r#"{
"id":"legacy-run",
"commit_sha":"abc123",
"target":"commit",
"status":"queued",
"phase":"queued",
"ledger_entries":0,
"error":null,
"created_at_unix":10,
"updated_at_unix":10,
"started_at_unix":null,
"completed_at_unix":null
}"#;
let run: ReviewRun = serde_json::from_str(legacy).unwrap();
assert_eq!(run.id, "legacy-run");
assert_eq!(run.worker_pid, None);
assert_eq!(run.entire_checkpoint, None);
}
#[test]
fn review_run_status_counts_read_only_status_fields() {
let temp = tempfile::tempdir().unwrap();
let store = ReviewRunStore::new(temp.path());
let queued = store.create_queued("abc123", "commit").unwrap();
let mut failed = ReviewRun::queued("failed-run", "def456", "commit");
failed.mark_failed("boom");
store.write(&failed).unwrap();
fs::write(store.path("malformed-run"), "{not-json\n").unwrap();
let counts = store.status_counts().unwrap();
assert_eq!(counts.queued, 1);
assert_eq!(counts.failed, 1);
assert_eq!(counts.running, 0);
assert_eq!(counts.skipped_records, 1);
assert_eq!(queued.status, ReviewRunStatus::Queued);
}
#[test]
fn review_cancel_marks_queued_run_and_removes_queue_item() {
let temp = tempfile::tempdir().unwrap();
let queue = ReviewQueue::new(temp.path());
let queued = queue.enqueue("abc123").unwrap();
run_review_run_command(
crate::cli::ReviewCommand::Cancel {
run_id: queued.run_id.clone(),
force: false,
},
temp.path(),
)
.unwrap();
assert!(queue.pending().unwrap().is_empty());
let run = ReviewRunStore::new(temp.path())
.read(&queued.run_id)
.unwrap();
assert_eq!(run.status, ReviewRunStatus::Cancelled);
}
/// A pid guaranteed to be dead: spawn a trivial process, reap it, return
/// its pid. Copilot (PR #13, watcher_cli.rs ~66 / watcher.rs ~844):
/// spawning the external `true` binary isn't portable (notably absent on
/// Windows). Spawning this test binary itself with `--help` (which the
/// standard libtest harness understands and exits 0 for) is a
/// guaranteed-dead-pid source with no external command dependency.
fn reaped_pid() -> u32 {
let mut child = Command::new(std::env::current_exe().unwrap())
.arg("--help")
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.spawn()
.expect("spawn self with --help");
let pid = child.id();
child.wait().expect("reap self-spawned process");
pid
}
/// Persist a run pinned to `Running` with a specific worker pid, bypassing the
/// normal lifecycle so liveness handling can be exercised deterministically.
fn write_running_run(store: &ReviewRunStore, worker_pid: Option<u32>) -> ReviewRun {
let mut run = store.create_queued("abc123", "commit").unwrap();
run.status = ReviewRunStatus::Running;
run.phase = "reviewing".to_owned();
run.worker_pid = worker_pid;
store.write(&run).unwrap();
run
}
#[test]
fn pid_liveness_probe_tracks_real_processes() {
assert!(pid_is_alive(std::process::id()));
assert!(!pid_is_alive(reaped_pid()));
assert_eq!(
crate::watcher::probe_pid_liveness(std::process::id()),
crate::watcher::PidLiveness::Alive
);
assert_eq!(
crate::watcher::probe_pid_liveness(reaped_pid()),
crate::watcher::PidLiveness::Dead
);
}
#[test]
fn reconcile_liveness_only_reaps_dead_running_runs() {
use crate::watcher::WorkerLiveness;
let mut queued = ReviewRun::queued("id", "abc123", "commit");
// A queued run is never touched, even if the probe reports dead.
assert_eq!(
queued.reconcile_liveness(|_| WorkerLiveness::Dead),
super::ReconcileOutcome::Unchanged
);
assert_eq!(queued.status, ReviewRunStatus::Queued);
queued.mark_running("reviewing");
// Running + alive stays running.
assert_eq!(
queued.reconcile_liveness(|_| WorkerLiveness::Alive),
super::ReconcileOutcome::Unchanged
);
assert_eq!(queued.status, ReviewRunStatus::Running);
// Running + dead flips to failed with a stale-worker reason.
assert_eq!(
queued.reconcile_liveness(|_| WorkerLiveness::Dead),
super::ReconcileOutcome::Reaped
);
assert_eq!(queued.status, ReviewRunStatus::Failed);
assert!(queued.error.as_deref().unwrap().contains("stale run"));
assert!(queued.worker_pid.is_none());
// A legacy running run with no recorded pid is left alone.
let mut legacy = ReviewRun::queued("id2", "def456", "commit");
legacy.status = ReviewRunStatus::Running;
legacy.worker_pid = None;
assert_eq!(
legacy.reconcile_liveness(|_| WorkerLiveness::Dead),
super::ReconcileOutcome::Unchanged
);
assert_eq!(legacy.status, ReviewRunStatus::Running);
}
#[test]
fn reconcile_liveness_probe_unknown_leaves_row_untouched() {
// 0.9.2 treated ANY probe failure as proof of death and failed live
// runs. An inconclusive probe must leave the row alone.
let mut run = ReviewRun::queued("id", "abc123", "commit");
run.status = ReviewRunStatus::Running;
run.worker_pid = Some(424242);
assert_eq!(
run.reconcile_liveness(|_| crate::watcher::WorkerLiveness::Unknown),
super::ReconcileOutcome::ProbeUnknown
);
assert_eq!(run.status, ReviewRunStatus::Running);
assert!(run.error.is_none());
}
#[cfg(unix)]
#[test]
fn reconcile_liveness_detects_pid_reuse_via_start_token() {
// The recorded worker identity is (pid, start token): a live pid
// under a DIFFERENT token means the worker died and the OS recycled
// its pid — proven dead, reap it. A matching token means genuinely
// alive.
let mut run = ReviewRun::queued("id", "abc123", "commit");
run.status = ReviewRunStatus::Running;
run.worker_pid = Some(std::process::id());
let identity = crate::watcher::ProcessIdentity::current();
run.worker_start_token = Some(identity.start_token.clone());
assert_eq!(
run.reconcile_liveness(crate::watcher::probe_worker_identity),
super::ReconcileOutcome::Unchanged
);
assert_eq!(run.status, ReviewRunStatus::Running);
run.worker_start_token = Some("a different process's start marker".to_owned());
assert_eq!(
run.reconcile_liveness(crate::watcher::probe_worker_identity),
super::ReconcileOutcome::Reaped
);
assert_eq!(run.status, ReviewRunStatus::Failed);
}
#[test]
fn mark_running_records_worker_start_identity() {
let temp = tempfile::tempdir().unwrap();
let store = ReviewRunStore::new(temp.path());
let run = store.create_queued("abc123", "commit").unwrap();
let running = store.mark_running(&run.id, "reviewing").unwrap();
assert_eq!(running.worker_pid, Some(std::process::id()));
#[cfg(unix)]
assert!(
running.worker_start_token.is_some(),
"unix platforms report a start marker via ps"
);
// Terminal transitions clear the identity again.
let completed = store.mark_completed(&run.id, 1).unwrap();
assert_eq!(completed.worker_pid, None);
assert_eq!(completed.worker_start_token, None);
}
#[test]
fn run_store_lock_excludes_concurrent_mutations() {
// The mutation lock is real mutual exclusion: a second acquirer waits
// until the first guard drops.
let temp = tempfile::tempdir().unwrap();
let store = ReviewRunStore::new(temp.path());
let guard = store.lock().unwrap();
let (sender, receiver) = std::sync::mpsc::channel();
let contender = std::thread::spawn({
let store = store.clone();
move || {
let _guard = store.lock().unwrap();
sender.send(()).unwrap();
}
});
assert!(
receiver.recv_timeout(Duration::from_millis(200)).is_err(),
"the contender acquired the lock while it was still held"
);
drop(guard);
receiver
.recv_timeout(Duration::from_secs(5))
.expect("the contender did not acquire the released lock");
contender.join().unwrap();
}
#[test]
fn reconcile_never_clobbers_a_concurrently_completed_run() {
// 0.9.2 race: the reconciler snapshot-reads a Running row, the worker
// completes it, then the reconciler writes its stale Failed snapshot
// over the Completed row. Under the run-store lock the read-probe-
// write is atomic in both orders, so the worker's truth always wins.
for _ in 0..25 {
let temp = tempfile::tempdir().unwrap();
let store = ReviewRunStore::new(temp.path());
let run = write_running_run(&store, Some(reaped_pid()));
let completer = std::thread::spawn({
let store = store.clone();
let run_id = run.id.clone();
move || store.mark_completed(&run_id, 1)
});
let reconciler = std::thread::spawn({
let store = store.clone();
move || store.reconcile_stale_runs()
});
completer.join().unwrap().unwrap();
reconciler.join().unwrap().unwrap();
// If the reconciler reaped first the completion lands on top
// (Failed → Completed); if the completion landed first the
// reconciler skips the non-Running row. Either way: Completed.
let final_run = store.read(&run.id).unwrap();
assert_eq!(
final_run.status,
ReviewRunStatus::Completed,
"a completed run must never be clobbered back to failed"
);
}
}
#[test]
fn review_status_reaps_running_run_with_dead_worker_and_persists() {
let temp = tempfile::tempdir().unwrap();
let store = ReviewRunStore::new(temp.path());
let run = write_running_run(&store, Some(reaped_pid()));
let reconciled = store.read_reconciled(&run.id).unwrap();
assert_eq!(reconciled.status, ReviewRunStatus::Failed);
assert!(reconciled.error.as_deref().unwrap().contains("stale run"));
// The correction is persisted, not just displayed.
assert_eq!(store.read(&run.id).unwrap().status, ReviewRunStatus::Failed);
// list_reconciled agrees.
let listed = store.list_reconciled().unwrap();
assert_eq!(listed.len(), 1);
assert_eq!(listed[0].status, ReviewRunStatus::Failed);
}
#[test]
fn review_status_leaves_running_run_with_live_worker() {
let temp = tempfile::tempdir().unwrap();
let store = ReviewRunStore::new(temp.path());
let run = write_running_run(&store, Some(std::process::id()));
let reconciled = store.read_reconciled(&run.id).unwrap();
assert_eq!(reconciled.status, ReviewRunStatus::Running);
}
#[test]
fn cancel_reaps_running_run_with_dead_worker_without_force() {
let temp = tempfile::tempdir().unwrap();
let store = ReviewRunStore::new(temp.path());
let run = write_running_run(&store, Some(reaped_pid()));
let cancelled = store.cancel(&run.id, false).unwrap();
assert_eq!(cancelled.status, ReviewRunStatus::Failed);
assert!(cancelled.error.as_deref().unwrap().contains("stale run"));
}
#[test]
fn cancel_refuses_live_running_run_without_force() {
let temp = tempfile::tempdir().unwrap();
let store = ReviewRunStore::new(temp.path());
let run = write_running_run(&store, Some(std::process::id()));
let error = store.cancel(&run.id, false).unwrap_err();
assert!(matches!(error, ReviewerError::ReviewRunStillAlive { .. }));
// The run is untouched.
assert_eq!(
store.read(&run.id).unwrap().status,
ReviewRunStatus::Running
);
}
#[test]
fn cancel_force_kills_live_worker_and_cancels() {
let temp = tempfile::tempdir().unwrap();
let store = ReviewRunStore::new(temp.path());
let mut child = Command::new("sleep")
.arg("30")
.spawn()
.expect("spawn sleep");
let pid = child.id();
let run = write_running_run(&store, Some(pid));
let cancelled = store.cancel(&run.id, true).unwrap();
assert_eq!(cancelled.status, ReviewRunStatus::Cancelled);
// Reap the killed child, then confirm it is truly gone.
let _ = child.wait();
assert!(!pid_is_alive(pid));
}
#[test]
fn cancel_legacy_running_run_requires_force_then_reaps() {
let temp = tempfile::tempdir().unwrap();
let store = ReviewRunStore::new(temp.path());
let run = write_running_run(&store, None);
let error = store.cancel(&run.id, false).unwrap_err();
assert!(matches!(
error,
ReviewerError::ReviewRunLivenessUnknown { .. }
));
let reaped = store.cancel(&run.id, true).unwrap();
assert_eq!(reaped.status, ReviewRunStatus::Failed);
}
#[test]
fn cancel_refuses_already_terminal_run() {
let temp = tempfile::tempdir().unwrap();
let store = ReviewRunStore::new(temp.path());
let run = store.create_queued("abc123", "commit").unwrap();
store.mark_completed(&run.id, 0).unwrap();
let error = store.cancel(&run.id, true).unwrap_err();
assert!(matches!(
error,
ReviewerError::CannotCancelReview {
status: ReviewRunStatus::Completed,
..
}
));
}
#[test]
fn execute_review_records_reject_verdict() {
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let job = review_job(false);
let runner = SequenceRunner::new([reject_json("unsupported")]);
let execution = execute_review_job(job, &runner, &store, None).unwrap();
assert_eq!(execution.entries.len(), 1);
assert_eq!(execution.entries[0].verdict, Verdict::Reject);
assert_eq!(
execution.entries[0].structured_findings[0].title,
"unsupported"
);
assert!(
execution.entries[0]
.raw_reviewer_output
.contains("\"REJECT\"")
);
assert_eq!(store.unresolved_rejections().unwrap().len(), 1);
}
#[test]
fn commit_petition_resolve_recovers_unqueued_enqueue_without_second_attempt() {
use crate::ledger::{
Disposition, ResolutionKind, TransitionProvenance, TransitionProvenanceKind,
};
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let queue = ReviewQueue::new(temp.path());
let rejected = crate::ledger::LedgerEntry::new(
"orig123",
crate::ledger::Verdict::Reject,
"CLAIM: original | verified: cargo test | evidence: tests:cargo-test",
vec!["tests:cargo-test".to_owned()],
crate::ledger::ReviewerConfig::new("claude", "claude-opus-4-1", false),
vec!["thin".to_owned()],
);
store.append_entry(&rejected).unwrap();
store
.append_petition_transition_with_provenance(
"orig123",
Disposition::Open,
ResolutionKind::Resolved,
"petition review enqueued: fix=fix456, attempts=1/2",
1,
Some(TransitionProvenance {
kind: TransitionProvenanceKind::PetitionEnqueued,
fix_sha: "fix456".to_owned(),
original_sha: "orig123".to_owned(),
attempts: 1,
}),
)
.unwrap();
let pending = queue.pending().unwrap();
assert_eq!(pending.len(), 1);
assert_eq!(pending[0].commit_sha, "fix456");
assert_eq!(pending[0].petition_for.as_deref(), Some("orig123"));
assert_eq!(store.petition_attempts_for("orig123").unwrap(), 1);
}
#[test]
fn commit_petition_resolve_refuses_in_flight_without_spending_attempt() {
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let queue = ReviewQueue::new(temp.path());
let rejected = crate::ledger::LedgerEntry::new(
"orig123",
crate::ledger::Verdict::Reject,
"CLAIM: original | verified: cargo test | evidence: tests:cargo-test",
vec!["tests:cargo-test".to_owned()],
crate::ledger::ReviewerConfig::new("claude", "claude-opus-4-1", false),
vec!["thin".to_owned()],
);
store.append_entry(&rejected).unwrap();
queue.enqueue_petition("fix456", "orig123").unwrap();
let error =
super::commit_petition_resolve(temp.path(), &store, "orig123", "fix456").unwrap_err();
assert!(matches!(error, ReviewerError::PetitionInFlight { .. }));
assert_eq!(store.petition_attempts_for("orig123").unwrap(), 0);
}
#[test]
fn drain_once_does_not_complete_new_run_from_historical_sha_verdict_alone() {
use crate::ledger::{LedgerEntry, ReviewerConfig, Verdict};
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let queue = ReviewQueue::new(temp.path());
let sha = "abc1234567890123456789012345678901234567890";
let run_store = ReviewRunStore::new(temp.path());
let queued = queue.enqueue(sha).unwrap();
run_store
.mark_running(&queued.run_id, REVIEW_LEDGER_PHASE)
.unwrap();
store
.append_entry(&LedgerEntry::new(
sha,
Verdict::Pass,
"CLAIM: historical | verified: cargo test | evidence: tests:cargo-test",
vec!["tests:cargo-test".to_owned()],
ReviewerConfig::new("claude", "claude-opus-4-1", false),
vec![],
))
.unwrap();
fs::write(
queue.path(),
format!(
"{}
",
serde_json::to_string(&QueuedReview {
run_id: queued.run_id.clone(),
commit_sha: sha.to_owned(),
enqueued_at_unix: 1,
petition_for: None,
..Default::default()
})
.unwrap()
),
)
.unwrap();
let before = store.read_history().unwrap().len();
let report = drain_once(
&queue,
&StaticLoader::new(),
&selection(),
"",
&ConstRunner::new(pass_json()),
&store,
&TruthMirrorConfig::default(),
)
.unwrap();
assert_eq!(report.reviewed, [sha]);
assert_eq!(store.read_history().unwrap().len(), before + 1);
assert!(
!run_ledger_phase_applied(
&run_store.read(&queued.run_id).unwrap(),
REVIEW_LEDGER_PHASE
) || run_store.read(&queued.run_id).unwrap().status == ReviewRunStatus::Completed
);
}
#[test]
fn direct_strict_review_resumes_from_pending_spool_without_rerunning_first_pass() {
use crate::ledger::{LedgerEntry, ReviewerConfig, Verdict};
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let run_store = ReviewRunStore::new(temp.path());
let run = run_store.create_queued("abc123", "commit").unwrap();
run_store
.mark_running(&run.id, REVIEW_LEDGER_PHASE)
.unwrap();
let first = LedgerEntry::new(
"abc123",
Verdict::Pass,
"CLAIM: first pass | verified: cargo test | evidence: tests:cargo-test",
vec!["tests:cargo-test".to_owned()],
ReviewerConfig::new("claude", "claude-opus-4-1", false),
vec![],
);
run_store
.persist_pending_review(
&run.id,
PendingReviewSpool {
entries: vec![first],
petition_transition_pending: false,
strict_pass_stdout: Some(pass_json()),
},
)
.unwrap();
let execution = super::complete_review_execution(
&review_job(true),
&SequenceRunner::new([pass_json()]),
&run_store,
&run.id,
run_store.read(&run.id).unwrap().pending_review,
None,
)
.unwrap();
assert_eq!(execution.entries.len(), 2);
assert_eq!(store.read_history().unwrap().len(), 0);
}
#[test]
fn drain_attaches_legacy_empty_run_id_before_processing_without_orphan_runs() {
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let queue = ReviewQueue::new(temp.path());
let sha = "abc1234567890123456789012345678901234567890";
fs::write(
queue.path(),
format!(
"{}
",
serde_json::to_string(&QueuedReview {
run_id: String::new(),
commit_sha: sha.to_owned(),
enqueued_at_unix: 1,
petition_for: None,
..Default::default()
})
.unwrap()
),
)
.unwrap();
fs::create_dir_all(temp.path().join("runs")).unwrap();
let orphan = ReviewRun::queued("run-legacy", sha, "commit");
fs::write(
temp.path().join("runs/run-legacy.json"),
serde_json::to_string(&orphan).unwrap(),
)
.unwrap();
let before_runs = ReviewRunStore::new(temp.path())
.list_lenient()
.unwrap()
.0
.len();
let report = drain_once(
&queue,
&StaticLoader::new(),
&selection(),
"",
&ConstRunner::new(pass_json()),
&store,
&TruthMirrorConfig::default(),
)
.unwrap();
assert_eq!(report.reviewed, [sha]);
let pending = queue.pending_read_only().unwrap();
assert!(pending.is_empty());
let attached = queue.pending_read_only().unwrap();
assert!(attached.is_empty());
let (runs, _) = ReviewRunStore::new(temp.path()).list_lenient().unwrap();
assert_eq!(runs.len(), before_runs);
assert!(runs.iter().any(|run| run.id == "run-legacy"));
}
#[test]
fn strict_two_pass_records_both_clean_passes() {
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let job = review_job(true);
let runner = SequenceRunner::new([pass_json(), pass_json()]);
let execution = execute_review_job(job, &runner, &store, None).unwrap();
assert_eq!(execution.entries.len(), 2);
assert_eq!(store.read_history().unwrap().len(), 2);
assert_eq!(execution.entries[0].reviewer.model, "gpt-5.5");
assert_eq!(execution.entries[1].reviewer.model, "claude-opus-4-8");
}
#[test]
fn strict_arbiter_model_must_be_third_model() {
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let mut job = review_job(true);
job.strict.as_mut().unwrap().arbiter_model = "gpt-5.5".to_owned();
let runner = SequenceRunner::new([pass_json()]);
let error = execute_review_job(job, &runner, &store, None).unwrap_err();
assert!(matches!(
error,
ReviewerError::StrictArbiterModelNotDistinct
));
}
#[test]
fn strict_goal_policy_stops_at_configured_lie_or_fuckup_count() {
let policy = StrictGoalPolicy {
stop_after_lies: 2,
stop_after_fuckups: 3,
};
assert_eq!(
policy.decide(StrictGoalCounters {
lies_exposed: 1,
fuckups_registered: 2
}),
StrictGoalDecision::Continue
);
assert_eq!(
policy.decide(StrictGoalCounters {
lies_exposed: 2,
fuckups_registered: 0
}),
StrictGoalDecision::Stop {
reason: StrictGoalStopReason::LiesExposed
}
);
assert_eq!(
policy.decide(StrictGoalCounters {
lies_exposed: 0,
fuckups_registered: 3
}),
StrictGoalDecision::Stop {
reason: StrictGoalStopReason::FuckupsRegistered
}
);
}
#[test]
fn drain_once_reviews_each_commit_once_and_clears_queue() {
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let queue = ReviewQueue::new(temp.path());
queue.enqueue("abc123").unwrap();
queue.enqueue("abc123").unwrap(); // duplicate SHA reviewed only once
queue.enqueue("def456").unwrap();
let loader = StaticLoader::new();
let runner = SequenceRunner::new([reject_json("unsupported"), pass_json()]);
let selection = selection();
let report = drain_once(
&queue,
&loader,
&selection,
"",
&runner,
&store,
&TruthMirrorConfig::default(),
)
.unwrap();
assert_eq!(report.reviewed, ["abc123", "def456"]);
assert_eq!(report.ledger_entries, 2);
assert!(queue.pending().unwrap().is_empty());
assert_eq!(store.read_history().unwrap().len(), 2);
assert_eq!(store.unresolved_rejections().unwrap().len(), 1);
let runs = ReviewRunStore::new(temp.path()).list().unwrap();
assert_eq!(runs.len(), 2);
assert_eq!(
runs.iter()
.filter(|run| run.status == ReviewRunStatus::Completed)
.count(),
2
);
assert_eq!(
runs.iter()
.filter(|run| run.status == ReviewRunStatus::Cancelled)
.count(),
0
);
}
#[test]
fn drain_once_skips_a_poisoned_claim_and_reviews_surrounding_items() {
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let queue = ReviewQueue::new(temp.path());
queue.enqueue("good-one").unwrap();
queue.enqueue("poisoned").unwrap();
queue.enqueue("good-two").unwrap();
let loader = PoisonedLoader;
let runner = ConstRunner::new(pass_json());
let report = drain_once(
&queue,
&loader,
&selection(),
"",
&runner,
&store,
&TruthMirrorConfig::default(),
)
.unwrap();
assert_eq!(report.reviewed, ["good-one", "good-two"]);
assert_eq!(report.failed, ["poisoned"]);
assert!(queue.pending().unwrap().is_empty());
assert_eq!(store.read_history().unwrap().len(), 2);
let poisoned_run = ReviewRunStore::new(temp.path())
.list()
.unwrap()
.into_iter()
.find(|run| run.commit_sha == "poisoned")
.unwrap();
assert_eq!(poisoned_run.status, ReviewRunStatus::Failed);
assert!(
poisoned_run
.error
.as_deref()
.is_some_and(|error| error.contains("missing evidence pointer"))
);
}
#[test]
fn drain_once_skips_a_verdict_schema_failure_and_reviews_surrounding_items() {
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let queue = ReviewQueue::new(temp.path());
queue.enqueue("good-one").unwrap();
queue.enqueue("bad-verdict").unwrap();
queue.enqueue("good-two").unwrap();
let invalid_verdict = serde_json::json!({
"verdict": "PASS",
"summary": "",
"findings": [],
"next_steps": [],
"memory_skill": {
"kind": "none",
"learning_source": "",
"reasoning": "No reusable learning was found."
}
})
.to_string();
let runner = SequenceRunner::new([pass_json(), invalid_verdict, pass_json()]);
let report = drain_once(
&queue,
&StaticLoader::new(),
&selection(),
"",
&runner,
&store,
&TruthMirrorConfig::default(),
)
.unwrap();
assert_eq!(report.reviewed, ["good-one", "good-two"]);
assert_eq!(report.failed, ["bad-verdict"]);
assert!(queue.pending().unwrap().is_empty());
assert_eq!(store.read_history().unwrap().len(), 2);
let failed_run = ReviewRunStore::new(temp.path())
.list()
.unwrap()
.into_iter()
.find(|run| run.commit_sha == "bad-verdict")
.unwrap();
assert_eq!(failed_run.status, ReviewRunStatus::Failed);
assert!(
failed_run
.error
.as_deref()
.is_some_and(|error| error.contains("summary must not be empty"))
);
}
#[test]
fn pass_with_disallowed_skill_proposal_is_salvaged_without_a_proposal() {
let output = serde_json::json!({
"verdict": "PASS",
"summary": "The claim is substantiated by the diff and evidence.",
"findings": [],
"next_steps": [],
"memory_skill": {
"kind": "anti_pattern_skill",
"learning_source": "avoid this unrelated pattern",
"reasoning": "The reviewer attached the wrong proposal kind."
}
})
.to_string();
let parsed = ParsedVerdict::parse(&output).unwrap();
assert_eq!(parsed.verdict, Verdict::Pass);
assert_eq!(
parsed.memory_skill_classification.kind,
MemorySkillClassificationKind::None
);
assert!(
parsed
.memory_skill_classification
.learning_source
.is_empty()
);
}
#[test]
fn drain_once_completes_when_memory_skill_extraction_fails() {
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let queue = ReviewQueue::new(temp.path());
let sha = "a".repeat(40);
queue.enqueue(&sha).unwrap();
let loader = StaticLoader::new();
let runner = ConstRunner::new(pass_json());
let mut config = TruthMirrorConfig::default();
config.skills.enabled = true;
config.memory_skill.enabled = true;
config.memory_skill.signals.min_occurrences = 1;
config.memory_skill.scan.blocked_patterns = vec!["Capture a verified procedure".to_owned()];
let report =
drain_once(&queue, &loader, &selection(), "", &runner, &store, &config).unwrap();
assert_eq!(report.reviewed, [sha]);
assert_eq!(report.ledger_entries, 1);
assert!(queue.pending().unwrap().is_empty());
assert_eq!(store.read_history().unwrap().len(), 1);
let candidate_dir =
crate::memory_skill::MemorySkillStore::new(temp.path(), &config.memory_skill)
.unwrap()
.candidate_dir()
.to_path_buf();
assert!(!candidate_dir.exists());
}
#[test]
fn drain_once_is_a_noop_on_empty_queue() {
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let queue = ReviewQueue::new(temp.path());
let loader = StaticLoader::new();
let runner = ConstRunner::new(pass_json());
let report = drain_once(
&queue,
&loader,
&selection(),
"",
&runner,
&store,
&TruthMirrorConfig::default(),
)
.unwrap();
assert!(report.reviewed.is_empty());
assert_eq!(report.ledger_entries, 0);
assert_eq!(store.read_history().unwrap().len(), 0);
}
#[test]
fn until_empty_keeps_watching_while_queue_has_items() {
// A non-empty queue is reported as `None` elapsed → keep watching.
assert_eq!(
until_empty_decision(None, 60),
UntilEmptyDecision::KeepWatching
);
}
#[test]
fn until_empty_keeps_watching_inside_grace_window() {
// Empty for less than the grace window → keep watching for late arrivals.
assert_eq!(
until_empty_decision(Some(0), 60),
UntilEmptyDecision::KeepWatching
);
assert_eq!(
until_empty_decision(Some(59), 60),
UntilEmptyDecision::KeepWatching
);
}
#[test]
fn until_empty_exits_once_grace_window_elapses() {
// Empty through a full grace window (>= grace) → exit.
assert_eq!(until_empty_decision(Some(60), 60), UntilEmptyDecision::Exit);
assert_eq!(
until_empty_decision(Some(120), 60),
UntilEmptyDecision::Exit
);
}
#[test]
fn until_empty_with_zero_grace_exits_immediately_when_empty() {
assert_eq!(until_empty_decision(Some(0), 0), UntilEmptyDecision::Exit);
// Still keeps watching while items remain, even with zero grace.
assert_eq!(
until_empty_decision(None, 0),
UntilEmptyDecision::KeepWatching
);
}
#[test]
fn strict_goal_loop_stops_at_configured_lie_count() {
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let policy = StrictGoalPolicy {
stop_after_lies: 1,
stop_after_fuckups: 0,
};
let runner = SequenceRunner::new([reject_json("lie")]);
let outcome = run_strict_goal_loop(
"abc123",
&claim(),
"diff",
"",
&selection(),
policy,
5,
&runner,
&store,
None,
)
.unwrap();
assert_eq!(outcome.passes, 1);
assert_eq!(outcome.counters.lies_exposed, 1);
assert_eq!(outcome.stop_reason, Some(StrictGoalStopReason::LiesExposed));
assert_eq!(store.read_history().unwrap().len(), 1);
}
#[test]
fn strict_goal_loop_terminates_at_max_passes_for_honest_agent() {
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let policy = StrictGoalPolicy {
stop_after_lies: 2,
stop_after_fuckups: 5,
};
let runner = ConstRunner::new(pass_json());
let outcome = run_strict_goal_loop(
"abc123",
&claim(),
"diff",
"",
&selection(),
policy,
3,
&runner,
&store,
None,
)
.unwrap();
assert_eq!(outcome.passes, 3);
assert_eq!(outcome.counters.lies_exposed, 0);
assert_eq!(outcome.stop_reason, None);
assert_eq!(store.read_history().unwrap().len(), 3);
}
#[test]
fn strict_goal_loop_stops_when_fuckups_accumulate() {
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let policy = StrictGoalPolicy {
stop_after_lies: 0,
stop_after_fuckups: 2,
};
// Each structured finding registers one fuckup; two passes hit N=2.
let runner = ConstRunner::new(reject_json("nit"));
let outcome = run_strict_goal_loop(
"abc123",
&claim(),
"diff",
"",
&selection(),
policy,
10,
&runner,
&store,
None,
)
.unwrap();
assert_eq!(outcome.passes, 2);
assert_eq!(outcome.counters.lies_exposed, 2);
assert_eq!(outcome.counters.fuckups_registered, 2);
assert_eq!(
outcome.stop_reason,
Some(StrictGoalStopReason::FuckupsRegistered)
);
}
proptest! {
#[test]
fn strict_goal_loop_never_exceeds_max_passes(max in 1u32..6) {
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
// Both thresholds disabled: only the ceiling can stop the loop.
let policy = StrictGoalPolicy { stop_after_lies: 0, stop_after_fuckups: 0 };
let runner = ConstRunner::new(pass_json());
let outcome = run_strict_goal_loop(
"abc123", &claim(), "diff", "", &selection(), policy, max, &runner, &store, None,
)
.unwrap();
prop_assert!(outcome.passes <= max);
prop_assert_eq!(outcome.passes, max);
prop_assert!(outcome.stop_reason.is_none());
}
}
proptest! {
#[test]
fn model_opposition_is_enforced_for_arbitrary_models(
watched in "[A-Za-z0-9._/-]{1,32}",
reviewer in "[A-Za-z0-9._/-]{1,32}",
) {
let request = ReviewRequest::new(
Agent::Codex,
watched.clone(),
ReviewerHarness::Codex,
reviewer.clone(),
false,
"review this",
);
let result = ReviewPlan::build(request);
// Use the same normalisation that ReviewPlan::build uses so that
// provider-prefixed models (e.g. "openai-codex/gpt-5.5" vs "gpt-5.5")
// are treated as the same model after the B2 fix.
if normalized_model(watched.trim()) == normalized_model(reviewer.trim()) {
let blocked = matches!(result, Err(ReviewerError::SameModelWithoutWaiver { .. }));
prop_assert!(blocked);
} else {
prop_assert!(result.is_ok());
}
}
}
fn claim() -> Claim {
Claim::new(
"add review",
"cargo test",
vec![EvidenceRef::parse("tests:cargo-test").unwrap()],
)
.unwrap()
}
#[test]
fn batched_planner_keeps_queue_order_and_assigns_each_batch_one_job_slot() {
let pending = vec![
QueuedReview {
run_id: "normal-1".to_owned(),
commit_sha: "normal-1".to_owned(),
..Default::default()
},
QueuedReview {
run_id: "petition-b".to_owned(),
commit_sha: "fix-sha".to_owned(),
petition_for: Some("rejection-b".to_owned()),
..Default::default()
},
QueuedReview {
run_id: "petition-a".to_owned(),
commit_sha: "fix-sha".to_owned(),
petition_for: Some("rejection-a".to_owned()),
..Default::default()
},
QueuedReview {
run_id: "normal-2".to_owned(),
commit_sha: "normal-2".to_owned(),
..Default::default()
},
];
let planned = plan_batched_drain(&pending, Path::new("/repo"), &selection(), "context", 32);
assert!(matches!(
&planned[..],
[
PlannedDrainJob::Single(first),
PlannedDrainJob::PetitionBatch(batch),
PlannedDrainJob::Single(last),
] if first.commit_sha == "normal-1"
&& last.commit_sha == "normal-2"
&& batch.iter().map(|item| item.petition_for.as_deref()).collect::<Vec<_>>()
== vec![Some("rejection-a"), Some("rejection-b")]
));
assert_eq!(
planned.len(),
3,
"one petition batch consumes one worker slot"
);
}
fn selection() -> ReviewSelection {
ReviewSelection {
watched_agent: Agent::Codex,
watched_model: "gpt-5.4".to_owned(),
reviewer_harness: ReviewerHarness::Codex,
reviewer_model: "gpt-5.5".to_owned(),
reviewer_effort: Effort::Xhigh,
allow_same_model: false,
strict: None,
}
}
/// Distinct commit objects for drain/worktree tests without needing deep
/// history (`HEAD^` fails on shallow CI checkouts with `fetch-depth: 1`).
/// Builds a chain via `commit-tree` from `HEAD`'s tree; does not move HEAD.
fn synthetic_commit_shas(count: usize) -> Vec<String> {
assert!(count >= 1);
let tree = git_output(["rev-parse", "HEAD^{tree}"])
.unwrap()
.trim()
.to_owned();
let mut parent = git_output(["rev-parse", "HEAD"]).unwrap().trim().to_owned();
let mut shas = vec![parent.clone()];
for index in 1..count {
let message = format!("truth-mirror-test-fixture-{index}");
let output = Command::new("git")
.env("GIT_AUTHOR_NAME", "truth-mirror-test")
.env("GIT_AUTHOR_EMAIL", "truth-mirror-test@example.com")
.env("GIT_COMMITTER_NAME", "truth-mirror-test")
.env("GIT_COMMITTER_EMAIL", "truth-mirror-test@example.com")
.args(["commit-tree", &tree, "-p", &parent, "-m", &message])
.output()
.expect("spawn git commit-tree");
assert!(
output.status.success(),
"git commit-tree failed: {}",
String::from_utf8_lossy(&output.stderr)
);
parent = String::from_utf8_lossy(&output.stdout).trim().to_owned();
shas.push(parent.clone());
}
shas
}
#[test]
fn parallel_drain_caps_workers_uses_isolated_cwds_and_preserves_queue_order() {
let temp = tempfile::tempdir().unwrap();
let queue = ReviewQueue::new(temp.path());
let store = LedgerStore::new(temp.path());
let shas = synthetic_commit_shas(3);
let first_sha = shas[0].clone();
let second_sha = shas[1].clone();
let third_sha = shas[2].clone();
queue.enqueue(&first_sha).unwrap();
queue.enqueue(&second_sha).unwrap();
queue.enqueue(&third_sha).unwrap();
let mut config = TruthMirrorConfig::default();
config.reviewer.max_concurrent_runs = 2;
let runner = ConcurrentRunner::with_overlap(2);
let report = drain_once_parallel(
&queue,
&StaticLoader::new(),
&selection(),
"",
&runner,
&store,
&config,
)
.unwrap();
assert_eq!(
report.reviewed,
vec![first_sha, second_sha, third_sha],
"report order follows the snapshot queue order, not completion order"
);
assert_eq!(
runner.max_active.load(Ordering::SeqCst),
2,
"worker cap must bind concurrent reviewer processes"
);
let dirs = runner.dirs.lock().unwrap().clone();
assert_eq!(dirs.len(), 3);
let unique = dirs.iter().collect::<std::collections::BTreeSet<_>>();
assert_eq!(
unique.len(),
3,
"each job must use an isolated worktree cwd"
);
assert!(
dirs.iter()
.all(|dir| { dir.starts_with(temp.path().join("worktrees")) && !dir.exists() })
);
assert!(queue.pending().unwrap().is_empty());
assert!(
fs::read_dir(temp.path().join("worktrees"))
.unwrap()
.next()
.is_none()
);
}
#[test]
fn serial_default_runs_one_job_at_a_time_without_worktree_cwd() {
let temp = tempfile::tempdir().unwrap();
let queue = ReviewQueue::new(temp.path());
let store = LedgerStore::new(temp.path());
let shas = synthetic_commit_shas(2);
let first_sha = shas[0].clone();
let second_sha = shas[1].clone();
queue.enqueue(&first_sha).unwrap();
queue.enqueue(&second_sha).unwrap();
let config = TruthMirrorConfig::default();
assert_eq!(config.reviewer.max_concurrent_runs, 1);
let runner = ConcurrentRunner::new();
let report = drain_once(
&queue,
&StaticLoader::new(),
&selection(),
"",
&runner,
&store,
&config,
)
.unwrap();
assert_eq!(report.reviewed, vec![first_sha, second_sha]);
assert_eq!(runner.max_active.load(Ordering::SeqCst), 1);
assert!(
runner.dirs.lock().unwrap().is_empty(),
"serial default must keep the shared checkout (no per-job worktree cwd)"
);
assert!(!temp.path().join("worktrees").exists());
assert!(queue.pending().unwrap().is_empty());
}
#[test]
fn parallel_once_snapshot_leaves_late_enqueues_for_later_invocation() {
let temp = tempfile::tempdir().unwrap();
let queue = ReviewQueue::new(temp.path());
let store = LedgerStore::new(temp.path());
let shas = synthetic_commit_shas(3);
let first_sha = shas[0].clone();
let second_sha = shas[1].clone();
let late_sha = shas[2].clone();
queue.enqueue(&first_sha).unwrap();
queue.enqueue(&second_sha).unwrap();
let mut config = TruthMirrorConfig::default();
config.reviewer.max_concurrent_runs = 2;
let runner = SnapshotRunner::new(queue.clone(), late_sha.clone());
let report = drain_once_parallel(
&queue,
&StaticLoader::new(),
&selection(),
"",
&runner,
&store,
&config,
)
.unwrap();
assert_eq!(report.reviewed, vec![first_sha, second_sha]);
let pending = queue.pending().unwrap();
assert_eq!(pending.len(), 1);
assert_eq!(pending[0].commit_sha, late_sha);
}
#[test]
fn durable_claim_precedes_spawn_and_stale_claim_recovers_without_losing_queue_row() {
let temp = tempfile::tempdir().unwrap();
let queue = ReviewQueue::new(temp.path());
let run_store = ReviewRunStore::new(temp.path());
let sha = "abc123deadbeef".to_owned();
let item = queue.enqueue(&sha).unwrap();
let job = PlannedDrainJob::Single(item);
let claimed = claim_planned_job(&queue, &run_store, job).unwrap();
let PlannedDrainJob::Single(claimed_item) = claimed else {
panic!("expected single job");
};
assert!(!claimed_item.run_id.is_empty());
let run = run_store.read(&claimed_item.run_id).unwrap();
assert_eq!(run.status, ReviewRunStatus::Running);
assert_eq!(run.phase, CLAIMED_PHASE);
assert!(run.claimed_at_unix.is_some());
assert_eq!(queue.pending_read_only().unwrap().len(), 1);
let mut crashed = run;
crashed.worker_pid = Some(reaped_pid());
run_store.write(&crashed).unwrap();
let pending = queue.pending().unwrap();
assert_eq!(pending.len(), 1);
let recovered = run_store.read(&claimed_item.run_id).unwrap();
assert_eq!(recovered.status, ReviewRunStatus::Queued);
assert!(recovered.claimed_at_unix.is_none());
assert!(recovered.worker_pid.is_none());
}
#[test]
fn provider_backoff_persists_without_spending_petition_attempts_or_removing_queue() {
let error = ReviewerError::ReviewerProcessFailed {
status: Some(429),
stderr: "rate_limit exceeded: try again later".to_owned(),
};
assert!(is_retryable_provider_failure(&error));
let temp = tempfile::tempdir().unwrap();
let queue = ReviewQueue::new(temp.path());
let run_store = ReviewRunStore::new(temp.path());
let store = LedgerStore::new(temp.path());
let original = "a".repeat(40);
let fix = "b".repeat(40);
seed_petition(temp.path(), &store, &original, &fix, "finding");
let pending_before = queue.pending().unwrap();
assert_eq!(pending_before.len(), 1);
let run_id = pending_before[0].run_id.clone();
let before_attempts = store.petition_attempts_for(&original).unwrap();
run_store
.mark_provider_backoff(&run_id, error.to_string(), DEFAULT_PROVIDER_BACKOFF_SECS)
.unwrap();
let run = run_store.read(&run_id).unwrap();
assert_eq!(run.status, ReviewRunStatus::Queued);
assert!(run.next_attempt_at_unix.is_some());
assert_eq!(queue.pending_read_only().unwrap().len(), 1);
let filtered = filter_pending_ready_for_attempt(
queue.pending_read_only().unwrap(),
&run_store,
unix_now(),
)
.unwrap();
assert!(filtered.is_empty(), "backoff must defer the attempt");
let after_attempts = store.petition_attempts_for(&original).unwrap();
assert_eq!(before_attempts, after_attempts);
let eligible = filter_pending_ready_for_attempt(
queue.pending_read_only().unwrap(),
&run_store,
run.next_attempt_at_unix.unwrap(),
)
.unwrap();
assert_eq!(eligible.len(), 1);
}
#[test]
fn strict_arbiter_provider_backoff_persists_without_terminating_drain() {
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let queue = ReviewQueue::new(temp.path());
let run_store = ReviewRunStore::new(temp.path());
seed_petition(
temp.path(),
&store,
"orig-a",
"fix-strict-backoff",
"finding-a",
);
// Mixed batch: flag member finishes; provisional PASS waits on strict arbiter.
seed_petition(
temp.path(),
&store,
"orig-b",
"fix-strict-backoff",
"finding-b",
);
let before_a = store.petition_attempts_for("orig-a").unwrap();
let before_b = store.petition_attempts_for("orig-b").unwrap();
let runner = RecordingBatchRunner::new([
BatchReply::Results(vec![
batch_result("orig-a", &pass_json()),
batch_result("orig-b", &flag_json("non-accepting debt")),
]),
BatchReply::ProcessFailed {
status: Some(429),
stderr: "rate_limit exceeded: try again later".to_owned(),
},
]);
let mut strict_selection = selection();
strict_selection.strict = Some(StrictReviewConfig {
arbiter_harness: ReviewerHarness::Claude,
arbiter_model: "claude-opus-4-8".to_owned(),
arbiter_effort: Effort::Xhigh,
});
let report = drain_once(
&queue,
&StaticLoader::new(),
&strict_selection,
"",
&runner,
&store,
&batching_config(32),
)
.expect("strict arbiter 429 must not terminate drain/watch");
assert!(report.failed.is_empty());
// Non-strict-pending member still finishes; backoff must not abort the drain.
assert_eq!(report.reviewed, ["fix-strict-backoff"]);
// Attempt counters were reserved at enqueue; backoff must not spend another.
assert_eq!(store.petition_attempts_for("orig-a").unwrap(), before_a);
assert_eq!(store.petition_attempts_for("orig-b").unwrap(), before_b);
let pending = queue.pending_read_only().unwrap();
assert_eq!(pending.len(), 1);
assert_eq!(pending[0].petition_for.as_deref(), Some("orig-a"));
let run = run_store.read(&pending[0].run_id).unwrap();
assert_eq!(run.status, ReviewRunStatus::Queued);
assert_eq!(run.phase, PROVIDER_BACKOFF_PHASE);
assert!(run.next_attempt_at_unix.is_some());
let deferred = filter_pending_ready_for_attempt(
queue.pending_read_only().unwrap(),
&run_store,
unix_now(),
)
.unwrap();
assert!(
deferred.is_empty(),
"strict 429 backoff must defer next attempt"
);
let eligible = filter_pending_ready_for_attempt(
queue.pending_read_only().unwrap(),
&run_store,
run.next_attempt_at_unix.unwrap(),
)
.unwrap();
assert_eq!(eligible.len(), 1);
assert_eq!(runner.prompts().len(), 2);
}
#[test]
fn orphan_worktree_maintenance_removes_terminal_paths_with_non_fatal_telemetry() {
let temp = tempfile::tempdir().unwrap();
let repository =
PathBuf::from(git_output(["rev-parse", "--show-toplevel"]).unwrap().trim());
let orphan = temp.path().join("worktrees").join("orphan-run");
fs::create_dir_all(orphan.parent().unwrap()).unwrap();
let sha = git_output(["rev-parse", "HEAD"]).unwrap().trim().to_owned();
create_review_worktree(&repository, &orphan, &sha).unwrap();
assert!(orphan.exists());
maintain_terminal_worktrees(temp.path(), &repository);
assert!(
!orphan.exists(),
"orphan worktree should be removed by maintenance cleanup"
);
}
#[test]
fn scheduler_status_read_only_reports_owner_claimed_inflight_workers_and_batches() {
let temp = tempfile::tempdir().unwrap();
let run_store = ReviewRunStore::new(temp.path());
let mut claimed = run_store.create_queued("sha-claimed", "commit").unwrap();
claimed.mark_claimed();
run_store.write(&claimed).unwrap();
let mut inflight = run_store.create_queued("sha-inflight", "commit").unwrap();
inflight.mark_claimed();
inflight.mark_running(REVIEW_LEDGER_PHASE);
inflight.batch = Some(super::BatchRunRef {
batch_id: "batch-status".to_owned(),
kind: super::BatchKind::Petition,
member_index: 0,
member_count: 1,
fix_sha: "sha-inflight".to_owned(),
});
run_store.write(&inflight).unwrap();
let snapshot = run_store
.scheduler_status_read_only(Some(4242), ["batch-queue".to_owned()])
.unwrap();
assert_eq!(snapshot.owner_pid, Some(4242));
assert_eq!(snapshot.claimed, 2);
assert_eq!(snapshot.inflight, 1);
assert_eq!(snapshot.active_workers, 1);
assert_eq!(snapshot.batches, 2);
}
struct StaticLoader {
claim: Claim,
diff: String,
}
impl StaticLoader {
fn new() -> Self {
Self {
claim: claim(),
diff: "diff --git a/src/lib.rs b/src/lib.rs".to_owned(),
}
}
}
impl MaterialLoader for StaticLoader {
fn load(&self, _sha: &str) -> Result<(Claim, String), ReviewerError> {
Ok((self.claim.clone(), self.diff.clone()))
}
}
struct CountingLoader {
calls: RefCell<usize>,
inner: StaticLoader,
}
impl CountingLoader {
fn new() -> Self {
Self {
calls: RefCell::new(0),
inner: StaticLoader::new(),
}
}
fn calls(&self) -> usize {
*self.calls.borrow()
}
}
impl MaterialLoader for CountingLoader {
fn load(&self, sha: &str) -> Result<(Claim, String), ReviewerError> {
*self.calls.borrow_mut() += 1;
self.inner.load(sha)
}
}
fn seed_petition(
state_dir: &std::path::Path,
store: &LedgerStore,
original_sha: &str,
fix_sha: &str,
finding: &str,
) {
let rejected = crate::ledger::LedgerEntry::new(
original_sha,
Verdict::Reject,
format!(
"CLAIM: original {original_sha} | verified: cargo test | evidence: tests:cargo-test"
),
vec!["tests:cargo-test".to_owned()],
crate::ledger::ReviewerConfig::new("claude", "claude-opus-4-1", false),
vec![finding.to_owned()],
);
store.append_entry(&rejected).unwrap();
super::commit_petition_resolve(state_dir, store, original_sha, fix_sha).unwrap();
}
fn batching_config(max_petition_batch_size: usize) -> TruthMirrorConfig {
let mut config = TruthMirrorConfig::default();
config.reviewer.batch_petitions = true;
config.reviewer.max_petition_batch_size = max_petition_batch_size;
config
}
struct PoisonedLoader;
impl MaterialLoader for PoisonedLoader {
fn load(&self, sha: &str) -> Result<(Claim, String), ReviewerError> {
if sha == "poisoned" {
return Err(crate::claim::ClaimError::MissingEvidence.into());
}
StaticLoader::new().load(sha)
}
}
struct ConstRunner {
output: String,
}
impl ConstRunner {
fn new(output: impl Into<String>) -> Self {
Self {
output: output.into(),
}
}
}
impl ProcessRunner for ConstRunner {
fn run(
&self,
_invocation: &InvocationPlan,
_prompt: &str,
_timeout: Option<Duration>,
) -> Result<ProcessOutput, ReviewerError> {
Ok(ProcessOutput {
status_code: Some(0),
stdout: self.output.clone(),
stderr: String::new(),
})
}
}
struct ConcurrentRunner {
active: AtomicUsize,
max_active: AtomicUsize,
arrivals: AtomicUsize,
overlap: usize,
dirs: Mutex<Vec<PathBuf>>,
}
impl ConcurrentRunner {
fn new() -> Self {
Self::with_overlap(1)
}
fn with_overlap(overlap: usize) -> Self {
Self {
active: AtomicUsize::new(0),
max_active: AtomicUsize::new(0),
arrivals: AtomicUsize::new(0),
overlap: overlap.max(1),
dirs: Mutex::new(Vec::new()),
}
}
fn record_max(&self, active: usize) {
let mut observed = self.max_active.load(Ordering::Relaxed);
while active > observed {
match self.max_active.compare_exchange(
observed,
active,
Ordering::Relaxed,
Ordering::Relaxed,
) {
Ok(_) => break,
Err(current) => observed = current,
}
}
}
fn observe(&self, current_dir: Option<&Path>) {
if let Some(dir) = current_dir {
self.dirs.lock().unwrap().push(dir.to_owned());
}
let ticket = self.arrivals.fetch_add(1, Ordering::SeqCst);
let active = self.active.fetch_add(1, Ordering::SeqCst) + 1;
self.record_max(active);
if current_dir.is_some() && ticket < self.overlap {
let started = std::time::Instant::now();
while self.arrivals.load(Ordering::SeqCst) < self.overlap
&& started.elapsed() < Duration::from_secs(10)
{
thread::sleep(Duration::from_millis(10));
}
self.record_max(self.active.load(Ordering::SeqCst));
thread::sleep(Duration::from_millis(50));
} else {
thread::sleep(Duration::from_millis(20));
}
self.active.fetch_sub(1, Ordering::SeqCst);
}
}
impl ProcessRunner for ConcurrentRunner {
fn run(
&self,
_invocation: &InvocationPlan,
_prompt: &str,
_timeout: Option<Duration>,
) -> Result<ProcessOutput, ReviewerError> {
self.observe(None);
Ok(ProcessOutput {
status_code: Some(0),
stdout: pass_json(),
stderr: String::new(),
})
}
fn run_in_dir(
&self,
invocation: &InvocationPlan,
prompt: &str,
timeout: Option<Duration>,
current_dir: Option<&Path>,
) -> Result<ProcessOutput, ReviewerError> {
let _ = timeout;
if current_dir.is_some() {
self.observe(current_dir);
Ok(ProcessOutput {
status_code: Some(0),
stdout: pass_json(),
stderr: String::new(),
})
} else {
self.run(invocation, prompt, timeout)
}
}
}
struct SnapshotRunner {
queue: ReviewQueue,
late_sha: String,
enqueued_late: AtomicBool,
inner: ConcurrentRunner,
}
impl SnapshotRunner {
fn new(queue: ReviewQueue, late_sha: String) -> Self {
Self {
queue,
late_sha,
enqueued_late: AtomicBool::new(false),
inner: ConcurrentRunner::new(),
}
}
fn maybe_enqueue_late(&self) {
if self
.enqueued_late
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
.is_ok()
{
self.queue.enqueue(&self.late_sha).unwrap();
}
}
}
impl ProcessRunner for SnapshotRunner {
fn run(
&self,
invocation: &InvocationPlan,
prompt: &str,
timeout: Option<Duration>,
) -> Result<ProcessOutput, ReviewerError> {
self.maybe_enqueue_late();
self.inner.run(invocation, prompt, timeout)
}
fn run_in_dir(
&self,
invocation: &InvocationPlan,
prompt: &str,
timeout: Option<Duration>,
current_dir: Option<&Path>,
) -> Result<ProcessOutput, ReviewerError> {
self.maybe_enqueue_late();
self.inner
.run_in_dir(invocation, prompt, timeout, current_dir)
}
}
struct PanicRunner;
impl ProcessRunner for PanicRunner {
fn run(
&self,
_invocation: &InvocationPlan,
_prompt: &str,
_timeout: Option<Duration>,
) -> Result<ProcessOutput, ReviewerError> {
panic!("provider must not run when pending_review spool is durable");
}
}
fn review_job(strict: bool) -> ReviewJob {
let claim = claim();
ReviewJob {
commit_sha: "abc123".to_owned(),
diff: "diff --git a/src/lib.rs b/src/lib.rs".to_owned(),
context: String::new(),
request: ReviewRequest::new(
Agent::Codex,
"gpt-5.4",
ReviewerHarness::Codex,
"gpt-5.5",
false,
"review this",
),
claim,
strict: strict.then_some(StrictReviewConfig {
arbiter_harness: ReviewerHarness::Claude,
arbiter_model: "claude-opus-4-8".to_owned(),
arbiter_effort: Effort::Xhigh,
}),
petition: None,
}
}
struct SequenceRunner {
outputs: RefCell<VecDeque<String>>,
}
impl SequenceRunner {
fn new<I, S>(outputs: I) -> Self
where
I: IntoIterator<Item = S>,
S: Into<String>,
{
Self {
outputs: RefCell::new(outputs.into_iter().map(Into::into).collect()),
}
}
}
impl ProcessRunner for SequenceRunner {
fn run(
&self,
_invocation: &InvocationPlan,
_prompt: &str,
_timeout: Option<Duration>,
) -> Result<ProcessOutput, ReviewerError> {
let stdout = self.outputs.borrow_mut().pop_front().unwrap();
Ok(ProcessOutput {
status_code: Some(0),
stdout,
stderr: String::new(),
})
}
}
enum BatchReply {
Results(Vec<serde_json::Value>),
PassEveryRequestedMember,
MalformedResultsArray,
Raw(String),
ProcessFailed { status: Option<i32>, stderr: String },
}
struct RecordingBatchRunner {
replies: RefCell<VecDeque<BatchReply>>,
prompts: RefCell<Vec<String>>,
}
impl RecordingBatchRunner {
fn new(replies: impl IntoIterator<Item = BatchReply>) -> Self {
Self {
replies: RefCell::new(replies.into_iter().collect()),
prompts: RefCell::new(Vec::new()),
}
}
fn prompts(&self) -> Vec<String> {
self.prompts.borrow().clone()
}
}
fn batch_prompt_value(prompt: &str, prefix: &str) -> String {
prompt
.lines()
.find_map(|line| line.strip_prefix(prefix))
.unwrap_or_else(|| panic!("batch prompt missing {prefix:?}"))
.to_owned()
}
fn batch_prompt_requested_shas(prompt: &str) -> Vec<String> {
prompt
.lines()
.filter_map(|line| line.split_once(". ORIGINAL REJECTION SHA: "))
.map(|(_, sha)| sha.to_owned())
.collect()
}
impl ProcessRunner for RecordingBatchRunner {
fn run(
&self,
_invocation: &InvocationPlan,
prompt: &str,
_timeout: Option<Duration>,
) -> Result<ProcessOutput, ReviewerError> {
self.prompts.borrow_mut().push(prompt.to_owned());
let reply = self.replies.borrow_mut().pop_front().unwrap();
if let BatchReply::ProcessFailed { status, stderr } = reply {
return Ok(ProcessOutput {
status_code: status,
stdout: String::new(),
stderr,
});
}
let stdout = match reply {
BatchReply::Raw(output) => output,
reply => {
let batch_id = batch_prompt_value(prompt, "- batch ID: ");
let fix_sha = batch_prompt_value(prompt, "- fix commit SHA: ");
let results = match reply {
BatchReply::Results(results) => serde_json::Value::Array(results),
BatchReply::PassEveryRequestedMember => serde_json::Value::Array(
batch_prompt_requested_shas(prompt)
.into_iter()
.map(|sha| batch_result(&sha, &pass_json()))
.collect(),
),
BatchReply::MalformedResultsArray => {
serde_json::json!({"not": "an array"})
}
BatchReply::Raw(_) | BatchReply::ProcessFailed { .. } => unreachable!(),
};
serde_json::json!({
"batch_id": batch_id,
"fix_sha": fix_sha,
"results": results,
})
.to_string()
}
};
Ok(ProcessOutput {
status_code: Some(0),
stdout,
stderr: String::new(),
})
}
}
#[test]
fn petition_batch_key_covers_repository_fix_selection_strict_and_context() {
let base = selection();
let mut different_reviewer = base.clone();
different_reviewer.reviewer_model = "different-reviewer".to_owned();
let mut strict = base.clone();
strict.strict = Some(StrictReviewConfig {
arbiter_harness: ReviewerHarness::Claude,
arbiter_model: "claude-opus-4-8".to_owned(),
arbiter_effort: Effort::High,
});
let key =
super::PetitionBatchKey::new(std::path::Path::new("repo-a"), "fix-a", &base, "ctx-a");
assert_ne!(
key,
super::PetitionBatchKey::new(std::path::Path::new("repo-b"), "fix-a", &base, "ctx-a")
);
assert_ne!(
key,
super::PetitionBatchKey::new(std::path::Path::new("repo-a"), "fix-b", &base, "ctx-a")
);
assert_ne!(
key,
super::PetitionBatchKey::new(
std::path::Path::new("repo-a"),
"fix-a",
&different_reviewer,
"ctx-a"
)
);
assert_ne!(
key,
super::PetitionBatchKey::new(std::path::Path::new("repo-a"), "fix-a", &strict, "ctx-a")
);
assert_ne!(
key,
super::PetitionBatchKey::new(std::path::Path::new("repo-a"), "fix-a", &base, "ctx-b")
);
}
#[test]
fn petition_batch_groups_same_fix_once_and_applies_sha_keyed_outcomes() {
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let queue = ReviewQueue::new(temp.path());
for (original, finding) in [
("orig-c", "finding-c"),
("orig-a", "finding-a"),
("orig-b", "finding-b"),
] {
seed_petition(temp.path(), &store, original, "fix-1", finding);
}
let loader = CountingLoader::new();
let runner = RecordingBatchRunner::new([BatchReply::Results(vec![
batch_result("orig-b", &reject_json("still broken")),
batch_result("orig-a", &pass_json()),
batch_result("orig-c", &flag_json("debt remains")),
])]);
let report = drain_once(
&queue,
&loader,
&selection(),
"MATERIALIZED CONTEXT",
&runner,
&store,
&batching_config(32),
)
.unwrap();
assert_eq!(loader.calls(), 1);
assert_eq!(runner.prompts().len(), 1);
assert_eq!(report.reviewed, ["fix-1", "fix-1", "fix-1"]);
assert!(report.failed.is_empty());
let prompt = &runner.prompts()[0];
assert_eq!(
prompt
.matches("diff --git a/src/lib.rs b/src/lib.rs")
.count(),
1
);
assert!(prompt.contains("finding-a"));
assert!(prompt.contains("finding-b"));
assert!(prompt.contains("finding-c"));
let a = prompt.find("ORIGINAL REJECTION SHA: orig-a").unwrap();
let b = prompt.find("ORIGINAL REJECTION SHA: orig-b").unwrap();
let c = prompt.find("ORIGINAL REJECTION SHA: orig-c").unwrap();
assert!(a < b && b < c);
assert_eq!(
store.show("orig-a").unwrap().disposition,
crate::ledger::Disposition::Resolved
);
assert_eq!(
store.show("orig-b").unwrap().disposition,
crate::ledger::Disposition::Open
);
assert_eq!(
store.show("orig-c").unwrap().disposition,
crate::ledger::Disposition::Open
);
for original in ["orig-a", "orig-b", "orig-c"] {
assert_eq!(store.petition_attempts_for(original).unwrap(), 1);
}
let petition_entries: Vec<_> = store
.read_history()
.unwrap()
.into_iter()
.filter(|entry| entry.commit_sha == "fix-1")
.collect();
assert_eq!(petition_entries.len(), 3);
assert!(
petition_entries
.iter()
.all(|entry| !entry.raw_reviewer_output.contains("\"results\""))
);
assert!(queue.pending().unwrap().is_empty());
}
#[test]
fn petition_batch_isolates_duplicate_missing_unknown_and_malformed_members() {
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let queue = ReviewQueue::new(temp.path());
for original in ["orig-a", "orig-b", "orig-c", "orig-d"] {
seed_petition(
temp.path(),
&store,
original,
"fix-isolation",
&format!("finding-{original}"),
);
}
let mut malformed = batch_result("orig-d", &pass_json());
malformed
.as_object_mut()
.unwrap()
.insert("summary".to_owned(), serde_json::json!(""));
let runner = RecordingBatchRunner::new([BatchReply::Results(vec![
batch_result("orig-a", &pass_json()),
batch_result("orig-b", &pass_json()),
batch_result("orig-b", &pass_json()),
malformed,
batch_result("unknown-original", &pass_json()),
])]);
let report = drain_once(
&queue,
&StaticLoader::new(),
&selection(),
"",
&runner,
&store,
&batching_config(32),
)
.unwrap();
assert_eq!(report.reviewed, ["fix-isolation"]);
assert_eq!(report.failed.len(), 3);
assert_eq!(
store.show("orig-a").unwrap().disposition,
crate::ledger::Disposition::Resolved
);
for original in ["orig-b", "orig-c", "orig-d"] {
assert_eq!(
store.show(original).unwrap().disposition,
crate::ledger::Disposition::Open
);
assert_eq!(store.petition_attempts_for(original).unwrap(), 1);
}
assert_eq!(
store
.read_history()
.unwrap()
.iter()
.filter(|entry| entry.commit_sha == "fix-isolation")
.count(),
1
);
let runs = ReviewRunStore::new(temp.path()).list().unwrap();
for (original, expected_error) in [
("orig-b", "duplicate batch results"),
("orig-c", "missing batch result"),
("orig-d", "summary must not be empty"),
] {
let run = runs
.iter()
.find(|run| run.petition_for.as_deref() == Some(original))
.unwrap();
assert_eq!(run.status, ReviewRunStatus::Failed);
assert!(run.error.as_deref().unwrap().contains(expected_error));
}
assert!(queue.pending().unwrap().is_empty());
let (_, attempts) =
super::commit_petition_resolve(temp.path(), &store, "orig-b", "fix-retry").unwrap();
assert_eq!(attempts, 2);
let retry = queue.pending().unwrap();
assert_eq!(retry.len(), 1);
assert_eq!(retry[0].commit_sha, "fix-retry");
assert_eq!(retry[0].petition_for.as_deref(), Some("orig-b"));
}
#[test]
fn malformed_outer_petition_batch_fails_every_member_without_a_verdict() {
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let queue = ReviewQueue::new(temp.path());
for original in ["orig-a", "orig-b"] {
seed_petition(
temp.path(),
&store,
original,
"fix-outer",
&format!("finding-{original}"),
);
}
let runner = RecordingBatchRunner::new([BatchReply::MalformedResultsArray]);
let report = drain_once(
&queue,
&StaticLoader::new(),
&selection(),
"",
&runner,
&store,
&batching_config(32),
)
.unwrap();
assert!(report.reviewed.is_empty());
assert_eq!(report.failed, ["fix-outer", "fix-outer"]);
assert_eq!(
store
.read_history()
.unwrap()
.iter()
.filter(|entry| entry.commit_sha == "fix-outer")
.count(),
0
);
for original in ["orig-a", "orig-b"] {
assert_eq!(store.petition_attempts_for(original).unwrap(), 1);
}
assert!(queue.pending().unwrap().is_empty());
}
#[test]
fn petition_batch_size_splits_deterministically_at_the_configured_cap() {
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let queue = ReviewQueue::new(temp.path());
for original in ["orig-e", "orig-c", "orig-a", "orig-d", "orig-b"] {
seed_petition(
temp.path(),
&store,
original,
"fix-capped",
&format!("finding-{original}"),
);
}
let runner = RecordingBatchRunner::new([
BatchReply::PassEveryRequestedMember,
BatchReply::PassEveryRequestedMember,
BatchReply::PassEveryRequestedMember,
]);
drain_once(
&queue,
&StaticLoader::new(),
&selection(),
"",
&runner,
&store,
&batching_config(2),
)
.unwrap();
let requested: Vec<Vec<String>> = runner
.prompts()
.iter()
.map(|prompt| batch_prompt_requested_shas(prompt))
.collect();
assert_eq!(
requested,
[
vec!["orig-a".to_owned(), "orig-b".to_owned()],
vec!["orig-c".to_owned(), "orig-d".to_owned()],
vec!["orig-e".to_owned()],
]
);
assert!(runner.prompts().iter().all(|prompt| {
prompt
.matches("FIX COMMIT DIFF (included exactly once)")
.count()
== 1
}));
}
#[test]
fn strict_petition_batch_arbitrates_only_provisional_pass_members() {
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let queue = ReviewQueue::new(temp.path());
for original in ["orig-a", "orig-b"] {
seed_petition(
temp.path(),
&store,
original,
"fix-strict",
&format!("finding-{original}"),
);
}
let runner = RecordingBatchRunner::new([
BatchReply::Results(vec![
batch_result("orig-a", &pass_json()),
batch_result("orig-b", &flag_json("non-accepting debt")),
]),
BatchReply::Results(vec![batch_result(
"orig-a",
&reject_json("arbiter found gap"),
)]),
]);
let mut strict_selection = selection();
strict_selection.strict = Some(StrictReviewConfig {
arbiter_harness: ReviewerHarness::Claude,
arbiter_model: "claude-opus-4-8".to_owned(),
arbiter_effort: Effort::Xhigh,
});
drain_once(
&queue,
&StaticLoader::new(),
&strict_selection,
"",
&runner,
&store,
&batching_config(32),
)
.unwrap();
let prompts = runner.prompts();
assert_eq!(prompts.len(), 2);
assert_eq!(
batch_prompt_requested_shas(&prompts[0]),
["orig-a", "orig-b"]
);
assert_eq!(batch_prompt_requested_shas(&prompts[1]), ["orig-a"]);
assert!(!prompts[1].contains("orig-b"));
assert_eq!(
store.show("orig-a").unwrap().disposition,
crate::ledger::Disposition::Open
);
assert_eq!(
store.show("orig-b").unwrap().disposition,
crate::ledger::Disposition::Open
);
let history = store.read_history().unwrap();
let a_verdicts: Vec<_> = history
.iter()
.filter(|entry| {
entry.commit_sha == "fix-strict" && entry.petition_for.as_deref() == Some("orig-a")
})
.map(|entry| entry.verdict)
.collect();
assert_eq!(a_verdicts, [Verdict::Pass, Verdict::Reject]);
let audit_order: Vec<_> = history
.iter()
.filter(|entry| entry.commit_sha == "fix-strict")
.map(|entry| entry.petition_for.as_deref().unwrap())
.collect();
assert_eq!(audit_order, ["orig-a", "orig-a", "orig-b"]);
assert!(history.iter().any(|entry| {
entry.commit_sha == "fix-strict"
&& entry.petition_for.as_deref() == Some("orig-b")
&& entry.verdict == Verdict::Flag
}));
}
#[test]
fn enabled_batching_keeps_normal_review_separate_from_same_fix_petitions() {
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let queue = ReviewQueue::new(temp.path());
queue.enqueue("fix-mixed").unwrap();
for original in ["orig-a", "orig-b"] {
seed_petition(
temp.path(),
&store,
original,
"fix-mixed",
&format!("finding-{original}"),
);
}
let loader = CountingLoader::new();
let runner = RecordingBatchRunner::new([
BatchReply::Raw(pass_json()),
BatchReply::PassEveryRequestedMember,
]);
let report = drain_once(
&queue,
&loader,
&selection(),
"",
&runner,
&store,
&batching_config(32),
)
.unwrap();
assert_eq!(loader.calls(), 2);
assert_eq!(runner.prompts().len(), 2);
assert!(!runner.prompts()[0].contains("BATCH:"));
assert_eq!(
batch_prompt_requested_shas(&runner.prompts()[1]),
["orig-a", "orig-b"]
);
assert_eq!(report.reviewed, ["fix-mixed", "fix-mixed", "fix-mixed"]);
let fix_entries: Vec<_> = store
.read_history()
.unwrap()
.into_iter()
.filter(|entry| entry.commit_sha == "fix-mixed")
.collect();
assert_eq!(fix_entries.len(), 3);
assert_eq!(
fix_entries
.iter()
.filter(|entry| entry.petition_for.is_none())
.count(),
1
);
assert!(queue.pending().unwrap().is_empty());
}
#[test]
fn disabled_petition_batching_preserves_one_invocation_per_petition() {
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let queue = ReviewQueue::new(temp.path());
for original in ["orig-a", "orig-b"] {
seed_petition(
temp.path(),
&store,
original,
"fix-legacy",
&format!("finding-{original}"),
);
}
let loader = CountingLoader::new();
let runner =
RecordingBatchRunner::new([BatchReply::Raw(pass_json()), BatchReply::Raw(pass_json())]);
drain_once(
&queue,
&loader,
&selection(),
"",
&runner,
&store,
&TruthMirrorConfig::default(),
)
.unwrap();
assert_eq!(loader.calls(), 2);
assert_eq!(runner.prompts().len(), 2);
assert!(
runner
.prompts()
.iter()
.all(|prompt| prompt.contains("PETITION:") && !prompt.contains("BATCH:"))
);
}
#[test]
fn extract_verdict_json_handles_raw_fenced_and_prose_wrapped_output() {
// Raw JSON (the documented contract) passes through untouched.
let raw = r#"{"verdict":"PASS"}"#;
assert_eq!(super::extract_verdict_json(raw), raw);
// Prose around a ```json fence: the fenced body is extracted.
let fenced = "Here is my verdict:\n```json\n{\"verdict\":\"PASS\"}\n```\nthanks";
assert_eq!(
super::extract_verdict_json(fenced),
"{\"verdict\":\"PASS\"}"
);
// Bare ``` fence without an info string.
let bare = "verdict below\n```\n{\"a\":1}\n```";
assert_eq!(super::extract_verdict_json(bare), "{\"a\":1}");
// A non-JSON fence first, the JSON fence second: skips to the JSON one.
let two = "```\nnot json\n```\nthen\n```json\n{\"b\":2}\n```";
assert_eq!(super::extract_verdict_json(two), "{\"b\":2}");
// No fence at all: the outermost brace span.
let prose = "The verdict is {\"a\": {\"b\":2}} as required.";
assert_eq!(super::extract_verdict_json(prose), "{\"a\": {\"b\":2}}");
// Nothing JSON-shaped: falls back to the trimmed output so serde's
// error carries the real text.
assert_eq!(super::extract_verdict_json(" nope "), "nope");
}
#[test]
fn parsed_verdict_accepts_markdown_fenced_reviewer_output() {
// The dead-watcher regression: reviewer models wrap the verdict in
// prose + a fenced block; 0.8.0 demanded JSON at byte 0 and the first
// real review errored out, so the queue never drained.
let fenced = format!(
"The fix looks good. Verdict follows.\n\n```json\n{}\n```\n",
pass_json()
);
let parsed = ParsedVerdict::parse(&fenced).unwrap();
assert_eq!(parsed.verdict, Verdict::Pass);
// `raw` preserves the reviewer's original output verbatim.
assert_eq!(parsed.raw, fenced);
}
#[test]
fn parsed_verdict_accepts_fenced_json_prefix_with_trailing_prose() {
let fenced = format!(
"Reviewer explanation.\n\n```json\n{}\nThe JSON above is the complete verdict.\n```\n",
pass_json()
);
let parsed = ParsedVerdict::parse(&fenced).unwrap();
assert_eq!(parsed.verdict, Verdict::Pass);
}
#[test]
fn legacy_queued_review_without_optional_fields_still_parses() {
let legacy = r#"{"commit_sha":"abc","enqueued_at_unix":1}"#;
let item: super::QueuedReview = serde_json::from_str(legacy).unwrap();
assert!(item.run_id.is_empty());
assert_eq!(item.petition_for, None);
}
#[test]
fn enqueue_petition_round_trips_petition_for() {
let temp = tempfile::tempdir().unwrap();
let queue = ReviewQueue::new(temp.path());
queue.enqueue("normal1").unwrap();
queue.enqueue_petition("fix456", "orig123").unwrap();
let pending = queue.pending().unwrap();
assert_eq!(pending.len(), 2);
assert_eq!(pending[0].petition_for, None);
assert_eq!(pending[1].commit_sha, "fix456");
assert_eq!(pending[1].petition_for.as_deref(), Some("orig123"));
}
#[test]
fn drain_once_runs_petition_jobs_and_transitions_the_original_rejection() {
use crate::ledger::{Disposition, LedgerEntry, ResolutionKind, ReviewerConfig};
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let queue = ReviewQueue::new(temp.path());
// The original rejection, plus the CLI's attempt bookkeeping
// (`resolve --fixed-by` counts the attempt when it enqueues).
let rejected = LedgerEntry::new(
"orig123",
Verdict::Reject,
"CLAIM: original | verified: cargo test | evidence: tests:cargo-test",
vec!["tests:cargo-test".to_owned()],
ReviewerConfig::new("claude", "claude-opus-4-1", false),
vec!["evidence too thin".to_owned()],
);
store.append_entry(&rejected).unwrap();
store
.append_petition_transition(
"orig123",
Disposition::Open,
ResolutionKind::Resolved,
"petition review enqueued: fix=fix456, attempts=1/2",
1,
)
.unwrap();
queue.enqueue_petition("fix456", "orig123").unwrap();
let loader = StaticLoader::new();
let runner = ConstRunner::new(pass_json());
let report = drain_once(
&queue,
&loader,
&selection(),
"",
&runner,
&store,
&TruthMirrorConfig::default(),
)
.unwrap();
assert_eq!(report.reviewed, ["fix456"]);
// The petition review entry lands under the FIX sha, tagged with the
// original, carrying the CLI's counter unchanged (no double increment).
let review = store.show("fix456").unwrap();
assert_eq!(review.petition_for.as_deref(), Some("orig123"));
assert_eq!(review.petition_attempts, 1);
// apply_petition_transition resolved the original rejection.
let original = store.show("orig123").unwrap();
assert_eq!(original.disposition, Disposition::Resolved);
assert!(queue.pending().unwrap().is_empty());
assert!(store.blocking_rejections().unwrap().is_empty());
}
#[test]
fn extract_verdict_json_takes_leading_json_before_trailing_prose() {
// Round-2 review finding: leading JSON + trailing prose used to be
// returned whole (starts_with '{'), and serde died on the trailer.
let leading = "{\"verdict\":\"PASS\"} \n\nHope this helps!";
assert_eq!(
super::extract_verdict_json(leading),
"{\"verdict\":\"PASS\"}"
);
// Prose on both sides without fences: first complete JSON value wins
// even when the trailing prose contains a stray closing brace.
let both = "verdict: {\"a\":1} trailing prose with a stray } brace";
assert_eq!(super::extract_verdict_json(both), "{\"a\":1}");
}
#[test]
fn extract_verdict_json_skips_non_json_brace_to_find_verdict() {
// Burned-petition regression: prose contained `getCapabilities({preview:true})`
// before the real verdict JSON. The old code tried only the first `{`,
// failed on the unquoted-key fragment, and fell through to trimmed prose.
let output = format!(
"The API call getCapabilities({{preview:true}}) returned nothing useful.\n\n{}",
flag_json("burned petition")
);
let parsed = ParsedVerdict::parse(&output).unwrap();
assert_eq!(parsed.verdict, Verdict::Flag);
assert_eq!(parsed.structured_findings[0].title, "burned petition");
}
#[test]
fn extract_verdict_json_prefers_verdict_object_over_stray_json() {
// A small valid JSON fragment in the prose must not shadow the real verdict.
let output = format!(
"For reference, here's an example object: {{\"example\": 1}}.\n\n{}",
reject_json("real finding")
);
let parsed = ParsedVerdict::parse(&output).unwrap();
assert_eq!(parsed.verdict, Verdict::Reject);
assert_eq!(parsed.structured_findings[0].title, "real finding");
}
#[test]
fn drain_once_reviews_normal_and_petition_items_for_the_same_fix_sha() {
use crate::ledger::{Disposition, LedgerEntry, ResolutionKind, ReviewerConfig};
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let queue = ReviewQueue::new(temp.path());
let rejected = LedgerEntry::new(
"orig123",
Verdict::Reject,
"CLAIM: original | verified: cargo test | evidence: tests:cargo-test",
vec!["tests:cargo-test".to_owned()],
ReviewerConfig::new("claude", "claude-opus-4-1", false),
vec!["evidence too thin".to_owned()],
);
store.append_entry(&rejected).unwrap();
store
.append_petition_transition(
"orig123",
Disposition::Open,
ResolutionKind::Resolved,
"petition review enqueued: fix=fix456, attempts=1/2",
1,
)
.unwrap();
// The SAME fix commit is queued twice with different identities: its
// own post-commit review AND a petition review. Dedup by bare SHA
// used to cancel one of them silently.
queue.enqueue("fix456").unwrap();
queue.enqueue_petition("fix456", "orig123").unwrap();
let loader = StaticLoader::new();
let runner = ConstRunner::new(pass_json());
let report = drain_once(
&queue,
&loader,
&selection(),
"",
&runner,
&store,
&TruthMirrorConfig::default(),
)
.unwrap();
assert_eq!(report.reviewed, ["fix456", "fix456"]);
assert!(queue.pending().unwrap().is_empty());
// Both entries exist in history: the fix commit's own PASS and the
// petition review tagged with the original rejection.
let history = store.read_history().unwrap();
let fix_entries: Vec<_> = history
.iter()
.filter(|entry| entry.commit_sha == "fix456")
.collect();
assert_eq!(fix_entries.len(), 2);
assert!(fix_entries.iter().any(|entry| entry.petition_for.is_none()));
assert!(
fix_entries
.iter()
.any(|entry| entry.petition_for.as_deref() == Some("orig123"))
);
// And the original rejection resolved via the petition leg.
assert_eq!(
store.show("orig123").unwrap().disposition,
crate::ledger::Disposition::Resolved
);
}
#[test]
fn pending_does_not_duplicate_legacy_queue_row_for_orphan_run() {
let temp = tempfile::tempdir().unwrap();
let queue = ReviewQueue::new(temp.path());
let sha = "abc1234567890123456789012345678901234567890";
fs::write(
queue.path(),
format!(
"{}\n",
serde_json::to_string(&QueuedReview {
run_id: String::new(),
commit_sha: sha.to_owned(),
enqueued_at_unix: 1,
petition_for: None,
..Default::default()
})
.unwrap()
),
)
.unwrap();
fs::create_dir_all(temp.path().join("runs")).unwrap();
let run = ReviewRun::queued("run-1", sha, "commit");
fs::write(
temp.path().join("runs/run-1.json"),
serde_json::to_string(&run).unwrap(),
)
.unwrap();
let pending = queue.pending().unwrap();
assert_eq!(pending.len(), 1);
assert_eq!(pending[0].commit_sha, sha);
assert_eq!(pending[0].run_id, "run-1");
}
#[test]
fn enqueue_inflight_recovery_republishes_queue_after_crash() {
let temp = tempfile::tempdir().unwrap();
let queue = ReviewQueue::new(temp.path());
let sha = "abc1234567890123456789012345678901234567890";
let run_store = ReviewRunStore::new(temp.path());
let run = run_store
.ensure_queued("run-inflight", sha, "commit", None)
.unwrap();
let item = QueuedReview {
run_id: run.id,
commit_sha: sha.to_owned(),
enqueued_at_unix: 1,
petition_for: None,
..Default::default()
};
write_enqueue_inflight(temp.path(), &item).unwrap();
let pending = queue.pending().unwrap();
assert_eq!(pending.len(), 1);
assert_eq!(pending[0].run_id, "run-inflight");
assert!(!temp.path().join(ENQUEUE_INFLIGHT_FILE).exists());
}
#[test]
fn drain_once_replay_applies_pending_spool_without_rerunning_provider() {
use crate::ledger::{LedgerEntry, ReviewerConfig, Verdict};
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let queue = ReviewQueue::new(temp.path());
let sha = "abc123";
let run_store = ReviewRunStore::new(temp.path());
let queued = queue.enqueue(sha).unwrap();
run_store
.mark_running(&queued.run_id, REVIEW_LEDGER_PHASE)
.unwrap();
let entry = LedgerEntry::new(
sha,
Verdict::Pass,
"CLAIM: ok | verified: cargo test | evidence: tests:cargo-test",
vec!["tests:cargo-test".to_owned()],
ReviewerConfig::new("claude", "claude-opus-4-1", false),
vec![],
);
run_store
.persist_pending_review(
&queued.run_id,
PendingReviewSpool {
entries: vec![entry],
petition_transition_pending: false,
strict_pass_stdout: None,
},
)
.unwrap();
let report = drain_once(
&queue,
&StaticLoader::new(),
&selection(),
"",
&PanicRunner,
&store,
&TruthMirrorConfig::default(),
)
.unwrap();
assert_eq!(report.reviewed, [sha]);
assert_eq!(report.ledger_entries, 1);
assert_eq!(store.read_history().unwrap().len(), 1);
let completed = run_store.read(&queued.run_id).unwrap();
assert_eq!(completed.status, ReviewRunStatus::Completed);
assert!(completed.pending_review.is_none());
assert_eq!(
completed.ledger_applied_phase.as_deref(),
Some(REVIEW_LEDGER_PHASE)
);
assert!(queue.pending().unwrap().is_empty());
}
#[test]
fn drain_once_replay_applies_pending_petition_transition_without_rerunning_provider() {
use crate::ledger::{Disposition, LedgerEntry, ReviewerConfig, Verdict};
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let queue = ReviewQueue::new(temp.path());
let run_store = ReviewRunStore::new(temp.path());
let rejected = LedgerEntry::new(
"orig123",
Verdict::Reject,
"CLAIM: original | verified: cargo test | evidence: tests:cargo-test",
vec!["tests:cargo-test".to_owned()],
ReviewerConfig::new("claude", "claude-opus-4-1", false),
vec!["evidence too thin".to_owned()],
);
store.append_entry(&rejected).unwrap();
store
.append_petition_transition(
"orig123",
Disposition::Open,
crate::ledger::ResolutionKind::Resolved,
"petition review enqueued: fix=fix456, attempts=1/2",
1,
)
.unwrap();
let queued = queue.enqueue_petition("fix456", "orig123").unwrap();
run_store
.mark_running(&queued.run_id, REVIEW_LEDGER_PHASE)
.unwrap();
let mut review = LedgerEntry::new(
"fix456",
Verdict::Pass,
"CLAIM: ok | verified: cargo test | evidence: tests:cargo-test",
vec!["tests:cargo-test".to_owned()],
ReviewerConfig::new("claude", "claude-opus-4-1", false),
vec![],
);
review.petition_for = Some("orig123".to_owned());
review.petition_attempts = 1;
run_store
.persist_pending_review(
&queued.run_id,
PendingReviewSpool {
entries: vec![review],
petition_transition_pending: true,
strict_pass_stdout: None,
},
)
.unwrap();
let report = drain_once(
&queue,
&StaticLoader::new(),
&selection(),
"",
&PanicRunner,
&store,
&TruthMirrorConfig::default(),
)
.unwrap();
assert_eq!(report.reviewed, ["fix456"]);
let fix = store.show("fix456").unwrap();
assert_eq!(fix.petition_for.as_deref(), Some("orig123"));
assert_eq!(fix.petition_attempts, 1);
let original = store.show("orig123").unwrap();
assert_eq!(original.disposition, Disposition::Resolved);
let completed = run_store.read(&queued.run_id).unwrap();
assert_eq!(completed.status, ReviewRunStatus::Completed);
assert!(completed.pending_review.is_none());
assert!(queue.pending().unwrap().is_empty());
}
#[test]
fn drain_once_reject_petition_appends_review_transition_after_enqueue_and_replay_is_idempotent()
{
use crate::ledger::{
Disposition, LedgerEntry, ReviewerConfig, TransitionProvenance,
TransitionProvenanceKind, Verdict,
};
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let queue = ReviewQueue::new(temp.path());
let run_store = ReviewRunStore::new(temp.path());
let rejected = LedgerEntry::new(
"orig123",
Verdict::Reject,
"CLAIM: original | verified: cargo test | evidence: tests:cargo-test",
vec!["tests:cargo-test".to_owned()],
ReviewerConfig::new("claude", "claude-opus-4-1", false),
vec!["evidence too thin".to_owned()],
);
store.append_entry(&rejected).unwrap();
store
.append_petition_transition_with_provenance(
"orig123",
Disposition::Open,
crate::ledger::ResolutionKind::Resolved,
"petition review enqueued: fix=fix456, attempts=1/2",
1,
Some(TransitionProvenance {
kind: TransitionProvenanceKind::PetitionEnqueued,
fix_sha: "fix456".to_owned(),
original_sha: "orig123".to_owned(),
attempts: 1,
}),
)
.unwrap();
let queued = queue.enqueue_petition("fix456", "orig123").unwrap();
run_store
.mark_running(&queued.run_id, REVIEW_LEDGER_PHASE)
.unwrap();
let mut review = LedgerEntry::new(
"fix456",
Verdict::Reject,
"CLAIM: still bad | verified: cargo test | evidence: tests:cargo-test",
vec!["tests:cargo-test".to_owned()],
ReviewerConfig::new("claude", "claude-opus-4-1", false),
vec!["fix insufficient".to_owned()],
);
review.petition_for = Some("orig123".to_owned());
review.petition_attempts = 1;
run_store
.persist_pending_review(
&queued.run_id,
PendingReviewSpool {
entries: vec![review.clone()],
petition_transition_pending: true,
strict_pass_stdout: None,
},
)
.unwrap();
let report = drain_once(
&queue,
&StaticLoader::new(),
&selection(),
"",
&PanicRunner,
&store,
&TruthMirrorConfig::default(),
)
.unwrap();
assert_eq!(report.reviewed, ["fix456"]);
let orig_history: Vec<_> = store
.read_history()
.unwrap()
.into_iter()
.filter(|entry| entry.commit_sha == "orig123")
.collect();
assert_eq!(orig_history.len(), 3);
let original = store.show("orig123").unwrap();
assert_eq!(original.disposition, Disposition::Open);
assert_eq!(original.petition_attempts, 1);
let provenance = original.transition_provenance.as_ref().unwrap();
assert_eq!(provenance.kind, TransitionProvenanceKind::PetitionReview);
assert_eq!(provenance.fix_sha, "fix456");
assert_eq!(provenance.original_sha, "orig123");
assert_eq!(provenance.attempts, 1);
let queued_replay = queue.enqueue_petition("fix456", "orig123").unwrap();
run_store
.mark_running(&queued_replay.run_id, REVIEW_LEDGER_PHASE)
.unwrap();
run_store
.persist_pending_review(
&queued_replay.run_id,
PendingReviewSpool {
entries: vec![review],
petition_transition_pending: true,
strict_pass_stdout: None,
},
)
.unwrap();
let replay = drain_once(
&queue,
&StaticLoader::new(),
&selection(),
"",
&PanicRunner,
&store,
&TruthMirrorConfig::default(),
)
.unwrap();
assert_eq!(replay.reviewed, ["fix456"]);
let orig_history_after: Vec<_> = store
.read_history()
.unwrap()
.into_iter()
.filter(|entry| entry.commit_sha == "orig123")
.collect();
assert_eq!(orig_history_after.len(), 3);
let original_after = store.show("orig123").unwrap();
assert_eq!(
original_after.transition_provenance,
original.transition_provenance
);
}
#[test]
fn drain_once_replay_applies_pending_strict_second_pass_without_rerunning_provider() {
use crate::ledger::{LedgerEntry, ReviewerConfig, Verdict};
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let queue = ReviewQueue::new(temp.path());
let sha = "abc123";
let run_store = ReviewRunStore::new(temp.path());
let queued = queue.enqueue(sha).unwrap();
run_store
.mark_running(&queued.run_id, REVIEW_LEDGER_PHASE)
.unwrap();
let first = LedgerEntry::new(
sha,
Verdict::Pass,
"CLAIM: first pass | verified: cargo test | evidence: tests:cargo-test",
vec!["tests:cargo-test".to_owned()],
ReviewerConfig::new("claude", "claude-opus-4-1", false),
vec![],
);
let second = LedgerEntry::new(
sha,
Verdict::Pass,
"CLAIM: strict second pass | verified: cargo test | evidence: tests:cargo-test",
vec!["tests:cargo-test".to_owned()],
ReviewerConfig::new("claude", "claude-opus-4-2", false),
vec![],
);
run_store
.persist_pending_review(
&queued.run_id,
PendingReviewSpool {
entries: vec![first, second],
petition_transition_pending: false,
strict_pass_stdout: None,
},
)
.unwrap();
let report = drain_once(
&queue,
&StaticLoader::new(),
&selection(),
"",
&PanicRunner,
&store,
&TruthMirrorConfig::default(),
)
.unwrap();
assert_eq!(report.reviewed, [sha]);
assert_eq!(report.ledger_entries, 2);
let history = store.read_history().unwrap();
assert_eq!(history.len(), 2);
assert!(history[0].claim.contains("first pass"));
assert!(history[1].claim.contains("strict second pass"));
let completed = run_store.read(&queued.run_id).unwrap();
assert_eq!(completed.status, ReviewRunStatus::Completed);
assert!(completed.pending_review.is_none());
assert!(queue.pending().unwrap().is_empty());
}
#[test]
fn drain_once_replay_skips_model_when_ledger_phase_is_durable() {
use crate::ledger::{LedgerEntry, ReviewerConfig, Verdict};
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let queue = ReviewQueue::new(temp.path());
let sha = "abc1234567890123456789012345678901234567890";
let run_store = ReviewRunStore::new(temp.path());
run_store
.ensure_queued("run-replay", sha, "commit", None)
.unwrap();
run_store
.mark_ledger_applied("run-replay", REVIEW_LEDGER_PHASE, 1)
.unwrap();
store
.append_entry(&LedgerEntry::new(
sha,
Verdict::Pass,
"CLAIM: ok | verified: cargo test | evidence: tests:cargo-test",
vec!["tests:cargo-test".to_owned()],
ReviewerConfig::new("claude", "claude-opus-4-1", false),
vec![],
))
.unwrap();
fs::write(
queue.path(),
format!(
"{}\n",
serde_json::to_string(&QueuedReview {
run_id: "run-replay".to_owned(),
commit_sha: sha.to_owned(),
enqueued_at_unix: 1,
petition_for: None,
..Default::default()
})
.unwrap()
),
)
.unwrap();
let loader = StaticLoader::new();
let runner = ConstRunner::new(pass_json());
let before = store.read_history().unwrap().len();
let report = drain_once(
&queue,
&loader,
&selection(),
"",
&runner,
&store,
&TruthMirrorConfig::default(),
)
.unwrap();
assert_eq!(report.reviewed, [sha]);
assert_eq!(store.read_history().unwrap().len(), before);
assert!(queue.pending().unwrap().is_empty());
}
#[test]
fn drain_once_replay_skips_model_when_running_and_verdict_is_durable() {
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let queue = ReviewQueue::new(temp.path());
let sha = "abc1234567890123456789012345678901234567890";
let run_store = ReviewRunStore::new(temp.path());
let _run = run_store
.ensure_queued("run-running", sha, "commit", None)
.unwrap();
run_store
.mark_running("run-running", REVIEW_LEDGER_PHASE)
.unwrap();
run_store
.mark_ledger_applied("run-running", REVIEW_LEDGER_PHASE, 1)
.unwrap();
fs::write(
queue.path(),
format!(
"{}
",
serde_json::to_string(&QueuedReview {
run_id: "run-running".to_owned(),
commit_sha: sha.to_owned(),
enqueued_at_unix: 1,
petition_for: None,
..Default::default()
})
.unwrap()
),
)
.unwrap();
let loader = StaticLoader::new();
let runner = ConstRunner::new(pass_json());
let before = store.read_history().unwrap().len();
let report = drain_once(
&queue,
&loader,
&selection(),
"",
&runner,
&store,
&TruthMirrorConfig::default(),
)
.unwrap();
assert_eq!(report.reviewed, [sha]);
assert_eq!(store.read_history().unwrap().len(), before);
assert!(queue.pending().unwrap().is_empty());
assert!(run_ledger_phase_applied(
&run_store.read("run-running").unwrap(),
REVIEW_LEDGER_PHASE
));
}
#[test]
fn flag_petition_verdict_does_not_resolve_the_original_rejection() {
use crate::ledger::{Disposition, LedgerEntry, ResolutionKind, ReviewerConfig};
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let queue = ReviewQueue::new(temp.path());
let rejected = LedgerEntry::new(
"orig123",
Verdict::Reject,
"CLAIM: original | verified: cargo test | evidence: tests:cargo-test",
vec!["tests:cargo-test".to_owned()],
ReviewerConfig::new("claude", "claude-opus-4-1", false),
vec!["evidence too thin".to_owned()],
);
store.append_entry(&rejected).unwrap();
store
.append_petition_transition(
"orig123",
Disposition::Open,
ResolutionKind::Resolved,
"petition review enqueued: fix=fix456, attempts=1/2",
1,
)
.unwrap();
queue.enqueue_petition("fix456", "orig123").unwrap();
// FLAG is auditable debt on the petition review entry but must not resolve.
let loader = StaticLoader::new();
let runner = ConstRunner::new(flag_json("evidence could be tighter"));
drain_once(
&queue,
&loader,
&selection(),
"",
&runner,
&store,
&TruthMirrorConfig::default(),
)
.unwrap();
assert_eq!(
store.show("orig123").unwrap().disposition,
crate::ledger::Disposition::Open
);
let history = store.read_history().unwrap();
let petition_entry = history
.iter()
.find(|entry| {
entry.petition_for.as_deref() == Some("orig123") && entry.commit_sha == "fix456"
})
.unwrap();
assert_eq!(petition_entry.verdict, Verdict::Flag);
}
#[test]
fn reconcile_stale_runs_reaps_dead_worker_rows_and_counts_live_view() {
// 0.9.2 field failure: harness-killed watchers left orphaned
// `running` rows behind and nothing ever reconciled them.
let temp = tempfile::tempdir().unwrap();
let store = ReviewRunStore::new(temp.path());
// An orphaned running row whose recorded worker pid is dead (a real
// reaped child: `kill -0` provably reports ESRCH for it).
let mut orphan = ReviewRun::queued("run-orphan", "abc123", "commit");
orphan.status = ReviewRunStatus::Running;
orphan.worker_pid = Some(reaped_pid());
store.write(&orphan).unwrap();
// A genuinely live running row (this test process).
let mut live = ReviewRun::queued("run-live", "def456", "commit");
live.mark_running("reviewing");
store.write(&live).unwrap();
// The read-only view counts the stale row without persisting.
assert_eq!(store.stale_running_count().unwrap(), 1);
assert_eq!(
store.read("run-orphan").unwrap().status,
ReviewRunStatus::Running
);
// Reconciliation reaps exactly the orphan, persisted.
assert_eq!(store.reconcile_stale_runs().unwrap(), 1);
let orphan = store.read("run-orphan").unwrap();
assert_eq!(orphan.status, ReviewRunStatus::Failed);
assert!(
orphan
.error
.as_deref()
.is_some_and(|error| error.contains("stale run")),
"stale reason should be recorded, got: {:?}",
orphan.error
);
assert_eq!(
store.read("run-live").unwrap().status,
ReviewRunStatus::Running
);
assert_eq!(store.stale_running_count().unwrap(), 0);
}
#[test]
fn drain_once_skips_petition_whose_target_is_already_resolved() {
// 0.9.2 field crash: a petition enqueued twice (short-sha + full-sha
// rows for one rejection) let the second copy reach the drain after
// the first had resolved it — NoOpenRejection killed the watcher.
// The dead petition must be consumed with a logged skip while the
// rest of the queue keeps draining.
use crate::ledger::{Disposition, LedgerEntry, ResolutionKind, ReviewerConfig};
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let queue = ReviewQueue::new(temp.path());
let rejected = LedgerEntry::new(
"orig123",
Verdict::Reject,
"CLAIM: original | verified: cargo test | evidence: tests:cargo-test",
vec!["tests:cargo-test".to_owned()],
ReviewerConfig::new("claude", "claude-opus-4-1", false),
vec!["evidence too thin".to_owned()],
);
store.append_entry(&rejected).unwrap();
// Resolved up front: the queued petition is now a dead letter.
store
.append_petition_transition(
"orig123",
Disposition::Resolved,
ResolutionKind::Resolved,
"already fixed elsewhere",
1,
)
.unwrap();
queue.enqueue_petition("fix456", "orig123").unwrap();
queue.enqueue("later999").unwrap();
let loader = StaticLoader::new();
let runner = ConstRunner::new(pass_json());
let report = drain_once(
&queue,
&loader,
&selection(),
"",
&runner,
&store,
&TruthMirrorConfig::default(),
)
.unwrap();
// The dead petition was consumed WITHOUT invoking the reviewer; the
// next queued commit still drained.
assert_eq!(report.failed, ["fix456"]);
assert_eq!(report.reviewed, ["later999"]);
assert!(queue.pending().unwrap().is_empty());
// The skipped run records the reason.
let run_store = ReviewRunStore::new(temp.path());
let runs = run_store.list().unwrap();
let skipped = runs.iter().find(|run| run.commit_sha == "fix456").unwrap();
assert_eq!(skipped.status, ReviewRunStatus::Failed);
assert!(
skipped
.error
.as_deref()
.is_some_and(|error| error.contains("no open rejection")),
"skip reason should be recorded, got: {:?}",
skipped.error
);
// No ledger entry exists for the dead petition (reviewer never ran).
assert!(
store
.read_history()
.unwrap()
.iter()
.all(|entry| entry.commit_sha != "fix456")
);
}
#[test]
fn drain_once_survives_petition_target_resolved_mid_review() {
// The race window between the drain's actionability check and the
// transition: the rejection is open when the review starts but gets
// resolved while the reviewer runs (a human waive, or a duplicate
// petition landing first). 0.9.2 propagated NoOpenRejection out of
// the drain and killed the watcher; now it is skip-and-continue.
use crate::ledger::{Disposition, LedgerEntry, ResolutionKind, ReviewerConfig};
struct ResolvingRunner {
state_dir: PathBuf,
}
impl ProcessRunner for ResolvingRunner {
fn run(
&self,
_invocation: &InvocationPlan,
_prompt: &str,
_timeout: Option<Duration>,
) -> Result<ProcessOutput, ReviewerError> {
// Mid-review resolution: the verdict will be recorded, but
// the transition target is gone by the time it lands.
LedgerStore::new(&self.state_dir)
.append_petition_transition(
"orig123",
Disposition::Resolved,
ResolutionKind::Resolved,
"raced resolve while the petition review ran",
1,
)
.unwrap();
Ok(ProcessOutput {
status_code: Some(0),
stdout: pass_json(),
stderr: String::new(),
})
}
}
let temp = tempfile::tempdir().unwrap();
let store = LedgerStore::new(temp.path());
let queue = ReviewQueue::new(temp.path());
let rejected = LedgerEntry::new(
"orig123",
Verdict::Reject,
"CLAIM: original | verified: cargo test | evidence: tests:cargo-test",
vec!["tests:cargo-test".to_owned()],
ReviewerConfig::new("claude", "claude-opus-4-1", false),
vec!["evidence too thin".to_owned()],
);
store.append_entry(&rejected).unwrap();
store
.append_petition_transition(
"orig123",
Disposition::Open,
ResolutionKind::Resolved,
"petition review enqueued: fix=fix456, attempts=1/2",
1,
)
.unwrap();
queue.enqueue_petition("fix456", "orig123").unwrap();
let runner = ResolvingRunner {
state_dir: temp.path().to_path_buf(),
};
let report = drain_once(
&queue,
&StaticLoader::new(),
&selection(),
"",
&runner,
&store,
&TruthMirrorConfig::default(),
)
.unwrap();
// Track A applies the verdict idempotently and no-ops the transition
// when the target was resolved mid-review — the watcher still lives,
// the queue drains, and the original disposition stays Resolved.
assert_eq!(report.reviewed, ["fix456"]);
assert!(report.failed.is_empty());
assert!(queue.pending().unwrap().is_empty());
assert_eq!(
store.show("orig123").unwrap().disposition,
Disposition::Resolved
);
}
}