car-server-core 0.55.0

//! The review gate for an unattended fix: N independent verdicts, a threshold.
//!
//! T4 of `docs/proposals/self-healing-issue-loop.md`. Pure — the verdicts are
//! an argument. Fetching them costs inference and a network; deciding what they
//! mean must be reproducible and reviewable, so the two are separated.
//!
//! ## This is the SECOND gate, never the first
//!
//! The first gate is the [`super::contract::OutcomeContract`], which the
//! runtime re-runs itself before asking for merge approval regardless of which
//! engine did the work. That is deterministic and non-negotiable: it answers
//! "does this actually work". No count of model verdicts substitutes for it or
//! overrides it, and [`decide`] takes the contract result as an input it can
//! only ever *narrow*.
//!
//! The distinction is not theoretical. In the peer-messaging work that
//! motivated this loop, a change shipped that made every *successful*
//! cross-host delivery report as a failure — and every model-written test
//! passed, because they all asserted inside the dispatcher and never
//! round-tripped the real client. A deterministic check caught it in one run.
//! Conversely, model review caught a change that compiled, passed every test,
//! and would have silently refused every fleet dispatch. Neither gate subsumes
//! the other.
//!
//! ## Independent verdicts, never synthesis
//!
//! CAR has a measured result that constrains this: the MoA / tau-bench
//! experiment found **cross-vendor synthesis degrades agentic tool-use below
//! the single strong model**. That was about *combining answers*, and
//! verification is a different operation, so the result does not forbid a panel
//! — but it is the reason this fans out to N verdicts and applies a threshold,
//! rather than merging reviewers' reasoning into one answer.
//!
//! Reviewers therefore never see each other's verdicts. A panel that reaches
//! consensus by reading itself is one reviewer with extra steps, and the
//! failure it is meant to catch — everyone missing the same thing — is exactly
//! the one correlation reintroduces.
//!
//! ## Unreachable is not approval
//!
//! A vendor that times out yields no verdict, and a missing verdict is never a
//! pass. Degrading to "2 of the 2 that answered" would let an outage silently
//! halve the panel, which is the review equivalent of turning the gate off on
//! the day it is most needed.

/// Proof that a [`GateOutcome`] came from [`decide`].
///
/// A private unit field is the whole mechanism: no other module can name it, so
/// no other module can build an outcome.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Sealed(());

/// One reviewer's answer.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Verdict {
    /// Which model produced it. Recorded so a panel that disagrees can be
    /// audited by vendor rather than by anonymous count.
    pub model: String,
    pub pass: bool,
    /// The reviewer's stated reason. Shown to humans, never parsed.
    pub reason: String,
}

/// A reviewer that could not be reached.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Unreachable {
    pub model: String,
    pub error: String,
}

/// What the panel found, and what follows from it.
///
/// Every variant carries [`Sealed`], which only this module can construct, so
/// [`decide`] is the **only** way to obtain a `GateOutcome`. Without that, any
/// caller could hand back `Approved` having run no contract and asked no
/// reviewer — and a gate that a caller can mint is not a gate. The first draft
/// of this had exactly that hole: the tick's injected `run_coder` returned a
/// `GateOutcome` directly and `decide` had no callers at all.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum GateOutcome {
    /// Deterministic checks passed and the panel cleared it.
    Approved {
        passes: usize,
        of: usize,
        seal: Sealed,
    },
    /// The deterministic gate failed. The panel is not consulted at all: a
    /// change that does not build is not a question for reviewers.
    ContractFailed { detail: String, seal: Sealed },
    /// Reviewers were reached but did not clear it.
    PanelRejected {
        passes: usize,
        required: usize,
        dissent: Vec<Verdict>,
        seal: Sealed,
    },
    /// Too few reviewers answered to reach a threshold at all.
    PanelIncomplete {
        answered: usize,
        required: usize,
        unreachable: Vec<Unreachable>,
        seal: Sealed,
    },
}

impl GateOutcome {
    pub fn approved(&self) -> bool {
        matches!(self, GateOutcome::Approved { .. })
    }

    /// One line an operator can read without opening the run.
    pub fn summary(&self) -> String {
        match self {
            GateOutcome::Approved { passes, of, .. } => {
                format!("approved by {passes}/{of} reviewers, checks green")
            }
            GateOutcome::ContractFailed { detail, .. } => {
                format!("outcome contract failed: {detail}")
            }
            GateOutcome::PanelRejected {
                passes,
                required,
                dissent,
                ..
            } => {
                let why = dissent
                    .iter()
                    .map(|v| format!("{}: {}", v.model, v.reason))
                    .collect::<Vec<_>>()
                    .join("; ");
                format!("{passes}/{required} required approvals — {why}")
            }
            GateOutcome::PanelIncomplete {
                answered,
                required,
                unreachable,
                ..
            } => {
                let who = unreachable
                    .iter()
                    .map(|u| format!("{} ({})", u.model, u.error))
                    .collect::<Vec<_>>()
                    .join("; ");
                format!(
                    "only {answered} reviewers answered, {required} approvals required — \
                     unreachable: {who}"
                )
            }
        }
    }
}

/// How many approvals a panel of `panel_size` needs.
///
/// A strict majority, so a panel can never be cleared by a minority and a
/// single compromised or malfunctioning reviewer cannot approve alone. For the
/// default panel of three this is two.
pub fn required_approvals(panel_size: usize) -> usize {
    panel_size / 2 + 1
}

/// Decide, from the deterministic result and the reviewers' answers.
///
/// `panel_size` is how many reviewers were *asked*, not how many replied — the
/// threshold is fixed before the vote, so an outage cannot lower the bar it has
/// to clear.
pub fn decide(
    contract_passed: bool,
    contract_detail: &str,
    panel_size: usize,
    verdicts: &[Verdict],
    unreachable: &[Unreachable],
) -> GateOutcome {
    // Deterministic first, and reviewers are not consulted on a red build.
    // Asking a model to bless code that does not compile invites it to explain
    // why the failure is acceptable, which is a conversation with only one bad
    // outcome.
    if !contract_passed {
        return GateOutcome::ContractFailed {
            detail: contract_detail.to_string(),
            seal: Sealed(()),
        };
    }

    if panel_size == 0 {
        return GateOutcome::ContractFailed {
            detail: "no reviewers were configured; a panel of zero cannot approve".into(),
            seal: Sealed(()),
        };
    }

    // ONE answer per model. `Verdict::model` is recorded so a panel can be
    // audited by vendor rather than by anonymous count — and then counting
    // anonymously would let one vendor, retried after a timeout and appended
    // twice, supply a majority by itself. First answer per model wins; a
    // retry does not get a second vote.
    let mut seen: Vec<&str> = Vec::new();
    let mut answers: Vec<Verdict> = Vec::new();
    for v in verdicts {
        if seen.contains(&v.model.as_str()) {
            continue;
        }
        seen.push(&v.model);
        answers.push(v.clone());
    }
    // More distinct answers than reviewers asked means the caller is confused
    // about its own panel; refusing is safer than picking an interpretation.
    if answers.len() > panel_size {
        return GateOutcome::ContractFailed {
            detail: format!(
                "{} distinct verdicts for a panel of {panel_size}",
                answers.len()
            ),
            seal: Sealed(()),
        };
    }

    let required = required_approvals(panel_size);

    // A missing verdict is not a pass. If too few answered to reach the
    // threshold even unanimously, that is incomplete rather than rejected —
    // the distinction matters because the remedy differs: retry the panel
    // versus fix the change.
    if answers.len() < required {
        return GateOutcome::PanelIncomplete {
            answered: answers.len(),
            required,
            unreachable: unreachable.to_vec(),
            seal: Sealed(()),
        };
    }

    let passes = answers.iter().filter(|v| v.pass).count();
    if passes >= required {
        return GateOutcome::Approved {
            passes,
            of: panel_size,
            seal: Sealed(()),
        };
    }

    GateOutcome::PanelRejected {
        passes,
        required,
        dissent: answers.iter().filter(|v| !v.pass).cloned().collect(),
        seal: Sealed(()),
    }
}

/// The criteria one reviewer is asked to judge against.
///
/// Written once, here, so every reviewer in a panel is asked the *same*
/// question — a panel whose members were prompted differently is measuring
/// prompt variance, not agreement.
///
/// It deliberately does not include the issue body verbatim. The body is
/// untrusted text (see [`super::provenance`]), and a reviewer prompt is exactly
/// the place where "ignore the above" would land. Reviewers judge the DIFF
/// against the stated intent and the deterministic results.
pub fn review_criteria(intent_summary: &str, contract_detail: &str) -> String {
    format!(
        "You are reviewing an automated code change before it is opened as a pull request.\n\
         \n\
         Stated intent: {intent_summary}\n\
         Deterministic checks: {contract_detail}\n\
         \n\
         The build and the project's own acceptance checks have ALREADY passed. Do not \
         re-litigate them. Judge only what they cannot:\n\
         - Does the change do what the intent says, or something adjacent to it?\n\
         - Does it introduce a defect the checks would not catch?\n\
         - Is it scoped to the intent, or does it change unrelated behaviour?\n\
         \n\
         Answer PASS or FAIL and one sentence of reason. Default to FAIL if you are \
         uncertain: a rejected change costs one retry, an approved bad one costs a human's \
         trust in every later change."
    )
}

#[cfg(test)]
mod tests {
    use super::*;

    fn v(model: &str, pass: bool) -> Verdict {
        Verdict {
            model: model.into(),
            pass,
            reason: if pass { "looks right" } else { "wrong scope" }.into(),
        }
    }

    #[test]
    fn a_red_contract_is_never_put_to_the_panel() {
        // Even with a unanimous panel: a change that does not build is not a
        // question reviewers should be asked.
        let out = decide(
            false,
            "cargo test failed",
            3,
            &[v("a", true), v("b", true), v("c", true)],
            &[],
        );
        assert!(matches!(out, GateOutcome::ContractFailed { .. }));
        assert!(!out.approved());
    }

    #[test]
    fn a_strict_majority_approves() {
        let out = decide(
            true,
            "green",
            3,
            &[v("a", true), v("b", true), v("c", false)],
            &[],
        );
        assert!(matches!(
            out,
            GateOutcome::Approved {
                passes: 2,
                of: 3,
                ..
            }
        ));
    }

    #[test]
    fn a_minority_does_not_approve() {
        let out = decide(
            true,
            "green",
            3,
            &[v("a", true), v("b", false), v("c", false)],
            &[],
        );
        match out {
            GateOutcome::PanelRejected {
                passes,
                required,
                dissent,
                ..
            } => {
                assert_eq!((passes, required), (1, 2));
                assert_eq!(dissent.len(), 2, "dissent is recorded, not just counted");
            }
            other => panic!("expected rejection, got {other:?}"),
        }
    }

    #[test]
    fn an_unreachable_vendor_cannot_halve_the_panel() {
        // The threshold is fixed before the vote. Two of three answering
        // unanimously is NOT two of two.
        let out = decide(
            true,
            "green",
            3,
            &[v("a", true)],
            &[Unreachable {
                model: "b".into(),
                error: "timeout".into(),
            }],
        );
        match out {
            GateOutcome::PanelIncomplete {
                answered, required, ..
            } => assert_eq!((answered, required), (1, 2)),
            other => panic!("expected incomplete, got {other:?}"),
        }
    }

    #[test]
    fn exactly_enough_answers_can_still_approve() {
        let out = decide(true, "green", 3, &[v("a", true), v("b", true)], &[]);
        assert!(matches!(
            out,
            GateOutcome::Approved {
                passes: 2,
                of: 3,
                ..
            }
        ));
    }

    #[test]
    fn exactly_enough_answers_can_also_reject() {
        let out = decide(true, "green", 3, &[v("a", true), v("b", false)], &[]);
        assert!(matches!(out, GateOutcome::PanelRejected { .. }));
    }

    #[test]
    fn a_single_reviewer_cannot_approve_alone_on_a_panel_of_three() {
        assert_eq!(required_approvals(3), 2);
        assert_eq!(required_approvals(5), 3);
        // A panel of one is a strict majority of itself — degenerate, and the
        // caller's problem to avoid, but the arithmetic must not surprise.
        assert_eq!(required_approvals(1), 1);
    }

    #[test]
    fn incomplete_and_rejected_are_different_outcomes() {
        // The remedy differs: retry the panel versus fix the change. Collapsing
        // them would send an operator to debug code when a vendor was down.
        let incomplete = decide(true, "green", 3, &[], &[]);
        let rejected = decide(true, "green", 3, &[v("a", false), v("b", false)], &[]);
        assert!(matches!(incomplete, GateOutcome::PanelIncomplete { .. }));
        assert!(matches!(rejected, GateOutcome::PanelRejected { .. }));
    }

    #[test]
    fn the_summary_names_the_dissenters_and_the_unreachable() {
        let rejected = decide(true, "green", 3, &[v("a", true), v("b", false)], &[]);
        assert!(rejected.summary().contains('b'), "{}", rejected.summary());

        let incomplete = decide(
            true,
            "green",
            3,
            &[],
            &[Unreachable {
                model: "gpt".into(),
                error: "429".into(),
            }],
        );
        assert!(incomplete.summary().contains("gpt"));
        assert!(incomplete.summary().contains("429"));
    }

    #[test]
    fn the_criteria_do_not_carry_the_issue_body() {
        // A reviewer prompt is exactly where "ignore the above" would land, so
        // untrusted tracker text must not reach it.
        let c = review_criteria("fix the off-by-one in the parser", "all checks green");
        assert!(c.contains("off-by-one"));
        assert!(
            c.contains("Default to FAIL"),
            "uncertainty must not read as approval"
        );
    }

    #[test]
    fn every_reviewer_is_asked_the_same_question() {
        let a = review_criteria("intent", "green");
        let b = review_criteria("intent", "green");
        assert_eq!(
            a, b,
            "a panel prompted differently measures prompt variance"
        );
    }
}