use crate::{CheckRecord, EvidenceTier};
use serde::{Deserialize, Serialize};
use std::collections::BTreeSet;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum VerifierAuthority {
Advisory,
Binding,
Operator,
}
impl VerifierAuthority {
pub const fn as_str(&self) -> &'static str {
match self {
VerifierAuthority::Advisory => "advisory",
VerifierAuthority::Binding => "binding",
VerifierAuthority::Operator => "operator",
}
}
pub const fn can_satisfy(&self) -> bool {
match self {
VerifierAuthority::Advisory => false,
VerifierAuthority::Binding | VerifierAuthority::Operator => true,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum VerifierCost {
Free,
Cheap,
Expensive,
Human,
}
impl VerifierCost {
pub const fn as_str(&self) -> &'static str {
match self {
VerifierCost::Free => "free",
VerifierCost::Cheap => "cheap",
VerifierCost::Expensive => "expensive",
VerifierCost::Human => "human",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct VerifierDescriptor {
pub id: String,
pub class: String,
pub tier: EvidenceTier,
pub authority: VerifierAuthority,
pub cost: VerifierCost,
pub deterministic: bool,
}
impl VerifierDescriptor {
pub fn binding(id: impl Into<String>, class: impl Into<String>) -> Self {
Self {
id: id.into(),
class: class.into(),
tier: EvidenceTier::DecisionProcedure,
authority: VerifierAuthority::Binding,
cost: VerifierCost::Free,
deterministic: true,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum VerifierOutcome {
Pass,
Fail,
Inconclusive,
Skipped,
}
impl VerifierOutcome {
pub const fn as_str(&self) -> &'static str {
match self {
VerifierOutcome::Pass => "pass",
VerifierOutcome::Fail => "fail",
VerifierOutcome::Inconclusive => "inconclusive",
VerifierOutcome::Skipped => "skipped",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct VerifierVerdict {
pub verifier: VerifierDescriptor,
pub outcome: VerifierOutcome,
#[serde(default)]
pub detail: String,
#[serde(default)]
pub verifies: String,
#[serde(default)]
pub cannot_verify: String,
#[serde(default)]
pub artifacts: Vec<String>,
}
impl VerifierVerdict {
pub fn pass(verifier: VerifierDescriptor) -> Self {
Self {
verifier,
outcome: VerifierOutcome::Pass,
detail: String::new(),
verifies: String::new(),
cannot_verify: String::new(),
artifacts: Vec::new(),
}
}
pub fn fail(verifier: VerifierDescriptor, detail: impl Into<String>) -> Self {
Self {
verifier,
outcome: VerifierOutcome::Fail,
detail: detail.into(),
verifies: String::new(),
cannot_verify: String::new(),
artifacts: Vec::new(),
}
}
pub fn to_check_record(&self) -> CheckRecord {
CheckRecord {
name: self.verifier.id.clone(),
ran: self.outcome != VerifierOutcome::Skipped,
verifies: self.verifies.clone(),
cannot_verify: self.cannot_verify.clone(),
findings: usize::from(self.outcome == VerifierOutcome::Fail),
tier: self.verifier.tier,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct EvidenceRequirement {
pub class: String,
#[serde(default)]
pub accepts_tiers: Vec<EvidenceTier>,
#[serde(default)]
pub operator_override: bool,
}
impl EvidenceRequirement {
pub fn new(class: impl Into<String>) -> Self {
Self {
class: class.into(),
accepts_tiers: Vec::new(),
operator_override: false,
}
}
pub fn accepting(mut self, tiers: impl IntoIterator<Item = EvidenceTier>) -> Self {
self.accepts_tiers = tiers.into_iter().collect();
self
}
pub fn with_operator_override(mut self) -> Self {
self.operator_override = true;
self
}
fn accepts(&self, tier: EvidenceTier) -> bool {
self.accepts_tiers.is_empty() || self.accepts_tiers.contains(&tier)
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub enum UnmetReason {
NoVerdict,
NotSatisfying {
detail: String,
},
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct UnmetRequirement {
pub class: String,
pub reason: UnmetReason,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum AdmissionOutcome {
Admit,
Reject,
NeedsMoreEvidence,
}
impl AdmissionOutcome {
pub const fn as_str(&self) -> &'static str {
match self {
AdmissionOutcome::Admit => "admit",
AdmissionOutcome::Reject => "reject",
AdmissionOutcome::NeedsMoreEvidence => "needs_more_evidence",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct AdmissionDecision {
pub outcome: AdmissionOutcome,
pub unmet: Vec<UnmetRequirement>,
pub failures: Vec<String>,
pub advisories: Vec<String>,
pub records: Vec<CheckRecord>,
}
pub fn admit(
requirements: &[EvidenceRequirement],
verdicts: &[VerifierVerdict],
) -> AdmissionDecision {
let records: Vec<CheckRecord> = verdicts.iter().map(|v| v.to_check_record()).collect();
let mut failures = Vec::new();
let mut advisories = Vec::new();
for v in verdicts {
if v.outcome != VerifierOutcome::Fail {
continue;
}
if v.verifier.authority.can_satisfy() {
failures.push(v.verifier.id.clone());
} else {
advisories.push(v.verifier.id.clone());
}
}
if !failures.is_empty() {
return AdmissionDecision {
outcome: AdmissionOutcome::Reject,
unmet: Vec::new(),
failures,
advisories,
records,
};
}
let mut unmet = Vec::new();
for req in requirements {
if requirement_met(req, verdicts) {
continue;
}
unmet.push(UnmetRequirement {
class: req.class.clone(),
reason: unmet_reason(req, verdicts),
});
}
let outcome = if unmet.is_empty() {
AdmissionOutcome::Admit
} else {
AdmissionOutcome::NeedsMoreEvidence
};
AdmissionDecision {
outcome,
unmet,
failures,
advisories,
records,
}
}
fn requirement_met(req: &EvidenceRequirement, verdicts: &[VerifierVerdict]) -> bool {
verdicts.iter().any(|v| {
if v.outcome != VerifierOutcome::Pass || !req.accepts(v.verifier.tier) {
return false;
}
let class_match = v.verifier.class == req.class && v.verifier.authority.can_satisfy();
let operator_stand_in =
req.operator_override && v.verifier.authority == VerifierAuthority::Operator;
class_match || operator_stand_in
})
}
fn unmet_reason(req: &EvidenceRequirement, verdicts: &[VerifierVerdict]) -> UnmetReason {
let near: Vec<&VerifierVerdict> = verdicts
.iter()
.filter(|v| v.verifier.class == req.class)
.collect();
if near.is_empty() {
return UnmetReason::NoVerdict;
}
let detail = near
.iter()
.map(|v| {
let why = if v.outcome != VerifierOutcome::Pass {
format!("outcome {}", v.outcome.as_str())
} else if !v.verifier.authority.can_satisfy() {
format!("authority {} cannot satisfy", v.verifier.authority.as_str())
} else {
format!("tier {} not accepted", v.verifier.tier.as_str())
};
format!("{} ({})", v.verifier.id, why)
})
.collect::<Vec<_>>()
.join("; ");
UnmetReason::NotSatisfying { detail }
}
pub fn required_classes(requirements: &[EvidenceRequirement]) -> Vec<String> {
requirements
.iter()
.map(|r| r.class.clone())
.collect::<BTreeSet<_>>()
.into_iter()
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
fn tests_verifier() -> VerifierDescriptor {
VerifierDescriptor {
id: "cargo_test".into(),
class: "tests".into(),
tier: EvidenceTier::DecisionProcedure,
authority: VerifierAuthority::Binding,
cost: VerifierCost::Expensive,
deterministic: true,
}
}
fn judge_verifier() -> VerifierDescriptor {
VerifierDescriptor {
id: "model_judge".into(),
class: "tests".into(),
tier: EvidenceTier::Heuristic,
authority: VerifierAuthority::Advisory,
cost: VerifierCost::Expensive,
deterministic: false,
}
}
fn operator() -> VerifierDescriptor {
VerifierDescriptor {
id: "operator_console".into(),
class: "human_review".into(),
tier: EvidenceTier::DecisionProcedure,
authority: VerifierAuthority::Operator,
cost: VerifierCost::Human,
deterministic: false,
}
}
#[test]
fn no_requirements_and_no_failures_admits() {
let d = admit(&[], &[]);
assert_eq!(d.outcome, AdmissionOutcome::Admit);
assert!(d.unmet.is_empty());
assert!(d.records.is_empty());
}
#[test]
fn missing_verdict_needs_more_evidence_not_admit() {
let d = admit(&[EvidenceRequirement::new("tests")], &[]);
assert_eq!(d.outcome, AdmissionOutcome::NeedsMoreEvidence);
assert_eq!(d.unmet.len(), 1);
assert_eq!(d.unmet[0].class, "tests");
assert_eq!(d.unmet[0].reason, UnmetReason::NoVerdict);
}
#[test]
fn binding_pass_satisfies_its_class() {
let d = admit(
&[EvidenceRequirement::new("tests")],
&[VerifierVerdict::pass(tests_verifier())],
);
assert_eq!(d.outcome, AdmissionOutcome::Admit);
}
#[test]
fn binding_failure_rejects() {
let d = admit(
&[],
&[VerifierVerdict::fail(tests_verifier(), "3 tests failed")],
);
assert_eq!(d.outcome, AdmissionOutcome::Reject);
assert_eq!(d.failures, vec!["cargo_test".to_string()]);
}
#[test]
fn rejection_wins_over_unmet_requirements() {
let d = admit(
&[EvidenceRequirement::new("types")],
&[VerifierVerdict::fail(tests_verifier(), "boom")],
);
assert_eq!(d.outcome, AdmissionOutcome::Reject);
assert!(d.unmet.is_empty());
}
#[test]
fn advisory_failure_records_but_does_not_reject() {
let d = admit(
&[],
&[VerifierVerdict::fail(judge_verifier(), "looks wrong to me")],
);
assert_eq!(d.outcome, AdmissionOutcome::Admit);
assert!(d.failures.is_empty());
assert_eq!(d.advisories, vec!["model_judge".to_string()]);
}
#[test]
fn advisory_pass_cannot_satisfy_a_requirement() {
let d = admit(
&[EvidenceRequirement::new("tests")],
&[VerifierVerdict::pass(judge_verifier())],
);
assert_eq!(d.outcome, AdmissionOutcome::NeedsMoreEvidence);
match &d.unmet[0].reason {
UnmetReason::NotSatisfying { detail } => {
assert!(detail.contains("model_judge"), "{detail}");
assert!(detail.contains("cannot satisfy"), "{detail}");
}
other => panic!("expected NotSatisfying, got {other:?}"),
}
}
#[test]
fn tier_restriction_excludes_a_same_class_pass() {
let mut heuristic_but_binding = judge_verifier();
heuristic_but_binding.authority = VerifierAuthority::Binding;
let req = EvidenceRequirement::new("tests").accepting([EvidenceTier::DecisionProcedure]);
let d = admit(&[req], &[VerifierVerdict::pass(heuristic_but_binding)]);
assert_eq!(d.outcome, AdmissionOutcome::NeedsMoreEvidence);
match &d.unmet[0].reason {
UnmetReason::NotSatisfying { detail } => {
assert!(detail.contains("tier heuristic not accepted"), "{detail}")
}
other => panic!("expected NotSatisfying, got {other:?}"),
}
}
#[test]
fn operator_approval_does_not_silently_satisfy_a_test_requirement() {
let d = admit(
&[EvidenceRequirement::new("tests")],
&[VerifierVerdict::pass(operator())],
);
assert_eq!(d.outcome, AdmissionOutcome::NeedsMoreEvidence);
assert_eq!(d.unmet[0].reason, UnmetReason::NoVerdict);
}
#[test]
fn operator_override_is_opt_in_and_then_works() {
let req = EvidenceRequirement::new("tests").with_operator_override();
let d = admit(&[req], &[VerifierVerdict::pass(operator())]);
assert_eq!(d.outcome, AdmissionOutcome::Admit);
}
#[test]
fn operator_override_still_respects_the_tier_filter() {
let mut sampled_operator = operator();
sampled_operator.tier = EvidenceTier::Sampled;
let req = EvidenceRequirement::new("tests")
.with_operator_override()
.accepting([EvidenceTier::DecisionProcedure]);
let d = admit(&[req], &[VerifierVerdict::pass(sampled_operator)]);
assert_eq!(d.outcome, AdmissionOutcome::NeedsMoreEvidence);
}
#[test]
fn operator_failure_rejects() {
let d = admit(&[], &[VerifierVerdict::fail(operator(), "not authorized")]);
assert_eq!(d.outcome, AdmissionOutcome::Reject);
assert_eq!(d.failures, vec!["operator_console".to_string()]);
}
#[test]
fn inconclusive_and_skipped_never_satisfy() {
for outcome in [VerifierOutcome::Inconclusive, VerifierOutcome::Skipped] {
let mut v = VerifierVerdict::pass(tests_verifier());
v.outcome = outcome;
let d = admit(&[EvidenceRequirement::new("tests")], &[v]);
assert_eq!(
d.outcome,
AdmissionOutcome::NeedsMoreEvidence,
"{} satisfied a requirement",
outcome.as_str()
);
}
}
#[test]
fn requirements_are_conjunctive() {
let reqs = [
EvidenceRequirement::new("tests"),
EvidenceRequirement::new("types"),
];
let d = admit(&reqs, &[VerifierVerdict::pass(tests_verifier())]);
assert_eq!(d.outcome, AdmissionOutcome::NeedsMoreEvidence);
assert_eq!(d.unmet.len(), 1);
assert_eq!(d.unmet[0].class, "types");
let types = VerifierDescriptor::binding("tsc", "types");
let d = admit(
&reqs,
&[
VerifierVerdict::pass(tests_verifier()),
VerifierVerdict::pass(types),
],
);
assert_eq!(d.outcome, AdmissionOutcome::Admit);
}
#[test]
fn verdicts_become_check_records_carrying_their_scope() {
let mut v = VerifierVerdict::pass(tests_verifier());
v.verifies = "the suite passes on this revision".into();
v.cannot_verify = "behaviour on inputs the suite does not cover".into();
let d = admit(&[], &[v]);
assert_eq!(d.records.len(), 1);
let r = &d.records[0];
assert_eq!(r.name, "cargo_test");
assert!(r.ran);
assert_eq!(r.findings, 0);
assert_eq!(r.tier, EvidenceTier::DecisionProcedure);
assert_eq!(
r.cannot_verify,
"behaviour on inputs the suite does not cover"
);
}
#[test]
fn skipped_verdict_records_as_not_run_and_a_failure_counts_one_finding() {
let mut skipped = VerifierVerdict::pass(tests_verifier());
skipped.outcome = VerifierOutcome::Skipped;
assert!(!skipped.to_check_record().ran);
assert_eq!(skipped.to_check_record().findings, 0);
let failed = VerifierVerdict::fail(tests_verifier(), "boom");
assert!(failed.to_check_record().ran);
assert_eq!(failed.to_check_record().findings, 1);
}
#[test]
fn records_are_emitted_even_on_rejection() {
let d = admit(
&[],
&[
VerifierVerdict::fail(tests_verifier(), "boom"),
VerifierVerdict::pass(VerifierDescriptor::binding("tsc", "types")),
],
);
assert_eq!(d.outcome, AdmissionOutcome::Reject);
assert_eq!(d.records.len(), 2);
}
#[test]
fn required_classes_dedupes_and_sorts() {
let reqs = [
EvidenceRequirement::new("types"),
EvidenceRequirement::new("tests"),
EvidenceRequirement::new("types"),
];
assert_eq!(required_classes(&reqs), vec!["tests", "types"]);
}
#[test]
fn admit_is_deterministic_and_order_stable() {
let reqs = [EvidenceRequirement::new("tests")];
let verdicts = [
VerifierVerdict::fail(judge_verifier(), "a"),
VerifierVerdict::pass(tests_verifier()),
];
let first = admit(&reqs, &verdicts);
let second = admit(&reqs, &verdicts);
assert_eq!(first, second);
assert_eq!(first.outcome, AdmissionOutcome::Admit);
assert_eq!(first.advisories, vec!["model_judge".to_string()]);
}
#[test]
fn labels_match_serde_representation() {
assert_eq!(
serde_json::to_value(VerifierAuthority::Operator).unwrap(),
serde_json::json!(VerifierAuthority::Operator.as_str())
);
assert_eq!(
serde_json::to_value(VerifierCost::Human).unwrap(),
serde_json::json!(VerifierCost::Human.as_str())
);
assert_eq!(
serde_json::to_value(VerifierOutcome::Inconclusive).unwrap(),
serde_json::json!(VerifierOutcome::Inconclusive.as_str())
);
assert_eq!(
serde_json::to_value(AdmissionOutcome::NeedsMoreEvidence).unwrap(),
serde_json::json!(AdmissionOutcome::NeedsMoreEvidence.as_str())
);
}
}