use crate::plan::same_inputs;
use crate::{
AuditReceipt, AuditVerdict, ContractError, Evaluator, EvidenceCategory, ExecutionReceipt,
Expectation, FamilyCatalogue, ManualObservation, ProcessStatus, QualityPlan, Subject,
VerifiedArtifacts, artifact_shape, identifier, nonempty,
};
use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, BTreeSet};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum CheckOutcome {
Passed,
Blocked,
Skipped,
Inconclusive,
Error,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum GateOutcome {
Passed,
Blocked,
Incomplete,
Error,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct CheckReport {
pub id: String,
pub required: bool,
pub attempt_id: Option<String>,
pub outcome: CheckOutcome,
pub reasons: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct ExpectationReport {
pub id: String,
pub required: bool,
pub outcome: CheckOutcome,
pub reasons: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct GateReport {
pub outcome: GateOutcome,
pub required_checks: usize,
pub passed_required_checks: usize,
pub checks: Vec<CheckReport>,
pub expectations: Vec<ExpectationReport>,
pub errors: Vec<String>,
}
impl GateReport {
pub fn passed(&self) -> bool {
self.outcome == GateOutcome::Passed
}
}
#[derive(Debug)]
pub struct EvaluationContext<'a> {
pub now_ms: u64,
pub current_subject: &'a Subject,
pub verified: &'a VerifiedArtifacts,
pub ancestor_revisions: &'a [String],
pub families: FamilyCatalogue,
pub audits: &'a [AuditReceipt],
pub manual_observations: &'a [ManualObservation],
}
impl<'a> EvaluationContext<'a> {
pub fn new(now_ms: u64, current_subject: &'a Subject, verified: &'a VerifiedArtifacts) -> Self {
Self {
now_ms,
current_subject,
verified,
ancestor_revisions: &[],
families: FamilyCatalogue::default(),
audits: &[],
manual_observations: &[],
}
}
}
fn context_limits(plan: &QualityPlan, context: &EvaluationContext<'_>) -> Vec<String> {
let mut reasons = Vec::new();
let recorded = &plan.subject;
let current = context.current_subject;
if recorded.repository != current.repository || recorded.configuration != current.configuration
{
reasons.push("current repository/configuration differs from reviewed subject".into());
}
if !same_inputs(&recorded.inputs, ¤t.inputs) {
reasons.push("selected source inputs changed since review".into());
}
if recorded.revision != current.revision
&& !context.ancestor_revisions.contains(&recorded.revision)
{
reasons.push(
"reviewed revision ancestry is unknown or no longer on the current branch".into(),
);
}
reasons
}
fn age_limit(at: u64, plan: &QualityPlan, context: &EvaluationContext<'_>) -> Option<String> {
match context.now_ms.checked_sub(at) {
None => Some("observation timestamp is in the future".into()),
Some(age) if plan.policy.max_age_ms.is_some_and(|max| age > max) => {
Some("observation exceeds configured maximum age".into())
}
_ => None,
}
}
fn check_receipt(
plan: &QualityPlan,
digest: &str,
check: &crate::CheckSpec,
receipt: &ExecutionReceipt,
context: &EvaluationContext<'_>,
) -> (CheckOutcome, Vec<String>) {
let error = |message: &str| (CheckOutcome::Error, vec![message.to_owned()]);
if let Err(e) = receipt.validate() {
return error(&e.to_string());
}
if receipt.plan_sha256 != digest
|| receipt.subject != plan.subject
|| receipt.invocation != check.invocation
{
return error("receipt plan, subject or invocation does not match the selected check");
}
if !same_inputs(&receipt.subject.inputs, &receipt.inputs_after) {
return (
CheckOutcome::Inconclusive,
vec!["selected inputs changed during execution".into()],
);
}
let mut limits = context_limits(plan, context);
if let Some(age) = age_limit(receipt.finished_at_ms, plan, context) {
limits.push(age);
}
if !limits.is_empty() {
return (CheckOutcome::Inconclusive, limits);
}
if receipt.process != ProcessStatus::Completed {
let outcome = if receipt.process == ProcessStatus::Unavailable {
CheckOutcome::Skipped
} else {
CheckOutcome::Error
};
return (
outcome,
vec![
receipt
.diagnostic
.clone()
.unwrap_or_else(|| "execution did not complete".into()),
],
);
}
if receipt.elapsed_ms > check.invocation.timeout_ms {
return error("completed execution exceeded the configured deadline");
}
let mut output_bytes = 0u64;
for output in receipt.stdout.iter().chain(receipt.stderr.iter()) {
if let Some(bytes) = context.verified.byte_len(output) {
let Some(total) = output_bytes.checked_add(bytes) else {
return error("output length overflow");
};
output_bytes = total;
} else {
return error("raw output artifact has not been byte-verified");
}
}
if output_bytes > check.invocation.max_output_bytes {
return error("verified raw output exceeds configured capture limit");
}
if let Some(snapshot) = &receipt.subject.working_tree
&& !context.verified.contains(snapshot)
{
return error("dirty working-tree snapshot has not been byte-verified");
}
match &receipt.observed_scope {
Some(paths) if !check.scope.iter().all(|p| paths.contains(p)) => {
return (
CheckOutcome::Inconclusive,
vec!["observed scope omits selected source inputs".into()],
);
}
None if plan.policy.require_observed_scope => {
return (
CheckOutcome::Inconclusive,
vec!["observed source scope is unknown".into()],
);
}
_ => {}
}
match &check.evaluator {
Evaluator::ExitCode { success_codes } => {
if receipt.reported.is_some() {
return error("unexpected adapter assessment for an exit-code evaluator");
}
if receipt
.exit_code
.is_some_and(|code| success_codes.contains(&code))
{
(
CheckOutcome::Passed,
vec!["completed exit code satisfies the configured policy".into()],
)
} else {
(
CheckOutcome::Blocked,
vec!["completed exit code does not satisfy the configured policy".into()],
)
}
}
Evaluator::Reported { adapter } => {
let Some(result) = &receipt.reported else {
return error("missing required adapter assessment");
};
if &result.adapter != adapter || !context.verified.contains(&result.artifact) {
return error("adapter identity or verified report artifact is missing");
}
(result.outcome, vec![result.reason.clone()])
}
}
}
fn referenced_check(
id: &str,
checks: &[CheckReport],
reasons: &mut Vec<String>,
outcome: &mut CheckOutcome,
) {
if let Some(check) = checks.iter().find(|c| c.id == id) {
if check.outcome != CheckOutcome::Passed {
reasons.push(format!("referenced check {id} did not pass"));
if check.outcome == CheckOutcome::Blocked {
*outcome = CheckOutcome::Blocked;
} else if *outcome != CheckOutcome::Blocked {
*outcome = CheckOutcome::Inconclusive;
}
}
} else {
reasons.push(format!("referenced check {id} is missing"));
if *outcome != CheckOutcome::Blocked {
*outcome = CheckOutcome::Inconclusive;
}
}
}
fn assessment_binding(
plan: &QualityPlan,
digest: &str,
subject: &Subject,
claimed: &str,
artifact: &crate::Artifact,
at: u64,
context: &EvaluationContext<'_>,
) -> Result<(), ContractError> {
crate::require(
subject == &plan.subject && claimed == digest,
"assessment plan/subject mismatch",
)?;
crate::require(
context_limits(plan, context).is_empty(),
"assessment no longer binds to the current repository, configuration, inputs or revision ancestry",
)?;
artifact_shape(artifact)?;
crate::require(
context.verified.contains(artifact),
"assessment artifact has not been byte-verified",
)?;
crate::require(
age_limit(at, plan, context).is_none(),
"assessment is stale or future-dated",
)
}
fn expectation_report(
plan: &QualityPlan,
digest: &str,
expectation: &Expectation,
checks: &[CheckReport],
runs: &BTreeMap<&str, &ExecutionReceipt>,
context: &EvaluationContext<'_>,
) -> ExpectationReport {
let mut outcome = CheckOutcome::Passed;
let mut reasons = Vec::new();
for id in &expectation.checks {
referenced_check(id, checks, &mut reasons, &mut outcome);
}
if let Some(control) = &expectation.positive_control {
referenced_check(control, checks, &mut reasons, &mut outcome);
}
if let Some(stabilizer) = &expectation.stabilizer {
referenced_check(&stabilizer.check_id, checks, &mut reasons, &mut outcome);
if !runs
.get(stabilizer.check_id.as_str())
.is_some_and(|r| r.observed_tests.contains(&stabilizer.test_id))
{
reasons.push("stabilizer test was not reported as executed".into());
if outcome != CheckOutcome::Blocked {
outcome = CheckOutcome::Inconclusive;
}
}
}
if let Some(limit) = &expectation.limitation {
reasons.push(format!("declared limitation: {limit}"));
}
if expectation.independent_audit
|| context
.audits
.iter()
.any(|a| a.expectation_id == expectation.id)
{
let audits: Vec<_> = context
.audits
.iter()
.filter(|a| a.expectation_id == expectation.id)
.collect();
let validation = if audits.len() != 1 {
Err(ContractError::Invalid(
"exactly one selected audit is required".into(),
))
} else {
let a = audits[0];
(|| {
assessment_binding(
plan,
digest,
&a.subject,
&a.plan_sha256,
&a.artifact,
a.captured_at_ms,
context,
)?;
crate::require(
nonempty(&a.observed)
&& identifier(&a.producer.agent_id)
&& identifier(&a.reviewer.agent_id),
"audit must identify actors and observation",
)?;
crate::require(
a.producer.agent_id != a.reviewer.agent_id
&& a.producer.attempt_id != a.reviewer.attempt_id,
"audit actors and attempts must be distinct",
)?;
crate::require(
expectation.checks.iter().any(|id| {
runs.get(id.as_str()).is_some_and(|r| {
r.attempt_id == a.producer.attempt_id
&& r.finished_at_ms <= a.captured_at_ms
})
}),
"producer does not reference supporting execution",
)?;
crate::require(
!expectation.checks.contains(&a.probe_check),
"audit requires a distinct adversarial probe",
)?;
crate::require(
runs.get(a.probe_check.as_str()).is_some_and(|r| {
r.attempt_id == a.reviewer.attempt_id
&& r.finished_at_ms <= a.captured_at_ms
}),
"reviewer does not reference the probe execution",
)?;
crate::require(
checks.iter().any(|c| {
c.id == a.probe_check
&& (c.outcome == CheckOutcome::Passed
|| (a.verdict == AuditVerdict::Contradicted
&& c.outcome == CheckOutcome::Blocked))
}),
"adversarial probe did not complete successfully",
)?;
if expectation.independent_audit {
sealmap_corpus::route::cross_family_with(
&a.producer.model,
&a.reviewer.model,
&context.families,
)
.map_err(|_| {
ContractError::Invalid(
"audit model-family independence is unknown or not satisfied".into(),
)
})?;
}
Ok(a.verdict)
})()
};
match validation {
Ok(AuditVerdict::Supported) => reasons.push(
if expectation.independent_audit {
"recorded independent audit supports the scoped claim"
} else {
"recorded audit supports the scoped claim; model independence was not required"
}
.into(),
),
Ok(AuditVerdict::Contradicted) => {
outcome = CheckOutcome::Blocked;
reasons.push("auditor recorded a counterexample".into());
}
Ok(AuditVerdict::Inconclusive) => {
if outcome != CheckOutcome::Blocked {
outcome = CheckOutcome::Inconclusive;
}
reasons.push("auditor could not conclude".into());
}
Err(e) => {
if outcome != CheckOutcome::Blocked {
outcome = CheckOutcome::Inconclusive;
}
reasons.push(e.to_string());
}
}
}
if expectation.category == EvidenceCategory::Manual
|| context
.manual_observations
.iter()
.any(|m| m.expectation_id == expectation.id)
{
let observations: Vec<_> = context
.manual_observations
.iter()
.filter(|m| m.expectation_id == expectation.id)
.collect();
let validation = if observations.len() != 1 {
Err(ContractError::Invalid(
"exactly one selected manual observation is required".into(),
))
} else {
let m = observations[0];
assessment_binding(
plan,
digest,
&m.subject,
&m.plan_sha256,
&m.artifact,
m.captured_at_ms,
context,
)
.and_then(|()| {
crate::require(
nonempty(&m.observer) && nonempty(&m.observed),
"manual observation needs an observer and observed effect",
)
})
.map(|()| m.verdict)
};
match validation {
Ok(AuditVerdict::Supported) => {
reasons.push("named manual observer supports the scoped claim".into())
}
Ok(AuditVerdict::Contradicted) => {
outcome = CheckOutcome::Blocked;
reasons.push("manual observation contradicts the claim".into());
}
Ok(AuditVerdict::Inconclusive) => {
if outcome != CheckOutcome::Blocked {
outcome = CheckOutcome::Inconclusive;
}
reasons.push("manual observation is inconclusive".into());
}
Err(e) => {
if outcome != CheckOutcome::Blocked {
outcome = CheckOutcome::Inconclusive;
}
reasons.push(e.to_string());
}
}
}
ExpectationReport {
id: expectation.id.clone(),
required: expectation.required,
outcome,
reasons,
}
}
pub fn evaluate(
plan: &QualityPlan,
receipts: &[ExecutionReceipt],
context: &EvaluationContext<'_>,
) -> GateReport {
let mut report = GateReport {
outcome: GateOutcome::Error,
required_checks: plan.checks.iter().filter(|c| c.required).count(),
passed_required_checks: 0,
checks: vec![],
expectations: vec![],
errors: vec![],
};
let digest = match plan.digest() {
Ok(digest) => digest,
Err(e) => {
report.errors.push(e.to_string());
return report;
}
};
if let Err(e) = context.current_subject.validate() {
report.errors.push(e.to_string());
return report;
}
let mut attempts = BTreeSet::new();
let mut runs = BTreeMap::new();
let ids: BTreeSet<_> = plan.checks.iter().map(|c| c.id.as_str()).collect();
for receipt in receipts {
if let Err(error) = receipt.validate() {
report.errors.push(error.to_string());
}
if let Some(check) = plan.checks.iter().find(|c| c.id == receipt.check_id)
&& (receipt.plan_sha256 != digest
|| receipt.subject != plan.subject
|| receipt.invocation != check.invocation)
{
report
.errors
.push("receipt does not bind to its declared plan/subject/invocation".into());
}
if !ids.contains(receipt.check_id.as_str()) {
report
.errors
.push("receipt references an unplanned check".into());
}
if !attempts.insert(receipt.attempt_id.as_str()) {
report.errors.push("duplicate execution attempt id".into());
}
if runs.insert(receipt.check_id.as_str(), receipt).is_some() {
report
.errors
.push("multiple attempts for one check; select an attempt explicitly".into());
}
}
let expectation_ids: BTreeSet<_> = plan.expectations.iter().map(|e| e.id.as_str()).collect();
for id in context
.audits
.iter()
.map(|a| a.expectation_id.as_str())
.chain(
context
.manual_observations
.iter()
.map(|m| m.expectation_id.as_str()),
)
{
if !expectation_ids.contains(id) {
report
.errors
.push("assessment references an unplanned expectation".into());
}
}
for check in &plan.checks {
let run = runs.get(check.id.as_str());
let (outcome, reasons) = match run {
Some(receipt) => check_receipt(plan, &digest, check, receipt, context),
None => (
CheckOutcome::Inconclusive,
vec!["no execution receipt was supplied".into()],
),
};
if check.required && outcome == CheckOutcome::Passed {
report.passed_required_checks += 1;
}
report.checks.push(CheckReport {
id: check.id.clone(),
required: check.required,
attempt_id: run.map(|r| r.attempt_id.clone()),
outcome,
reasons,
});
}
for expectation in &plan.expectations {
report.expectations.push(expectation_report(
plan,
&digest,
expectation,
&report.checks,
&runs,
context,
));
}
let required: Vec<_> = report
.checks
.iter()
.filter(|r| r.required)
.map(|r| r.outcome)
.chain(
report
.expectations
.iter()
.filter(|r| r.required)
.map(|r| r.outcome),
)
.collect();
report.outcome = if !report.errors.is_empty() || required.contains(&CheckOutcome::Error) {
GateOutcome::Error
} else if required.contains(&CheckOutcome::Blocked) {
GateOutcome::Blocked
} else if required.iter().any(|o| *o != CheckOutcome::Passed) {
GateOutcome::Incomplete
} else {
GateOutcome::Passed
};
report
}