use assay_evidence::{BundleWriter, EvidenceEvent};
use assert_cmd::Command;
use chrono::{TimeZone, Utc};
use serde_json::{json, Value};
use std::fs;
use tempfile::tempdir;
const DECISION_EVENT_TYPE: &str = "assay.tool_decision_surface.v0";
fn fixture(name: &str) -> Value {
let p = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("../assay-mcp-server/tests/fixtures/side_effect")
.join(name);
serde_json::from_str(&fs::read_to_string(&p).unwrap()).unwrap()
}
fn bundle_with_asserted_decision(path: &std::path::Path) {
let mut surface = fixture("verified.json");
let decision = &mut surface["observed_tool_decisions"][0];
decision["response"]["side_effect"] = json!({ "asserted": true, "level": "asserted" });
decision["response"]["side_effect_verified"] = json!(false);
let mut event = EvidenceEvent::new(
DECISION_EVENT_TYPE,
"urn:assay:test:side-effects-cli",
"run-side-effects",
0,
surface,
);
event.time = Utc.timestamp_opt(1_700_000_000, 0).unwrap();
let file = fs::File::create(path).unwrap();
let mut writer = BundleWriter::new(file);
writer.add_event(event);
writer.finish().unwrap();
}
fn import_dir(dir: &std::path::Path, record_fixture: &str) -> std::path::PathBuf {
let out = dir.join("audit");
fs::create_dir_all(&out).unwrap();
fs::write(
out.join("record.json"),
serde_json::to_string(&fixture(record_fixture)).unwrap(),
)
.unwrap();
out
}
fn run(bundle: &std::path::Path, import: Option<&std::path::Path>) -> Value {
let mut cmd = Command::cargo_bin("assay").unwrap();
cmd.arg("evidence")
.arg("verify-side-effects")
.arg(bundle)
.arg("--format")
.arg("json");
if let Some(dir) = import {
cmd.arg("--audit-import").arg(dir);
}
let out = cmd.assert().success().get_output().stdout.clone();
serde_json::from_slice(&out).expect("json report")
}
#[test]
fn a_binding_audit_record_promotes_the_call_to_verified() {
let dir = tempdir().unwrap();
let bundle = dir.path().join("b.tar.gz");
bundle_with_asserted_decision(&bundle);
let import = import_dir(dir.path(), "audit_record_github_deploy_key.json");
let report = run(&bundle, Some(&import));
assert_eq!(report["promoted"], json!(1));
assert_eq!(report["calls"][0]["level"], json!("verified"));
assert_eq!(report["audit_records_unmatched"], json!(0));
assert!(report["calls"][0]["subject_digest"].is_string());
}
#[test]
fn a_mismatched_record_leaves_the_call_asserted_and_says_why() {
let dir = tempdir().unwrap();
let bundle = dir.path().join("b.tar.gz");
bundle_with_asserted_decision(&bundle);
let import = import_dir(dir.path(), "audit_record_mismatch.json");
let report = run(&bundle, Some(&import));
assert_eq!(report["promoted"], json!(0));
assert_eq!(report["calls"][0]["level"], json!("asserted"));
assert_eq!(
report["calls"][0]["binding"]["outcome"],
json!("binds_different_call"),
"a rejected record must be reported with its reason"
);
assert_eq!(
report["audit_records_unmatched"],
json!(1),
"an imported record that bound to nothing is counted, not discarded"
);
}
#[test]
fn no_import_leaves_everything_asserted_without_failing() {
let dir = tempdir().unwrap();
let bundle = dir.path().join("b.tar.gz");
bundle_with_asserted_decision(&bundle);
let report = run(&bundle, None);
assert_eq!(report["promoted"], json!(0));
assert_eq!(report["calls"][0]["level"], json!("asserted"));
assert_eq!(report["audit_records_imported"], json!(0));
assert!(
report["calls"][0]["binding"].is_null(),
"nothing was considered"
);
}
#[test]
fn the_report_declares_what_it_does_not_claim() {
let dir = tempdir().unwrap();
let bundle = dir.path().join("b.tar.gz");
bundle_with_asserted_decision(&bundle);
let import = import_dir(dir.path(), "audit_record_github_deploy_key.json");
let report = run(&bundle, Some(&import));
let claims: Vec<&str> = report["claims_not_made"]
.as_array()
.unwrap()
.iter()
.map(|v| v.as_str().unwrap())
.collect();
assert!(claims.contains(&"provider_query"));
assert!(claims.contains(&"audit_record_authenticity_beyond_its_own_signature"));
}
#[test]
fn the_table_render_path_works_and_names_the_rejection() {
let dir = tempdir().unwrap();
let bundle = dir.path().join("b.tar.gz");
bundle_with_asserted_decision(&bundle);
let import = import_dir(dir.path(), "audit_record_mismatch.json");
let out = Command::cargo_bin("assay")
.unwrap()
.arg("evidence")
.arg("verify-side-effects")
.arg(&bundle)
.arg("--audit-import")
.arg(&import)
.assert()
.success()
.get_output()
.stdout
.clone();
let text = String::from_utf8(out).unwrap();
assert!(text.contains("level=asserted"), "{text}");
assert!(
text.contains("not promoted"),
"a rejection must be visible, not only in JSON"
);
assert!(text.contains("binds_different_call"), "{text}");
assert!(text.contains("promoted to verified: 0"), "{text}");
}
#[test]
fn an_asserted_side_effect_cannot_carry_an_occurrence_claim() {
let dir = tempdir().unwrap();
let bundle = dir.path().join("b.tar.gz");
bundle_with_asserted_decision(&bundle);
let report = run(&bundle, None);
assert_eq!(report["calls"][0]["level"], json!("asserted"));
assert_ne!(
report["calls"][0]["occurrence_claim"],
json!("allowed"),
"producer-reported evidence must not license an occurrence claim"
);
}
#[test]
fn a_verified_side_effect_can_carry_an_occurrence_claim() {
let dir = tempdir().unwrap();
let bundle = dir.path().join("b.tar.gz");
bundle_with_asserted_decision(&bundle);
let import = import_dir(dir.path(), "audit_record_github_deploy_key.json");
let report = run(&bundle, Some(&import));
assert_eq!(report["calls"][0]["level"], json!("verified"));
assert_eq!(report["calls"][0]["occurrence_claim"], json!("allowed"));
}
#[test]
fn no_level_ever_supports_an_absence_claim() {
let dir = tempdir().unwrap();
let bundle = dir.path().join("b.tar.gz");
bundle_with_asserted_decision(&bundle);
let import = import_dir(dir.path(), "audit_record_github_deploy_key.json");
for report in [run(&bundle, None), run(&bundle, Some(&import))] {
assert_ne!(
report["calls"][0]["bounded_negative_claim"],
json!("allowed"),
"no side-effect level is coverage of a dimension"
);
}
}
fn health(dir: &std::path::Path, coverage: &str, correlation: &str) -> std::path::PathBuf {
let p = dir.join("observation_health.json");
fs::write(
&p,
serde_json::to_string(&json!({
"schema": "assay.runner.observation_health.v0",
"kernel_layer": "complete",
"ringbuf_drops": 0,
"network_protocol_coverage": coverage,
"cgroup_correlation": correlation,
}))
.unwrap(),
)
.unwrap();
p
}
fn run_with_health(
bundle: &std::path::Path,
import: Option<&std::path::Path>,
oh: &std::path::Path,
) -> Value {
let mut cmd = Command::cargo_bin("assay").unwrap();
cmd.arg("evidence")
.arg("verify-side-effects")
.arg(bundle)
.arg("--observation-health")
.arg(oh)
.arg("--format")
.arg("json");
if let Some(d) = import {
cmd.arg("--audit-import").arg(d);
}
let out = cmd.assert().success().get_output().stdout.clone();
serde_json::from_slice(&out).unwrap()
}
#[test]
fn a_watching_observer_that_saw_nothing_refutes_the_egress() {
let dir = tempdir().unwrap();
let bundle = dir.path().join("b.tar.gz");
bundle_with_asserted_decision(&bundle);
let oh = health(dir.path(), "connect_only", "clean");
let report = run_with_health(&bundle, None, &oh);
assert_eq!(report["calls"][0]["egress"]["outcome"], json!("refuted"));
assert_eq!(
report["calls"][0]["egress"]["watched_surface"],
json!("cgroup_sock_addr:connect4"),
"a refutation names the surface it watched, not the world"
);
}
#[test]
fn a_blind_observer_does_not_refute_anything() {
let dir = tempdir().unwrap();
let bundle = dir.path().join("b.tar.gz");
bundle_with_asserted_decision(&bundle);
let oh = health(dir.path(), "absent", "clean");
let report = run_with_health(&bundle, None, &oh);
assert_eq!(
report["calls"][0]["egress"]["outcome"],
json!("no_coverage")
);
}
#[test]
fn a_refutation_blocks_the_occurrence_claim_even_when_an_audit_record_verified_it() {
let dir = tempdir().unwrap();
let bundle = dir.path().join("b.tar.gz");
bundle_with_asserted_decision(&bundle);
let import = import_dir(dir.path(), "audit_record_github_deploy_key.json");
let oh = health(dir.path(), "connect_only", "clean");
let report = run_with_health(&bundle, Some(&import), &oh);
let call = &report["calls"][0];
assert_eq!(
call["level"],
json!("verified"),
"the audit record still bound"
);
assert_eq!(
call["egress"]["outcome"],
json!("refuted"),
"and the kernel still disagrees"
);
assert_eq!(
call["occurrence_claim"],
json!("blocked"),
"a contradicted occurrence must not be claimable from either side"
);
}
#[test]
fn partial_correlation_cannot_overturn_a_verified_record() {
let dir = tempdir().unwrap();
let bundle = dir.path().join("b.tar.gz");
bundle_with_asserted_decision(&bundle);
let import = import_dir(dir.path(), "audit_record_github_deploy_key.json");
let oh = health(dir.path(), "connect_only", "partial");
let report = run_with_health(&bundle, Some(&import), &oh);
assert_eq!(
report["calls"][0]["egress"]["outcome"],
json!("coverage_degraded")
);
assert_eq!(
report["calls"][0]["occurrence_claim"],
json!("allowed"),
"a degraded observer must not overturn an audit record that bound"
);
}
#[test]
fn a_peer_set_from_a_different_run_cannot_refute() {
let dir = tempdir().unwrap();
let bundle = dir.path().join("b.tar.gz");
bundle_with_asserted_decision(&bundle);
let oh = dir.path().join("oh.json");
fs::write(
&oh,
serde_json::to_string(&json!({
"schema": "assay.runner.observation_health.v0",
"run_id": "run-A",
"kernel_layer": "complete", "ringbuf_drops": 0,
"network_protocol_coverage": "connect_only", "cgroup_correlation": "clean",
}))
.unwrap(),
)
.unwrap();
let peers = dir.path().join("peers.json");
fs::write(
&peers,
serde_json::to_string(&json!({
"schema": "assay.monitor.observed_peers.v0",
"run_id": "run-B",
"peers": [],
}))
.unwrap(),
)
.unwrap();
let out = Command::cargo_bin("assay")
.unwrap()
.arg("evidence")
.arg("verify-side-effects")
.arg(&bundle)
.arg("--observation-health")
.arg(&oh)
.arg("--observed-peers")
.arg(&peers)
.arg("--format")
.arg("json")
.assert()
.success()
.get_output()
.stdout
.clone();
let report: Value = serde_json::from_slice(&out).unwrap();
assert_eq!(
report["calls"][0]["egress"]["outcome"],
json!("no_coverage"),
"a mismatched run pair must refuse, not refute"
);
}
#[test]
fn the_ladder_ordering_is_visible_in_the_report_and_not_only_in_the_type_system() {
let dir = tempdir().unwrap();
let bundle = dir.path().join("b.tar.gz");
bundle_with_asserted_decision(&bundle);
let import = import_dir(dir.path(), "audit_record_github_deploy_key.json");
let asserted = run(&bundle, None);
let verified = run(&bundle, Some(&import));
assert_eq!(
asserted["calls"][0]["occurrence_ceiling"],
json!("asserted"),
"a provider's own word caps at `asserted`, whatever else is true of the run"
);
assert_eq!(
verified["calls"][0]["occurrence_ceiling"],
json!("independently_confirmed"),
"a bound record from the system that would know is what the top rung is for"
);
assert_ne!(
asserted["calls"][0]["occurrence_ceiling"], verified["calls"][0]["occurrence_ceiling"],
"if these ever agree the ladder has been flattened and buys nothing"
);
}
fn bundle_with_one_bindable_and_one_unbindable_call(path: &std::path::Path) {
let mut surface = fixture("verified.json");
let mut first = surface["observed_tool_decisions"][0].clone();
first["response"]["side_effect"] = json!({ "asserted": true, "level": "asserted" });
first["response"]["side_effect_verified"] = json!(false);
let mut second = first.clone();
second["tool"]["name"] = json!("github.add_deploy_key_other");
second["action"]["target"]["repo"] = json!("some-other-repo");
surface["observed_tool_decisions"] = json!([first, second]);
let mut event = EvidenceEvent::new(
DECISION_EVENT_TYPE,
"urn:assay:test:side-effects-cli",
"run-side-effects-two",
0,
surface,
);
event.time = Utc.timestamp_opt(1_700_000_000, 0).unwrap();
let file = fs::File::create(path).unwrap();
let mut writer = BundleWriter::new(file);
writer.add_event(event);
writer.finish().unwrap();
}
#[test]
fn the_run_level_ceiling_is_the_weakest_rung_and_never_the_strongest() {
let dir = tempdir().unwrap();
let bundle = dir.path().join("two.tar.gz");
bundle_with_one_bindable_and_one_unbindable_call(&bundle);
let import = import_dir(dir.path(), "audit_record_github_deploy_key.json");
let report = run(&bundle, Some(&import));
let rows: Vec<&Value> = report["calls"]
.as_array()
.unwrap()
.iter()
.filter(|c| c["asserted"] == json!(true))
.collect();
assert_eq!(rows.len(), 2, "the discriminating input must survive");
let strongest = rows
.iter()
.map(|c| &c["occurrence_ceiling"])
.max_by_key(|c| ladder_index(c))
.unwrap();
let weakest = rows
.iter()
.map(|c| &c["occurrence_ceiling"])
.min_by_key(|c| ladder_index(c))
.unwrap();
assert_ne!(
strongest, weakest,
"without two different rungs this test cannot tell a fold from a maximum"
);
assert_eq!(
report["weakest_occurrence_ceiling"],
json!({ "state": "rung", "ceiling": weakest }),
"the run-level rung must be the weakest of its asserting calls"
);
}
#[test]
fn a_refutation_takes_the_rung_with_it_rather_than_leaving_a_strong_number_behind() {
let dir = tempdir().unwrap();
let bundle = dir.path().join("b.tar.gz");
bundle_with_asserted_decision(&bundle);
let import = import_dir(dir.path(), "audit_record_github_deploy_key.json");
let oh = health(dir.path(), "connect_only", "clean");
let report = run_with_health(&bundle, Some(&import), &oh);
assert_eq!(report["calls"][0]["occurrence_claim"], json!("blocked"));
assert_eq!(
report["calls"][0]["occurrence_ceiling"],
Value::Null,
"a blocked occurrence must not advertise a rung"
);
assert_eq!(
report["weakest_occurrence_ceiling"],
json!({ "state": "blocked" }),
"and one blocked asserting call collapses the run-level answer, rather than lowering it"
);
}
fn bundle_with_no_asserted_side_effect(path: &std::path::Path) {
let mut surface = fixture("verified.json");
let decision = &mut surface["observed_tool_decisions"][0];
decision["response"]["side_effect"] = json!({ "asserted": false, "level": "asserted" });
decision["response"]["side_effect_asserted"] = json!(false);
decision["response"]["side_effect_verified"] = json!(false);
let mut event = EvidenceEvent::new(
DECISION_EVENT_TYPE,
"urn:assay:test:side-effects-cli",
"run-side-effects-none",
0,
surface,
);
event.time = Utc.timestamp_opt(1_700_000_000, 0).unwrap();
let file = fs::File::create(path).unwrap();
let mut writer = BundleWriter::new(file);
writer.add_event(event);
writer.finish().unwrap();
}
#[test]
fn a_run_that_claimed_nothing_is_distinguishable_from_a_run_that_was_contradicted() {
let dir = tempdir().unwrap();
let quiet = dir.path().join("quiet.tar.gz");
bundle_with_no_asserted_side_effect(&quiet);
let quiet_report = run(&quiet, None);
let contradicted = dir.path().join("contradicted.tar.gz");
bundle_with_asserted_decision(&contradicted);
let import = import_dir(dir.path(), "audit_record_github_deploy_key.json");
let oh = health(dir.path(), "connect_only", "clean");
let contradicted_report = run_with_health(&contradicted, Some(&import), &oh);
assert_eq!(
quiet_report["weakest_occurrence_ceiling"],
json!({ "state": "nothing_claimed" })
);
assert_eq!(
contradicted_report["weakest_occurrence_ceiling"],
json!({ "state": "blocked" })
);
assert_ne!(
quiet_report["weakest_occurrence_ceiling"],
contradicted_report["weakest_occurrence_ceiling"],
"silence and contradiction must not serialize to the same thing"
);
assert_eq!(quiet_report["calls"][0]["asserted"], json!(false));
assert_eq!(quiet_report["calls"][0]["occurrence_ceiling"], Value::Null);
}
fn bundle_with_no_decision_events(path: &std::path::Path) {
let event = EvidenceEvent::new(
"assay.some_other_event.v0",
"urn:assay:test:side-effects-cli",
"run-no-decisions",
0,
json!({"irrelevant": true}),
);
let file = fs::File::create(path).unwrap();
let mut writer = BundleWriter::new(file);
writer.add_event(event);
writer.finish().unwrap();
}
#[test]
fn a_bundle_with_no_decision_events_fails_rather_than_reporting_nothing_claimed() {
let dir = tempdir().unwrap();
let bundle = dir.path().join("empty.tar.gz");
bundle_with_no_decision_events(&bundle);
let out = Command::cargo_bin("assay")
.unwrap()
.arg("evidence")
.arg("verify-side-effects")
.arg(&bundle)
.arg("--format")
.arg("json")
.assert()
.failure()
.get_output()
.stderr
.clone();
let stderr = String::from_utf8(out).unwrap();
assert!(
stderr.contains("no") && stderr.contains(DECISION_EVENT_TYPE),
"the error must name the missing event type: {stderr}"
);
}
fn bundle_with_non_array_decisions(path: &std::path::Path) {
let mut surface = fixture("verified.json");
surface["observed_tool_decisions"] = json!("not_an_array");
let mut event = EvidenceEvent::new(
DECISION_EVENT_TYPE,
"urn:assay:test:side-effects-cli",
"run-bad-decisions",
0,
surface,
);
event.time = chrono::Utc.timestamp_opt(1_700_000_000, 0).unwrap();
let file = fs::File::create(path).unwrap();
let mut writer = BundleWriter::new(file);
writer.add_event(event);
writer.finish().unwrap();
}
#[test]
fn a_non_array_observed_tool_decisions_fails_rather_than_silently_skipping() {
let dir = tempdir().unwrap();
let bundle = dir.path().join("bad.tar.gz");
bundle_with_non_array_decisions(&bundle);
let out = Command::cargo_bin("assay")
.unwrap()
.arg("evidence")
.arg("verify-side-effects")
.arg(&bundle)
.arg("--format")
.arg("json")
.assert()
.failure()
.get_output()
.stderr
.clone();
let stderr = String::from_utf8(out).unwrap();
assert!(
stderr.contains("not an array"),
"the error must explain what went wrong: {stderr}"
);
}
fn bundle_with_non_bool_asserted(path: &std::path::Path) {
let mut surface = fixture("verified.json");
let decision = &mut surface["observed_tool_decisions"][0];
decision["response"]["side_effect_asserted"] = json!("yes");
let mut event = EvidenceEvent::new(
DECISION_EVENT_TYPE,
"urn:assay:test:side-effects-cli",
"run-bad-asserted",
0,
surface,
);
event.time = chrono::Utc.timestamp_opt(1_700_000_000, 0).unwrap();
let file = fs::File::create(path).unwrap();
let mut writer = BundleWriter::new(file);
writer.add_event(event);
writer.finish().unwrap();
}
#[test]
fn a_non_bool_side_effect_asserted_fails_rather_than_defaulting_to_false() {
let dir = tempdir().unwrap();
let bundle = dir.path().join("bad.tar.gz");
bundle_with_non_bool_asserted(&bundle);
let out = Command::cargo_bin("assay")
.unwrap()
.arg("evidence")
.arg("verify-side-effects")
.arg(&bundle)
.arg("--format")
.arg("json")
.assert()
.failure()
.get_output()
.stderr
.clone();
let stderr = String::from_utf8(out).unwrap();
assert!(
stderr.contains("not a boolean"),
"the error must explain what went wrong: {stderr}"
);
}
fn bundle_with_missing_asserted(path: &std::path::Path) {
let mut surface = fixture("verified.json");
let decision = &mut surface["observed_tool_decisions"][0];
if let Some(resp) = decision.get_mut("response").and_then(Value::as_object_mut) {
resp.remove("side_effect_asserted");
}
let mut event = EvidenceEvent::new(
DECISION_EVENT_TYPE,
"urn:assay:test:side-effects-cli",
"run-missing-asserted",
0,
surface,
);
event.time = chrono::Utc.timestamp_opt(1_700_000_000, 0).unwrap();
let file = fs::File::create(path).unwrap();
let mut writer = BundleWriter::new(file);
writer.add_event(event);
writer.finish().unwrap();
}
#[test]
fn a_missing_side_effect_asserted_fails_rather_than_defaulting_to_false() {
let dir = tempdir().unwrap();
let bundle = dir.path().join("bad.tar.gz");
bundle_with_missing_asserted(&bundle);
let out = Command::cargo_bin("assay")
.unwrap()
.arg("evidence")
.arg("verify-side-effects")
.arg(&bundle)
.arg("--format")
.arg("json")
.assert()
.failure()
.get_output()
.stderr
.clone();
let stderr = String::from_utf8(out).unwrap();
assert!(
stderr.contains("side_effect_asserted"),
"the error must name the missing field: {stderr}"
);
}
fn bundle_with_two_identical_asserting_calls(path: &std::path::Path) {
let mut surface = fixture("verified.json");
let mut call = surface["observed_tool_decisions"][0].clone();
call["response"]["side_effect"] = json!({ "asserted": true, "level": "asserted" });
call["response"]["side_effect_verified"] = json!(false);
surface["observed_tool_decisions"] = json!([call.clone(), call]);
let mut event = EvidenceEvent::new(
DECISION_EVENT_TYPE,
"urn:assay:test:side-effects-cli",
"run-two-identical",
0,
surface,
);
event.time = chrono::Utc.timestamp_opt(1_700_000_000, 0).unwrap();
let file = fs::File::create(path).unwrap();
let mut writer = BundleWriter::new(file);
writer.add_event(event);
writer.finish().unwrap();
}
#[test]
fn one_audit_record_cannot_vouch_for_two_identical_calls() {
let dir = tempdir().unwrap();
let bundle = dir.path().join("two.tar.gz");
bundle_with_two_identical_asserting_calls(&bundle);
let import = import_dir(dir.path(), "audit_record_github_deploy_key.json");
let report = run(&bundle, Some(&import));
let calls = report["calls"].as_array().unwrap();
assert_eq!(calls.len(), 2, "both calls must appear");
let verified_count = calls
.iter()
.filter(|c| c["level"] == json!("verified"))
.count();
let asserted_count = calls
.iter()
.filter(|c| c["level"] == json!("asserted"))
.count();
assert_eq!(
verified_count, 1,
"exactly one call may be promoted by one record"
);
assert_eq!(
asserted_count, 1,
"the second call stays at asserted because its record was consumed"
);
assert_eq!(report["promoted"], json!(1));
assert_eq!(
report["weakest_occurrence_ceiling"],
json!({ "state": "rung", "ceiling": "asserted" }),
"the weakest rung governs when one record is consumed and one call is left at asserted"
);
}
fn ladder_index(v: &Value) -> usize {
match v.as_str() {
Some("asserted") => 0,
Some("asserted_signed") => 1,
Some("observed_at_receiver") => 2,
Some("observed_in_path") => 3,
Some("independently_confirmed") => 4,
None if v.is_null() => panic!("no rung on this row; the caller must filter before ranking"),
other => panic!("unknown ceiling rung in report: {other:?}"),
}
}