use std::path::{
Path,
PathBuf,
};
use burn::prelude::Backend;
use crate::{
audit::{
AuditProbe,
AuditStreamRecorder,
AuditStreamVerifier,
reports::{
BaselineMode,
ReportsOptions,
backend_label,
},
},
errors::BunsenResult,
support::testing::backend_device,
};
pub trait AuditBody {
fn run<B: Backend>(
&self,
probe: &mut AuditProbe<'_>,
device: &B::Device,
) -> BunsenResult<()>;
}
pub fn audit_across<R: Backend, T: Backend>(body: &impl AuditBody) -> BunsenResult<()> {
let mut recorder = AuditStreamRecorder::default();
body.run::<R>(
&mut AuditProbe::new(vec![&mut recorder]),
&backend_device::<R>(),
)?;
let mut verifier = recorder.into_verifier();
body.run::<T>(
&mut AuditProbe::new(vec![&mut verifier]),
&backend_device::<T>(),
)?;
verifier.finish()
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum BaselineOutcome {
Recorded(PathBuf),
Verified(PathBuf),
}
pub fn audit_baseline<B: Backend>(
options: &ReportsOptions,
name: &str,
body: &impl AuditBody,
) -> BunsenResult<BaselineOutcome> {
let device = backend_device::<B>();
let path = options.report_path(&backend_label::<B>(&device), &format!("{name}.cbor"))?;
audit_baseline_at::<B>(&path, options.mode(), body)
}
pub fn audit_baseline_at<B: Backend>(
path: &Path,
mode: BaselineMode,
body: &impl AuditBody,
) -> BunsenResult<BaselineOutcome> {
let device = backend_device::<B>();
let path = path.to_path_buf();
let record = match mode {
BaselineMode::Auto => !path.exists(),
BaselineMode::Record => true,
BaselineMode::Verify => false,
};
if record {
let mut recorder = AuditStreamRecorder::default();
body.run::<B>(&mut AuditProbe::new(vec![&mut recorder]), &device)?;
recorder.save(&path)?;
Ok(BaselineOutcome::Recorded(path))
} else {
let mut verifier = AuditStreamVerifier::load(&path)?;
body.run::<B>(&mut AuditProbe::new(vec![&mut verifier]), &device)?;
verifier.finish()?;
Ok(BaselineOutcome::Verified(path))
}
}
#[cfg(test)]
mod tests {
use burn::tensor::Distribution;
use super::*;
use crate::{
burner::descriptors::TolerancePolicy,
errors::{
BunsenErrorKind,
LookupError,
LookupProblem,
ParseError,
ValueMismatch,
testing::{
ErrorMatcher,
predicate,
},
},
support::testing::{
CpuBackend,
PerformanceBackend,
seeded_tensor,
},
};
struct Body {
extra: bool,
shift: f32,
}
impl Body {
const EXTRA: Body = Body {
extra: true,
shift: 0.0,
};
const PLAIN: Body = Body {
extra: false,
shift: 0.0,
};
const SHIFTED: Body = Body {
extra: false,
shift: 1e-3,
};
}
impl AuditBody for Body {
fn run<B: Backend>(
&self,
probe: &mut AuditProbe<'_>,
device: &B::Device,
) -> BunsenResult<()> {
let x = seeded_tensor::<B, 3>(1, [2, 3, 4], Distribution::Uniform(-1.0, 1.0), device)
+ self.shift;
probe.assert_eq_as::<f32>("x", &x)?;
probe.assert_approx_eq_as::<f32>(
"x*x/3 - 1/x",
&((x.clone() * x.clone()) / 3.0 - x.clone().recip()),
TolerancePolicy::Balanced,
)?;
if self.extra {
probe.assert_eq_as::<f32>("extra", &x)?;
}
Ok(())
}
}
fn record(body: &Body) -> AuditStreamRecorder {
let mut recorder = AuditStreamRecorder::default();
body.run::<CpuBackend>(
&mut AuditProbe::new(vec![&mut recorder]),
&backend_device::<CpuBackend>(),
)
.unwrap();
recorder
}
fn verify(
verifier: &mut AuditStreamVerifier,
body: &Body,
) -> BunsenResult<()> {
body.run::<CpuBackend>(
&mut AuditProbe::new(vec![verifier]),
&backend_device::<CpuBackend>(),
)
}
#[test]
fn test_stream_round_trips_through_a_file() {
let recorder = record(&Body::PLAIN);
assert_eq!(recorder.events().len(), 2);
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("round_trip").join("stream.cbor");
recorder.save(&path).unwrap();
let mut verifier = AuditStreamVerifier::load(&path).unwrap();
verify(&mut verifier, &Body::PLAIN).unwrap();
verifier.finish().unwrap();
}
#[test]
fn test_verifier_rejects_a_changed_value() {
let mut verifier = record(&Body::PLAIN).into_verifier();
ErrorMatcher::kind(BunsenErrorKind::Policy)
.frame_contains("audit mismatch at event 0 (\"x\")")
.assert_err(&verify(&mut verifier, &Body::SHIFTED));
}
#[test]
fn test_verifier_rejects_an_extra_event() {
let mut verifier = record(&Body::PLAIN).into_verifier();
ErrorMatcher::kind(BunsenErrorKind::Policy)
.message_contains("unexpected audit event")
.has_cause::<ValueMismatch>()
.assert_err(&verify(&mut verifier, &Body::EXTRA));
}
#[test]
fn test_verifier_rejects_a_missing_event() {
let mut verifier = record(&Body::EXTRA).into_verifier();
verify(&mut verifier, &Body::PLAIN).unwrap();
ErrorMatcher::kind(BunsenErrorKind::Policy)
.message_contains("saw 2 of 3 events")
.has_cause::<ValueMismatch>()
.assert_err(&verifier.finish());
}
#[test]
fn test_load_missing_stream_is_not_found() {
let dir = tempfile::tempdir().unwrap();
ErrorMatcher::kind(BunsenErrorKind::Lookup)
.cause(predicate("a missing path", |c: &LookupError| {
c.problem == LookupProblem::Missing && c.key.ends_with("nope.cbor")
}))
.assert_err(&AuditStreamVerifier::load(&dir.path().join("nope.cbor")));
}
#[test]
fn test_load_garbage_stream_is_invalid_resource() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("garbage.cbor");
std::fs::write(&path, b"\xff\xff not cbor").unwrap();
ErrorMatcher::kind(BunsenErrorKind::InvalidResource)
.has_cause::<ParseError>()
.assert_err(&AuditStreamVerifier::load(&path));
}
#[test]
#[serial_test::serial]
fn test_audit_across_backends() {
audit_across::<CpuBackend, PerformanceBackend>(&Body::PLAIN).unwrap();
}
#[test]
fn test_baseline_at_records_then_verifies() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("baseline").join("body.cbor");
let outcome = audit_baseline_at::<CpuBackend>(&path, BaselineMode::Auto, &Body::PLAIN);
assert_eq!(outcome.unwrap(), BaselineOutcome::Recorded(path.clone()));
let outcome = audit_baseline_at::<CpuBackend>(&path, BaselineMode::Auto, &Body::PLAIN);
assert_eq!(outcome.unwrap(), BaselineOutcome::Verified(path.clone()));
assert!(
audit_baseline_at::<CpuBackend>(&path, BaselineMode::Verify, &Body::SHIFTED).is_err()
);
let outcome = audit_baseline_at::<CpuBackend>(&path, BaselineMode::Record, &Body::SHIFTED);
assert_eq!(outcome.unwrap(), BaselineOutcome::Recorded(path.clone()));
let missing = dir.path().join("baseline_missing").join("body.cbor");
assert!(
audit_baseline_at::<CpuBackend>(&missing, BaselineMode::Verify, &Body::PLAIN).is_err()
);
}
#[test]
fn test_baseline_uses_the_call_site_location() {
let dir = tempfile::tempdir().unwrap();
let options = ReportsOptions::new(dir.path());
let device = backend_device::<CpuBackend>();
let path = options
.report_path(&backend_label::<CpuBackend>(&device), "group/body.cbor")
.unwrap();
assert!(path.starts_with(dir.path()));
let outcome = audit_baseline::<CpuBackend>(&options, "group/body", &Body::PLAIN);
assert_eq!(outcome.unwrap(), BaselineOutcome::Recorded(path.clone()));
assert!(path.exists());
let outcome = audit_baseline::<CpuBackend>(&options, "group/body", &Body::PLAIN);
assert_eq!(outcome.unwrap(), BaselineOutcome::Verified(path.clone()));
let verify = options.with_mode(BaselineMode::Verify);
assert!(audit_baseline::<CpuBackend>(&verify, "group/body", &Body::SHIFTED).is_err());
assert!(audit_baseline::<CpuBackend>(&verify, "absent", &Body::PLAIN).is_err());
}
}