use std::path::{
Path,
PathBuf,
};
use burn::tensor::Device;
use crate::{
audit::{
AuditProbe,
AuditStreamRecorder,
AuditStreamVerifier,
reports::{
BaselineMode,
ReportsOptions,
backend_label,
},
},
errors::BunsenResult,
};
pub trait AuditBody {
fn run(
&self,
probe: &mut AuditProbe<'_>,
device: &Device,
) -> BunsenResult<()>;
}
impl<F> AuditBody for F
where
F: Fn(&mut AuditProbe<'_>, &Device) -> BunsenResult<()>,
{
fn run(
&self,
probe: &mut AuditProbe<'_>,
device: &Device,
) -> BunsenResult<()> {
self(probe, device)
}
}
pub fn audit_across(
reference: &Device,
target: &Device,
body: &impl AuditBody,
) -> BunsenResult<()> {
let mut recorder = AuditStreamRecorder::default();
body.run(&mut AuditProbe::new(vec![&mut recorder]), reference)?;
let mut verifier = recorder.into_verifier();
body.run(&mut AuditProbe::new(vec![&mut verifier]), target)?;
verifier.finish()
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum BaselineOutcome {
Recorded(PathBuf),
Verified(PathBuf),
}
pub fn audit_baseline(
options: &ReportsOptions,
device: &Device,
name: &str,
body: &impl AuditBody,
) -> BunsenResult<BaselineOutcome> {
let path = options.report_path(&backend_label(device), &format!("{name}.cbor"))?;
audit_baseline_at(&path, options.mode(), device, body)
}
pub fn audit_baseline_at(
path: &Path,
mode: BaselineMode,
device: &Device,
body: &impl AuditBody,
) -> BunsenResult<BaselineOutcome> {
let path = path.to_path_buf();
let record = match mode {
BaselineMode::Auto => !path.exists(),
BaselineMode::Record => true,
BaselineMode::Verify => false,
};
if record {
let mut recorder = AuditStreamRecorder::default();
body.run(&mut AuditProbe::new(vec![&mut recorder]), device)?;
recorder.save(&path)?;
Ok(BaselineOutcome::Recorded(path))
} else {
let mut verifier = AuditStreamVerifier::load(&path)?;
body.run(&mut AuditProbe::new(vec![&mut verifier]), device)?;
verifier.finish()?;
Ok(BaselineOutcome::Verified(path))
}
}
#[cfg(test)]
mod tests {
use burn::tensor::Distribution;
use super::*;
use crate::{
burner::descriptors::TolerancePolicy,
errors::{
BunsenErrorKind,
LookupError,
LookupProblem,
ParseError,
ValueMismatch,
testing::{
ErrorMatcher,
predicate,
},
},
support::testing::{
cpu_device,
performance_device,
seeded_tensor,
},
};
struct Body {
extra: bool,
shift: f32,
}
impl Body {
const EXTRA: Body = Body {
extra: true,
shift: 0.0,
};
const PLAIN: Body = Body {
extra: false,
shift: 0.0,
};
const SHIFTED: Body = Body {
extra: false,
shift: 1e-3,
};
}
impl AuditBody for Body {
fn run(
&self,
probe: &mut AuditProbe<'_>,
device: &Device,
) -> BunsenResult<()> {
let x = seeded_tensor::<3>(1, [2, 3, 4], Distribution::Uniform(-1.0, 1.0), device)
+ self.shift;
probe.assert_eq_as::<f32>("x", &x)?;
probe.assert_approx_eq_as::<f32>(
"x*x/3 - 1/x",
&((x.clone() * x.clone()) / 3.0 - x.clone().recip()),
TolerancePolicy::Balanced,
)?;
if self.extra {
probe.assert_eq_as::<f32>("extra", &x)?;
}
Ok(())
}
}
fn record(body: &Body) -> AuditStreamRecorder {
let mut recorder = AuditStreamRecorder::default();
body.run(&mut AuditProbe::new(vec![&mut recorder]), &cpu_device())
.unwrap();
recorder
}
fn verify(
verifier: &mut AuditStreamVerifier,
body: &Body,
) -> BunsenResult<()> {
body.run(&mut AuditProbe::new(vec![verifier]), &cpu_device())
}
#[test]
fn test_stream_round_trips_through_a_file() {
let recorder = record(&Body::PLAIN);
assert_eq!(recorder.events().len(), 2);
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("round_trip").join("stream.cbor");
recorder.save(&path).unwrap();
let mut verifier = AuditStreamVerifier::load(&path).unwrap();
verify(&mut verifier, &Body::PLAIN).unwrap();
verifier.finish().unwrap();
}
#[test]
fn test_verifier_rejects_a_changed_value() {
let mut verifier = record(&Body::PLAIN).into_verifier();
ErrorMatcher::kind(BunsenErrorKind::Policy)
.frame_contains("audit mismatch at event 0 (\"x\")")
.assert_err(&verify(&mut verifier, &Body::SHIFTED));
}
#[test]
fn test_verifier_rejects_an_extra_event() {
let mut verifier = record(&Body::PLAIN).into_verifier();
ErrorMatcher::kind(BunsenErrorKind::Policy)
.message_contains("unexpected audit event")
.has_cause::<ValueMismatch>()
.assert_err(&verify(&mut verifier, &Body::EXTRA));
}
#[test]
fn test_verifier_rejects_a_missing_event() {
let mut verifier = record(&Body::EXTRA).into_verifier();
verify(&mut verifier, &Body::PLAIN).unwrap();
ErrorMatcher::kind(BunsenErrorKind::Policy)
.message_contains("saw 2 of 3 events")
.has_cause::<ValueMismatch>()
.assert_err(&verifier.finish());
}
#[test]
fn test_load_missing_stream_is_not_found() {
let dir = tempfile::tempdir().unwrap();
ErrorMatcher::kind(BunsenErrorKind::Lookup)
.cause(predicate("a missing path", |c: &LookupError| {
c.problem == LookupProblem::Missing && c.key.ends_with("nope.cbor")
}))
.assert_err(&AuditStreamVerifier::load(&dir.path().join("nope.cbor")));
}
#[test]
fn test_load_garbage_stream_is_invalid_resource() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("garbage.cbor");
std::fs::write(&path, b"\xff\xff not cbor").unwrap();
ErrorMatcher::kind(BunsenErrorKind::InvalidResource)
.has_cause::<ParseError>()
.assert_err(&AuditStreamVerifier::load(&path));
}
#[test]
#[serial_test::serial]
fn test_audit_across_backends() {
audit_across(&cpu_device(), &performance_device(), &Body::PLAIN).unwrap();
}
#[test]
fn test_baseline_at_records_then_verifies() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("baseline").join("body.cbor");
let outcome = audit_baseline_at(&path, BaselineMode::Auto, &cpu_device(), &Body::PLAIN);
assert_eq!(outcome.unwrap(), BaselineOutcome::Recorded(path.clone()));
let outcome = audit_baseline_at(&path, BaselineMode::Auto, &cpu_device(), &Body::PLAIN);
assert_eq!(outcome.unwrap(), BaselineOutcome::Verified(path.clone()));
assert!(
audit_baseline_at(&path, BaselineMode::Verify, &cpu_device(), &Body::SHIFTED).is_err()
);
let outcome = audit_baseline_at(&path, BaselineMode::Record, &cpu_device(), &Body::SHIFTED);
assert_eq!(outcome.unwrap(), BaselineOutcome::Recorded(path.clone()));
let missing = dir.path().join("baseline_missing").join("body.cbor");
assert!(
audit_baseline_at(&missing, BaselineMode::Verify, &cpu_device(), &Body::PLAIN).is_err()
);
}
#[test]
fn test_closure_body() {
let body = |probe: &mut AuditProbe<'_>, device: &Device| -> BunsenResult<()> {
let x = seeded_tensor::<2>(3, [2, 2], Distribution::Default, device);
probe.assert_eq_as::<f32>("x", &x)
};
audit_across(&cpu_device(), &cpu_device(), &body).unwrap();
}
#[test]
fn test_baseline_uses_the_call_site_location() {
let dir = tempfile::tempdir().unwrap();
let options = ReportsOptions::new(dir.path());
let device = cpu_device();
let path = options
.report_path(&backend_label(&device), "group/body.cbor")
.unwrap();
assert!(path.starts_with(dir.path()));
let outcome = audit_baseline(&options, &device, "group/body", &Body::PLAIN);
assert_eq!(outcome.unwrap(), BaselineOutcome::Recorded(path.clone()));
assert!(path.exists());
let outcome = audit_baseline(&options, &device, "group/body", &Body::PLAIN);
assert_eq!(outcome.unwrap(), BaselineOutcome::Verified(path.clone()));
let verify = options.with_mode(BaselineMode::Verify);
assert!(audit_baseline(&verify, &device, "group/body", &Body::SHIFTED).is_err());
assert!(audit_baseline(&verify, &device, "absent", &Body::PLAIN).is_err());
}
}