use std::collections::{BTreeMap, HashMap};
use std::fmt::Write as _;
use std::io::{Read as _, Write as _};
use std::path::{Path, PathBuf};
use stegobench_core::exit;
use stegobench_core::result::{
ArmMetrics, Configuration, ContaminationCheck, CorpusSource, Determinism, Domain, Isolation,
Pairing, PinnedBy, PluginRef, RateUnit, Result1, SplitDiscipline,
};
use crate::cli::ReportFormat;
pub const MAX_RESULT_BYTES: u64 = 1024 * 1024;
pub const MAX_RESULTS: usize = 10_000;
pub const MAX_DEPTH: usize = 8;
pub const MAX_ARM_LINES_SHOWN: usize = 200;
#[derive(Debug, thiserror::Error)]
pub enum ReportError {
#[error("cannot read {path}: {}", crate::plain_io(.source))]
Read {
path: String,
#[source]
source: std::io::Error,
},
#[error(
"{path} cannot be read: {why}.\n\
`report` takes result documents, or the directories `score --out` \
wrote them to. Check the path, and `stegobench score --help` says \
where the documents land"
)]
PathUnusable { path: String, why: String },
#[error(
"nothing under {paths} is a result document, so there is no table to \
print. An empty table under an exit code of zero reads as \"checked, \
nothing to worry about\", which is a different claim from \"nothing \
was found\". Point this at result-v1 files, or at a directory holding \
them"
)]
NothingFound { paths: String },
#[error(
"{found} result documents were found under {paths} and this refuses \
past {cap}. A table nobody reads is not the problem; a table that \
quietly stopped short would be. Narrow the paths, or ask for \
--format csv over smaller batches"
)]
TooMany {
found: usize,
cap: usize,
paths: String,
},
#[error(
"the directory tree under {path} goes deeper than {cap} levels. \
Refusing rather than walking further: results sit one or two levels \
down, and anything deeper is more likely a link cycle than a corpus \
of measurements"
)]
TooDeep { path: String, cap: usize },
#[error("could not write the report to {path}: {source}")]
Write {
path: String,
#[source]
source: std::io::Error,
},
}
impl ReportError {
pub fn exit_code(&self) -> i32 {
match self {
ReportError::NothingFound { .. } => exit::USAGE,
ReportError::TooMany { .. }
| ReportError::TooDeep { .. }
| ReportError::PathUnusable { .. } => exit::PREFLIGHT_REFUSED,
ReportError::Read { .. } | ReportError::Write { .. } => exit::FAILURE,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
pub struct Skipped {
pub path: String,
pub reason: String,
pub invalid_document: bool,
}
#[derive(Debug, Clone)]
pub struct Row {
pub source: String,
pub detector: String,
pub subject_version: String,
pub pinned_by: &'static str,
pub isolation: &'static str,
pub corpus_name: String,
pub corpus_digest: String,
pub corpus_source: CorpusSource,
pub corpus_tier: Option<String>,
pub corpus_pairs: u64,
pub configuration: Configuration,
pub embedder: String,
pub rate: Option<(f64, RateUnit)>,
pub domain: Domain,
pub format: String,
pub auc: f64,
pub auc_ci95: Option<[f64; 2]>,
pub tpr_at_fpr: BTreeMap<String, f64>,
pub fpr_achieved: BTreeMap<String, f64>,
pub per_arm: Vec<ArmMetrics>,
pub pairing: Pairing,
pub split: SplitDiscipline,
pub n_clean: u64,
pub n_stego: u64,
pub n_error: u64,
pub trained_on: Option<String>,
pub self_reported: bool,
pub network_reachable: bool,
pub harness_version: String,
pub started_utc: String,
pub flags: Vec<String>,
}
impl Row {
pub fn arm(&self) -> String {
match self.rate {
None => self.embedder.clone(),
Some((v, RateUnit::Bpp)) => format!("{} at {v} bpp", self.embedder),
Some((v, RateUnit::CapacityFraction)) => {
format!("{} at {:.3}% of capacity", self.embedder, v * 100.0)
}
}
}
fn order(&self) -> (String, String, String, String, String) {
let rate = match self.rate {
Some((v, unit)) => format!("{}-{:016.6}", unit_str(unit), v),
None => "0-none".to_string(),
};
(
self.embedder.clone(),
rate,
self.detector.clone(),
self.subject_version.clone(),
self.source.clone(),
)
}
}
#[derive(Debug, Clone)]
pub struct Group {
pub corpus_name: String,
pub corpus_digest: String,
pub configuration: Configuration,
pub rows: Vec<Row>,
}
impl Group {
pub fn comparability(&self) -> String {
match self.configuration {
Configuration::Named => format!(
"NAMED: quotable beside another `named` run over this same \
digest of {}, and beside nothing else.",
self.corpus_name
),
Configuration::Custom => format!(
"CUSTOM: {} declared no digest in advance. Quote these \
figures beside nothing else.",
self.corpus_name
),
}
}
}
#[derive(Debug, Clone)]
pub struct Report {
pub groups: Vec<Group>,
pub skipped: Vec<Skipped>,
pub read: usize,
pub asked_for: Vec<String>,
}
impl Report {
pub fn any_cross_group_comparison_is_valid(&self) -> bool {
self.groups.len() <= 1
}
pub fn exit_code(&self) -> i32 {
if self.skipped.iter().any(|s| s.invalid_document) {
exit::SCHEMA_INVALID
} else if !self.skipped.is_empty() {
exit::FAILURE
} else {
exit::OK
}
}
}
fn discover(paths: &[PathBuf]) -> Result<(Vec<PathBuf>, Vec<Skipped>), ReportError> {
let mut found = Vec::new();
let mut skipped = Vec::new();
for path in paths {
let meta = std::fs::metadata(path).map_err(|e| match e.kind() {
std::io::ErrorKind::NotFound | std::io::ErrorKind::PermissionDenied => {
ReportError::PathUnusable {
path: path.display().to_string(),
why: crate::plain_io(&e),
}
}
_ => ReportError::Read {
path: path.display().to_string(),
source: e,
},
})?;
if meta.is_dir() {
walk(path, 0, &mut found, &mut skipped)?;
} else {
found.push(path.clone());
}
if found.len() > MAX_RESULTS {
return Err(ReportError::TooMany {
found: found.len(),
cap: MAX_RESULTS,
paths: display_paths(paths),
});
}
}
found.sort();
found.dedup();
Ok((found, skipped))
}
fn walk(
dir: &Path,
depth: usize,
found: &mut Vec<PathBuf>,
skipped: &mut Vec<Skipped>,
) -> Result<(), ReportError> {
if depth >= MAX_DEPTH {
return Err(ReportError::TooDeep {
path: dir.display().to_string(),
cap: MAX_DEPTH,
});
}
let entries = std::fs::read_dir(dir).map_err(|e| ReportError::Read {
path: dir.display().to_string(),
source: e,
})?;
for entry in entries {
let entry = entry.map_err(|e| ReportError::Read {
path: dir.display().to_string(),
source: e,
})?;
let path = entry.path();
let kind = entry.file_type().map_err(|e| ReportError::Read {
path: path.display().to_string(),
source: e,
})?;
if kind.is_symlink() {
if path.extension().is_some_and(|e| e == "json") {
skipped.push(Skipped {
path: path.display().to_string(),
reason: "is a symbolic link, and a directory walk does \
not follow links. Name it on the command line to \
include it"
.to_string(),
invalid_document: false,
});
}
continue;
}
if kind.is_dir() {
walk(&path, depth + 1, found, skipped)?;
} else if path.extension().is_some_and(|e| e == "json") {
found.push(path);
}
if found.len() > MAX_RESULTS {
return Err(ReportError::TooMany {
found: found.len(),
cap: MAX_RESULTS,
paths: dir.display().to_string(),
});
}
}
Ok(())
}
fn load(path: &Path) -> Result<Result1, Skipped> {
let shown = path.display().to_string();
let unreadable = |e: std::io::Error| Skipped {
path: shown.clone(),
reason: format!("cannot be read: {e}"),
invalid_document: false,
};
let file = std::fs::File::open(path).map_err(unreadable)?;
let mut text = String::new();
std::io::Read::read_to_string(&mut file.take(MAX_RESULT_BYTES + 1), &mut text)
.map_err(unreadable)?;
if text.len() as u64 > MAX_RESULT_BYTES {
return Err(Skipped {
path: shown,
reason: format!(
"is larger than the {MAX_RESULT_BYTES} byte cap this reads a \
result document up to. A real result-v1 document is a couple \
of kilobytes"
),
invalid_document: false,
});
}
let parsed: Result1 = serde_json::from_str(&text).map_err(|e| Skipped {
path: shown.clone(),
reason: format!("is not a result-v1 document: {e}"),
invalid_document: true,
})?;
parsed.validate().map_err(|problems| Skipped {
path: shown,
reason: format!("does not validate: {}", problems.join("; ")),
invalid_document: true,
})?;
Ok(parsed)
}
fn agreed(plugins: &[PluginRef], of: impl Fn(&PluginRef) -> &'static str) -> &'static str {
let mut seen = plugins.iter().map(of);
match seen.next() {
None => "unrecorded",
Some(first) if seen.all(|other| other == first) => first,
Some(_) => "mixed",
}
}
fn named(r: &Result1, want: Determinism) -> String {
let mut names: Vec<&str> = r
.provenance
.plugins
.iter()
.filter(|p| p.determinism == want)
.map(|p| p.name.as_str())
.collect();
names.sort_unstable();
names.join(", ")
}
fn to_row(source: &Path, r: Result1) -> Row {
let pinned_by = agreed(&r.provenance.plugins, |p| match p.pinned_by {
PinnedBy::ImageDigest => "image-digest",
PinnedBy::ExecutableHash => "executable-hash",
PinnedBy::Unpinned => "unpinned",
});
let isolation = agreed(&r.provenance.plugins, |p| match p.isolation {
Isolation::SandboxNoNetwork => "sandbox-no-network",
Isolation::Host => "host",
Isolation::RemoteService => "remote-service",
Isolation::Unstated => "unstated",
});
let mut flags = Vec::new();
match r.declarations.pairing {
Pairing::SingleVariable => {}
Pairing::Confounded => {
flags.push("CONFOUNDED: the pair differs in more than the payload".to_string())
}
Pairing::Unverified if r.declarations.pairing_evidence.named_a_half_not_here > 0 => flags
.push(format!(
"PAIRING UNVERIFIED: {} pair(s) name a clean half this corpus \
does not contain, so the rule could not be checked on a corpus \
that asked for it to be",
r.declarations.pairing_evidence.named_a_half_not_here
)),
Pairing::Unverified => {
flags.push("PAIRING UNVERIFIED: nothing could be compared".to_string())
}
}
if r.provenance.answers.resumed > 0 && r.provenance.answers.measured == 0 {
flags.push(format!(
"NOTHING MEASURED HERE: all {} answer(s) were already in the \
records file, so this run summarised them rather than producing \
them",
r.provenance.answers.resumed
));
}
if r.declarations.split_discipline == SplitDiscipline::ByFile {
flags.push("SPLIT BY FILE: inflates every number here".to_string());
}
if r.metrics.n_error > 0 {
flags.push(format!(
"{} image(s) unscored and not counted",
r.metrics.n_error
));
}
if r.declarations.trained_on.as_deref().is_some_and(|t| {
let t = t.trim();
t.eq_ignore_ascii_case(r.corpus.name.trim())
|| r.corpus
.id
.as_deref()
.is_some_and(|id| t.eq_ignore_ascii_case(id.trim()))
}) {
flags.push("TRAINED ON THIS CORPUS: not being measured".to_string());
} else if let Some(trained) = r.declarations.trained_on.as_deref() {
if r.declarations.contamination_check == ContaminationCheck::DeclaredButUncheckable {
flags.push(format!(
"TRAINED ON {trained}, AND THIS CORPUS COULD NOT BE IDENTIFIED: \
the two were compared by name, so whether they are the same \
corpus is unknown rather than no"
));
} else {
flags.push(format!("trained on {trained}, which is not this corpus"));
}
}
if let Some(side) = r.declarations.split_scored.as_deref() {
flags.push(format!("scored the {side} split only"));
} else if r.declarations.split_discipline != SplitDiscipline::NotApplicable {
flags.push(
"SCORED TRAIN AND TEST TOGETHER: quotable only for a detector that \
learned nothing from this corpus"
.to_string(),
);
}
if r.declarations.self_reported {
flags.push("self-reported, not re-run by anybody else".to_string());
}
if r.corpus.digest.is_empty() {
flags.push("NO CORPUS DIGEST: nobody can check which bytes this measured".to_string());
}
let varies = named(&r, Determinism::Nondeterministic);
if !varies.is_empty() {
flags.push(format!(
"NONDETERMINISTIC: two runs need not agree ({varies})"
));
}
let unchecked = named(&r, Determinism::Unstated);
if !unchecked.is_empty() {
flags.push(format!(
"determinism unstated, so reproducibility is unverified ({unchecked})"
));
}
if r.provenance.network_reachable {
flags.push("the plugins could reach the network during this run".to_string());
}
let mut per_arm = r.metrics.per_arm;
per_arm.sort_by(|a, b| a.arm.cmp(&b.arm));
Row {
source: source.display().to_string(),
detector: r.subject.name,
subject_version: r.subject.version,
pinned_by,
isolation,
corpus_name: r.corpus.name,
corpus_digest: r.corpus.digest,
corpus_source: r.corpus.source,
corpus_tier: r.corpus.tier,
corpus_pairs: r.corpus.pairs,
configuration: r.declarations.configuration,
embedder: r.arm.embedder,
rate: r.arm.rate.map(|x| (x.value, x.unit)),
domain: r.arm.domain,
format: r.arm.format,
auc: r.metrics.auc,
auc_ci95: r.metrics.auc_ci95,
tpr_at_fpr: r.metrics.tpr_at_fpr,
fpr_achieved: r.metrics.fpr_achieved,
per_arm,
pairing: r.declarations.pairing,
split: r.declarations.split_discipline,
n_clean: r.metrics.n_clean,
n_stego: r.metrics.n_stego,
n_error: r.metrics.n_error,
trained_on: r.declarations.trained_on,
self_reported: r.declarations.self_reported,
network_reachable: r.provenance.network_reachable,
harness_version: r.provenance.harness_version,
started_utc: r.provenance.started_utc,
flags,
}
}
pub fn build(paths: &[PathBuf]) -> Result<Report, ReportError> {
let (files, mut skipped) = discover(paths)?;
let mut rows: Vec<Row> = Vec::new();
for file in &files {
match load(file) {
Ok(doc) => rows.push(to_row(file, doc)),
Err(s) => skipped.push(s),
}
}
if rows.is_empty() && skipped.is_empty() {
return Err(ReportError::NothingFound {
paths: display_paths(paths),
});
}
let mut buckets: BTreeMap<(u8, String, String), Vec<Row>> = BTreeMap::new();
for row in rows {
let rank = match row.configuration {
Configuration::Named => 0,
Configuration::Custom => 1,
};
buckets
.entry((rank, row.corpus_name.clone(), row.corpus_digest.clone()))
.or_default()
.push(row);
}
let mut read = 0;
let mut groups = Vec::new();
for ((rank, corpus_name, corpus_digest), mut rows) in buckets {
rows.sort_by_key(|r| r.order());
mark_repeats(&mut rows);
read += rows.len();
groups.push(Group {
corpus_name,
corpus_digest,
configuration: if rank == 0 {
Configuration::Named
} else {
Configuration::Custom
},
rows,
});
}
skipped.sort();
Ok(Report {
groups,
skipped,
read,
asked_for: paths.iter().map(|p| p.display().to_string()).collect(),
})
}
fn mark_repeats(rows: &mut [Row]) {
let mut counts: HashMap<(String, String, String), usize> = HashMap::new();
for row in rows.iter() {
*counts
.entry((row.detector.clone(), row.embedder.clone(), row.arm()))
.or_default() += 1;
}
for row in rows.iter_mut() {
let key = (row.detector.clone(), row.embedder.clone(), row.arm());
if let Some(&n) = counts.get(&key) {
if n > 1 {
row.flags.push(format!(
"one of {n} runs of this detector on this arm in this \
report; they are separate measurements, not a best-of"
));
}
}
}
}
fn display_paths(paths: &[PathBuf]) -> String {
paths
.iter()
.map(|p| p.display().to_string())
.collect::<Vec<_>>()
.join(", ")
}
pub fn abbreviate(digest: &str) -> String {
if digest.is_empty() {
return "none".to_string();
}
let first_eight = |s: &str| s.chars().take(8).collect::<String>();
match digest.split_once(':') {
Some((algo, hex)) => {
if hex.chars().count() > 8 {
format!("{algo}:{}", first_eight(hex))
} else {
digest.to_string()
}
}
None => {
if digest.chars().count() > 8 {
first_eight(digest)
} else {
digest.to_string()
}
}
}
}
fn unit_str(unit: RateUnit) -> &'static str {
match unit {
RateUnit::Bpp => "bpp",
RateUnit::CapacityFraction => "capacity_fraction",
}
}
fn domain_str(d: Domain) -> &'static str {
match d {
Domain::Spatial => "spatial",
Domain::Jpeg => "jpeg",
Domain::Structural => "structural",
Domain::Mixed => "mixed",
Domain::Unstated => "unstated",
}
}
fn pairing_str(p: Pairing) -> &'static str {
match p {
Pairing::SingleVariable => "single-variable",
Pairing::Confounded => "confounded",
Pairing::Unverified => "unverified",
}
}
fn split_str(s: SplitDiscipline) -> &'static str {
match s {
SplitDiscipline::ByCover => "by-cover",
SplitDiscipline::ByFile => "by-file",
SplitDiscipline::NotApplicable => "not-applicable",
}
}
fn configuration_str(c: Configuration) -> &'static str {
match c {
Configuration::Named => "named",
Configuration::Custom => "custom",
}
}
fn source_str(s: CorpusSource) -> &'static str {
match s {
CorpusSource::Fetched => "fetched",
CorpusSource::Supplied => "supplied",
}
}
fn tpr_at(map: &BTreeMap<String, f64>, target: f64) -> Option<f64> {
map.iter()
.find(|(k, _)| k.parse::<f64>().is_ok_and(|v| (v - target).abs() < 1e-9))
.map(|(_, v)| *v)
}
fn budget_unexpressible(row: &Row, target: f64) -> Option<f64> {
tpr_at(&row.fpr_achieved, target).filter(|a| *a < target - 1e-9)
}
fn tpr_cell(row: &Row, target: f64) -> String {
match tpr_at(&row.tpr_at_fpr, target) {
Some(v) => match budget_unexpressible(row, target) {
Some(achieved) => format!("{v:.4} at {} FA", percent(achieved)),
None => format!("{v:.4}"),
},
None => "not reported".to_string(),
}
}
fn percent(rate: f64) -> String {
let shown = format!("{:.2}", rate * 100.0);
format!("{}%", shown.trim_end_matches('0').trim_end_matches('.'))
}
const FA_NOTE: &str = "Where a cell names a false-alarm rate after the figure, \
the run was measured at that rate rather than at the one \
its column asks for: this corpus has too few clean \
images to express that budget, so the figure is real and \
the heading is not.";
fn any_budget_unexpressible(group: &Group) -> bool {
group.rows.iter().any(|r| {
FIXED_FPR_COLUMNS
.iter()
.any(|t| budget_unexpressible(r, *t).is_some())
})
}
const FIXED_FPR_COLUMNS: [f64; 2] = [0.01, 0.10];
fn flags_cell(flags: &[String]) -> String {
if flags.is_empty() {
return "nothing flagged".to_string();
}
flags.join(" · ")
}
fn auc_cell(auc: f64, ci95: Option<[f64; 2]>) -> String {
match ci95 {
Some([lo, hi]) => format!("{auc:.4} [{lo:.3}, {hi:.3}]"),
None => format!("{auc:.4}"),
}
}
const HEADINGS: [&str; 12] = [
"detector",
"isolation",
"corpus",
"config",
"arm",
"domain",
"AUC",
"TPR@1%FA",
"TPR@10%FA",
"pairing",
"split",
"clean/stego/unscored",
];
fn cells(row: &Row) -> [String; 12] {
[
row.detector.clone(),
row.isolation.to_string(),
format!("{} @ {}", row.corpus_name, abbreviate(&row.corpus_digest)),
configuration_str(row.configuration).to_string(),
row.arm(),
domain_str(row.domain).to_string(),
auc_cell(row.auc, row.auc_ci95),
tpr_cell(row, FIXED_FPR_COLUMNS[0]),
tpr_cell(row, FIXED_FPR_COLUMNS[1]),
pairing_str(row.pairing).to_string(),
split_str(row.split).to_string(),
format!("{}/{}/{}", row.n_clean, row.n_stego, row.n_error),
]
}
const ARM_HEADINGS: [&str; 5] = ["detector", "corpus", "arm", "AUC", "clean/stego"];
fn arm_cells(row: &Row, arm: &ArmMetrics) -> [String; 5] {
[
row.detector.clone(),
format!("{} @ {}", row.corpus_name, abbreviate(&row.corpus_digest)),
arm.arm.clone(),
auc_cell(arm.auc, arm.auc_ci95),
format!("{}/{}", arm.n_clean, arm.n_stego),
]
}
const ARM_INTRO: &str = "The AUC above pools the arms this run covered, and a \
pooled figure describes none of them: chance on one \
arm beside detection on another averages to \
something in between that nothing measured.";
fn arm_table(group: &Group) -> Option<(Vec<[String; 5]>, usize)> {
let mut lines: Vec<[String; 5]> = Vec::new();
let mut withheld = 0usize;
let mut full = false;
for row in group.rows.iter().filter(|r| !r.per_arm.is_empty()) {
let fits = lines.is_empty() || lines.len() + row.per_arm.len() <= MAX_ARM_LINES_SHOWN;
if full || !fits {
full = true;
withheld += 1;
continue;
}
for arm in &row.per_arm {
lines.push(arm_cells(row, arm));
}
}
if lines.is_empty() {
return None;
}
Some((lines, withheld))
}
fn arm_withheld_note(withheld: usize) -> String {
format!(
"{withheld} further document(s) in this table carry a per-arm \
breakdown that is not printed here, because this stops at \
{MAX_ARM_LINES_SHOWN} lines. --format csv and --json carry every arm \
of every document."
)
}
fn incompleteness(report: &Report) -> Option<String> {
if report.skipped.is_empty() {
return None;
}
let mut s = format!(
"THIS REPORT IS INCOMPLETE. {} file(s) could not be turned into a \
row:\n",
report.skipped.len()
);
for skip in &report.skipped {
let _ = writeln!(s, " - {}: {}", skip.path, skip.reason);
}
Some(s)
}
fn preamble(report: &Report) -> String {
let mut s = format!(
"{} result document(s) from {}, in {} table(s).\n",
report.read,
display_list(&report.asked_for),
report.groups.len()
);
if !report.any_cross_group_comparison_is_valid() {
s.push_str("Nothing may be read across a table boundary.\n");
}
s.push_str(
"Ordered by arm then detector, never by score. Not a ranking. \
(`stegobench help reports`)\n",
);
s
}
fn render_text(report: &Report) -> String {
let mut out = String::new();
if let Some(block) = incompleteness(report) {
for line in block.lines() {
let _ = writeln!(out, "{}", wrap(line, 78));
}
out.push('\n');
}
for line in preamble(report).lines() {
let _ = writeln!(out, "{}", wrap(line, 78));
}
for group in &report.groups {
let _ = write!(
out,
"\n{} ({})\ncorpus digest: {}\n\n{}\n\n",
group.corpus_name,
configuration_str(group.configuration),
if group.corpus_digest.is_empty() {
"none, so nobody can check which bytes these numbers came from"
} else {
&group.corpus_digest
},
wrap(&group.comparability(), 78)
);
let mut table: Vec<Vec<String>> = vec![HEADINGS.iter().map(|h| h.to_string()).collect()];
for row in &group.rows {
table.push(cells(row).to_vec());
}
for line in stegobench_core::table::align_with(&table, " ") {
let _ = writeln!(out, "{line}");
}
if any_budget_unexpressible(group) {
let _ = writeln!(out, "\n{}", wrap(FA_NOTE, 78));
}
if let Some((lines, withheld)) = arm_table(group) {
let _ = write!(out, "\n{}\n\n", wrap(&format!("Per arm. {ARM_INTRO}"), 78));
let mut arms: Vec<Vec<String>> =
vec![ARM_HEADINGS.iter().map(|h| h.to_string()).collect()];
arms.extend(lines.into_iter().map(|l| l.to_vec()));
for line in stegobench_core::table::align_with(&arms, " ") {
let _ = writeln!(out, "{line}");
}
if withheld > 0 {
let _ = writeln!(out, "{}", wrap(&arm_withheld_note(withheld), 78));
}
}
let flagged: Vec<&Row> = group.rows.iter().filter(|r| !r.flags.is_empty()).collect();
out.push('\n');
if flagged.is_empty() {
let _ = writeln!(
out,
"{}",
wrap(
&format!(
"Nothing was flagged on any of the {} row(s).",
group.rows.len()
),
78
)
);
} else {
let _ = writeln!(out, "Conditions, read before quoting a figure:");
for row in &flagged {
let _ = writeln!(
out,
"{}",
wrap(
&format!(
" {} / {}: {}",
row.detector,
row.arm(),
flags_cell(&row.flags)
),
78
)
);
}
let quiet = group.rows.len() - flagged.len();
if quiet > 0 {
let _ = writeln!(
out,
"{}",
wrap(
&format!(
" the other {quiet} row(s) in this table had \
nothing flagged"
),
78
)
);
}
}
let _ = writeln!(
out,
"{}",
wrap(
" --format csv carries the full digests and each row's \
source file.",
78
)
);
}
out
}
fn render_markdown(report: &Report) -> String {
let mut out = String::from("# Steganalysis results\n\n");
if let Some(block) = incompleteness(report) {
for line in block.lines() {
let _ = writeln!(out, "> {line}");
}
out.push('\n');
}
out.push_str(&preamble(report));
out.push('\n');
for group in &report.groups {
let _ = write!(
out,
"## {} ({})\n\nCorpus digest: `{}`\n\n{}\n\n",
group.corpus_name,
configuration_str(group.configuration),
if group.corpus_digest.is_empty() {
"none"
} else {
&group.corpus_digest
},
group.comparability()
);
if group.corpus_digest.is_empty() {
out.push_str(
"This corpus carries no digest, so nobody can check which \
bytes these numbers came from.\n\n",
);
}
let mut headings: Vec<String> = HEADINGS.iter().map(|h| h.to_string()).collect();
headings.push("conditions".to_string());
let _ = writeln!(out, "| {} |", headings.join(" | "));
let _ = writeln!(
out,
"|{}|",
headings.iter().map(|_| "---").collect::<Vec<_>>().join("|")
);
for row in &group.rows {
let mut line: Vec<String> = cells(row).iter().map(|c| escape_md(c)).collect();
line.push(escape_md(&flags_cell(&row.flags)));
let _ = writeln!(out, "| {} |", line.join(" | "));
}
out.push('\n');
if any_budget_unexpressible(group) {
let _ = writeln!(out, "{FA_NOTE}\n");
}
if let Some((lines, withheld)) = arm_table(group) {
let _ = write!(out, "### Per arm\n\n{ARM_INTRO}\n\n");
let _ = writeln!(out, "| {} |", ARM_HEADINGS.join(" | "));
let _ = writeln!(
out,
"|{}|",
ARM_HEADINGS
.iter()
.map(|_| "---")
.collect::<Vec<_>>()
.join("|")
);
for line in lines {
let cells: Vec<String> = line.iter().map(|c| escape_md(c)).collect();
let _ = writeln!(out, "| {} |", cells.join(" | "));
}
let _ = writeln!(out);
if withheld > 0 {
let _ = writeln!(out, "{}\n", arm_withheld_note(withheld));
}
}
}
out
}
fn escape_md(cell: &str) -> String {
cell.replace('|', "\\|").replace('\n', " ")
}
fn render_csv(report: &Report) -> String {
let rates = fpr_columns(report);
let mut columns: Vec<String> = [
"record_type",
"corpus_name",
"corpus_digest",
"corpus_source",
"corpus_tier",
"corpus_pairs",
"configuration",
"detector",
"subject_version",
"pinned_by",
"isolation",
"embedder",
"arm_name",
"rate_value",
"rate_unit",
"domain",
"image_format",
"auc",
"auc_ci95_low",
"auc_ci95_high",
]
.iter()
.map(|c| (*c).to_string())
.collect();
for (name, _) in &rates {
columns.push(name.clone());
columns.push(format!("{name}_achieved_fpr"));
}
columns.extend(
[
"pairing",
"split_discipline",
"n_clean",
"n_stego",
"n_error",
"trained_on",
"self_reported",
"network_reachable",
"harness_version",
"started_utc",
"conditions",
"source_path",
]
.iter()
.map(|c| (*c).to_string()),
);
let mut out = String::new();
let _ = writeln!(out, "{}", columns.join(","));
for skip in &report.skipped {
let mut line = vec![String::new(); columns.len()];
line[0] = "skipped_file".into();
line[columns.len() - 2] = skip.reason.clone();
line[columns.len() - 1] = skip.path.clone();
let _ = writeln!(
out,
"{}",
line.iter()
.map(|c| csv_field(c))
.collect::<Vec<_>>()
.join(",")
);
}
for group in &report.groups {
for row in &group.rows {
let mut emit = |line: Vec<String>| {
debug_assert_eq!(line.len(), columns.len());
let _ = writeln!(
out,
"{}",
line.iter()
.map(|c| csv_field(c))
.collect::<Vec<_>>()
.join(",")
);
};
emit(csv_line(row, None, &rates));
for arm in &row.per_arm {
emit(csv_line(row, Some(arm), &rates));
}
}
}
out
}
fn csv_line(row: &Row, arm: Option<&ArmMetrics>, rates: &[(String, f64)]) -> Vec<String> {
let (rate_value, rate_unit) = match row.rate {
Some((v, u)) => (v.to_string(), unit_str(u).to_string()),
None => (String::new(), String::new()),
};
let (auc, ci95) = match arm {
Some(a) => (a.auc, a.auc_ci95),
None => (row.auc, row.auc_ci95),
};
let (ci_low, ci_high) = match ci95 {
Some([lo, hi]) => (format!("{lo:.4}"), format!("{hi:.4}")),
None => (String::new(), String::new()),
};
let mut curve: Vec<String> = Vec::with_capacity(rates.len() * 2);
for (_, rate) in rates {
let (tpr, achieved) = match arm {
Some(_) => (None, None),
None => (
tpr_at(&row.tpr_at_fpr, *rate),
tpr_at(&row.fpr_achieved, *rate),
),
};
curve.push(tpr.map(|v| format!("{v:.4}")).unwrap_or_default());
curve.push(achieved.map(|v| format!("{v:.4}")).unwrap_or_default());
}
let mut line = vec![
match arm {
Some(_) => "arm".to_string(),
None => "result".to_string(),
},
row.corpus_name.clone(),
row.corpus_digest.clone(),
source_str(row.corpus_source).to_string(),
row.corpus_tier.clone().unwrap_or_default(),
row.corpus_pairs.to_string(),
configuration_str(row.configuration).to_string(),
row.detector.clone(),
row.subject_version.clone(),
row.pinned_by.to_string(),
row.isolation.to_string(),
row.embedder.clone(),
arm.map(|a| a.arm.clone()).unwrap_or_default(),
rate_value,
rate_unit,
domain_str(row.domain).to_string(),
row.format.clone(),
format!("{auc:.4}"),
ci_low,
ci_high,
];
line.extend(curve);
line.extend([
pairing_str(row.pairing).to_string(),
split_str(row.split).to_string(),
match arm {
Some(a) => a.n_clean.to_string(),
None => row.n_clean.to_string(),
},
match arm {
Some(a) => a.n_stego.to_string(),
None => row.n_stego.to_string(),
},
match arm {
Some(_) => String::new(),
None => row.n_error.to_string(),
},
row.trained_on.clone().unwrap_or_default(),
row.self_reported.to_string(),
row.network_reachable.to_string(),
row.harness_version.clone(),
row.started_utc.clone(),
flags_cell(&row.flags),
row.source.clone(),
]);
line
}
fn fpr_columns(report: &Report) -> Vec<(String, f64)> {
let mut rates: Vec<f64> = Vec::new();
for group in &report.groups {
for row in &group.rows {
for rate in row.tpr_at_fpr.keys().filter_map(|k| k.parse::<f64>().ok()) {
if !rates.iter().any(|r| (r - rate).abs() < 1e-9) {
rates.push(rate);
}
}
}
}
rates.sort_by(|a, b| a.total_cmp(b));
rates
.into_iter()
.map(|rate| (format!("tpr_at_fpr_{rate}"), rate))
.collect()
}
fn csv_field(value: &str) -> String {
if value.contains([',', '"', '\n', '\r']) {
format!("\"{}\"", value.replace('"', "\"\""))
} else {
value.to_string()
}
}
fn wrap(text: &str, width: usize) -> String {
let indent: String = text.chars().take_while(|c| *c == ' ').collect();
let room = width.saturating_sub(indent.chars().count()).max(1);
let mut out = String::new();
let mut line = String::new();
for word in text.split_whitespace() {
if !line.is_empty() && line.chars().count() + 1 + word.chars().count() > room {
out.push_str(&indent);
out.push_str(&line);
out.push('\n');
line.clear();
}
if !line.is_empty() {
line.push(' ');
}
line.push_str(word);
}
if !line.is_empty() {
out.push_str(&indent);
out.push_str(&line);
}
out
}
fn display_list(items: &[String]) -> String {
if items.is_empty() {
return "nowhere".to_string();
}
items.join(", ")
}
pub fn render(report: &Report, format: ReportFormat) -> String {
match format {
ReportFormat::Text => render_text(report),
ReportFormat::Markdown => render_markdown(report),
ReportFormat::Csv => render_csv(report),
}
}
pub fn to_json(report: &Report) -> serde_json::Value {
serde_json::json!({
"ok": report.skipped.is_empty(),
"results_read": report.read,
"files_skipped": report.skipped.len(),
"complete": report.skipped.is_empty(),
"cross_table_comparison_valid": report.any_cross_group_comparison_is_valid(),
"skipped": report.skipped.iter().map(|s| serde_json::json!({
"path": s.path,
"reason": s.reason,
"invalid_document": s.invalid_document,
})).collect::<Vec<_>>(),
"groups": report.groups.iter().map(|g| serde_json::json!({
"corpus": { "name": g.corpus_name, "digest": g.corpus_digest },
"configuration": configuration_str(g.configuration),
"comparability": g.comparability(),
"rows": g.rows.iter().map(|r| serde_json::json!({
"source": r.source,
"detector": r.detector,
"subject_version": r.subject_version,
"pinned_by": r.pinned_by,
"isolation": r.isolation,
"corpus": {
"name": r.corpus_name,
"digest": r.corpus_digest,
"digest_short": abbreviate(&r.corpus_digest),
"source": source_str(r.corpus_source),
"tier": r.corpus_tier,
"pairs": r.corpus_pairs,
},
"configuration": configuration_str(r.configuration),
"arm": r.arm(),
"embedder": r.embedder,
"domain": domain_str(r.domain),
"format": r.format,
"auc": r.auc,
"auc_ci95": r.auc_ci95,
"tpr_at_fpr": r.tpr_at_fpr,
"fpr_achieved": r.fpr_achieved,
"per_arm": r.per_arm,
"pairing": pairing_str(r.pairing),
"split_discipline": split_str(r.split),
"n_clean": r.n_clean,
"n_stego": r.n_stego,
"n_error": r.n_error,
"trained_on": r.trained_on,
"self_reported": r.self_reported,
"network_reachable": r.network_reachable,
"harness_version": r.harness_version,
"started_utc": r.started_utc,
"conditions": r.flags,
})).collect::<Vec<_>>(),
})).collect::<Vec<_>>(),
})
}
pub fn write_atomically(path: &Path, body: &str) -> Result<(), ReportError> {
let temp = temp_path_for(path);
let write = |temp: &Path| -> std::io::Result<()> {
let mut file = std::fs::File::create(temp)?;
file.write_all(body.as_bytes())?;
file.sync_all()?;
Ok(())
};
if let Err(source) = write(&temp) {
let _ = std::fs::remove_file(&temp);
return Err(ReportError::Write {
path: path.display().to_string(),
source,
});
}
if let Err(source) = std::fs::rename(&temp, path) {
let _ = std::fs::remove_file(&temp);
return Err(ReportError::Write {
path: path.display().to_string(),
source,
});
}
Ok(())
}
fn temp_path_for(path: &Path) -> PathBuf {
let name = path
.file_name()
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_else(|| "report".to_string());
let temp_name = format!(".{name}.part-{}", std::process::id());
match path.parent().filter(|p| !p.as_os_str().is_empty()) {
Some(dir) => dir.join(temp_name),
None => PathBuf::from(temp_name),
}
}
#[cfg(test)]
mod tests {
use super::*;
fn flat(text: &str) -> String {
text.split_whitespace().collect::<Vec<_>>().join(" ")
}
fn group_named<'a>(report: &'a Report, corpus: &str) -> &'a Group {
report
.groups
.iter()
.find(|g| g.corpus_name == corpus)
.unwrap_or_else(|| panic!("no group for {corpus}"))
}
fn repo_root() -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../..")
}
fn shipped_results() -> PathBuf {
repo_root().join("results/v1")
}
struct Doc {
detector: String,
corpus: String,
digest: String,
configuration: String,
pairing: String,
split: String,
embedder: String,
auc: f64,
n_error: u64,
trained_on: Option<String>,
named_a_half_not_here: u64,
contamination_check: String,
split_scored: Option<String>,
resumed: u64,
measured: u64,
}
impl Default for Doc {
fn default() -> Self {
Doc {
detector: "detector-a".into(),
corpus: "corpus-a".into(),
digest: format!("sha256:{}", "a".repeat(64)),
configuration: "custom".into(),
pairing: "single-variable".into(),
split: "not-applicable".into(),
embedder: "suniward".into(),
auc: 0.9,
n_error: 0,
trained_on: None,
named_a_half_not_here: 0,
contamination_check: "not-declared".into(),
split_scored: None,
resumed: 0,
measured: 10,
}
}
}
impl Doc {
fn value(&self) -> serde_json::Value {
let mut declarations = serde_json::json!({
"split_discipline": self.split,
"pairing": self.pairing,
"configuration": self.configuration,
"self_reported": false,
"pairing_evidence": {
"compared": 10 - self.named_a_half_not_here,
"named_a_half_not_here": self.named_a_half_not_here,
"differed": 0,
},
});
if let Some(t) = &self.trained_on {
declarations["trained_on"] = serde_json::json!(t);
}
declarations["contamination_check"] = serde_json::json!(self.contamination_check);
if let Some(s) = &self.split_scored {
declarations["split_scored"] = serde_json::json!(s);
}
serde_json::json!({
"schema": stegobench_core::result::RESULT_SCHEMA_ID,
"subject": {
"name": self.detector,
"version": format!("sha256:{}", "b".repeat(64)),
"kind": "detector",
},
"corpus": {
"name": self.corpus,
"source": "supplied",
"digest": self.digest,
"pairs": 10,
},
"arm": {
"embedder": self.embedder,
"rate": {"value": 0.4, "unit": "bpp"},
"domain": "spatial",
"format": "png",
},
"metrics": {
"auc": self.auc,
"tpr_at_fpr": {"0.01": 0.11, "0.1": 0.55},
"n_clean": 5,
"n_stego": 5,
"n_error": self.n_error,
},
"provenance": {
"plugins": [{
"name": self.detector,
"image": format!("sha256:{}", "c".repeat(64)),
"determinism": "exact",
"pinned_by": "executable-hash",
"isolation": "host",
}],
"harness_version": "0.1.0",
"started_utc": "2026-09-27T00:00:00Z",
"elapsed_seconds": 1.0,
"network_reachable": false,
"answers": {
"measured": self.measured,
"resumed": self.resumed,
},
},
"declarations": declarations,
})
}
fn write(&self, dir: &Path, name: &str) -> PathBuf {
let path = dir.join(name);
std::fs::write(&path, serde_json::to_string_pretty(&self.value()).unwrap()).unwrap();
path
}
}
fn write_with(dir: &Path, name: &str, edit: impl FnOnce(&mut serde_json::Value)) -> PathBuf {
let mut v = Doc::default().value();
edit(&mut v);
let path = dir.join(name);
std::fs::write(&path, serde_json::to_string_pretty(&v).unwrap()).unwrap();
path
}
fn write_sealed(dir: &Path, name: &str, edit: impl FnOnce(&mut serde_json::Value)) -> PathBuf {
let mut v = Doc::default().value();
edit(&mut v);
let mut doc: Result1 = serde_json::from_value(v).expect("the fixture is a result-v1");
doc.seal();
let path = dir.join(name);
std::fs::write(&path, serde_json::to_string_pretty(&doc).unwrap()).unwrap();
path
}
fn shipped_count() -> usize {
std::fs::read_dir(shipped_results())
.expect("results/v1 is in the repository")
.filter_map(|e| e.ok())
.filter(|e| e.path().extension().is_some_and(|x| x == "json"))
.count()
}
#[test]
fn the_shipped_results_produce_a_report_in_every_format() {
let paths = vec![shipped_results()];
let report = build(&paths).expect("the shipped results build a report");
assert_eq!(report.read, shipped_count());
assert!(report.read > 0, "results/v1 is empty");
assert!(report.skipped.is_empty(), "{:?}", report.skipped);
assert_eq!(report.exit_code(), exit::OK);
for group in &report.groups {
assert!(!group.rows.is_empty());
}
for format in [
ReportFormat::Text,
ReportFormat::Markdown,
ReportFormat::Csv,
] {
let text = render(&report, format);
assert!(text.contains("round3-q95"), "{format:?}");
assert!(!text.is_empty());
}
}
#[test]
fn every_markdown_row_carries_its_conditions() {
let report = build(&[shipped_results()]).unwrap();
let md = render(&report, ReportFormat::Markdown);
let rows: Vec<&str> = md
.lines()
.filter(|l| l.starts_with("| ") && !l.contains("detector") && !l.starts_with("|---"))
.collect();
assert_eq!(rows.len(), report.read, "expected one row per result");
for row in rows {
assert!(row.contains(" @ sha256:"), "no corpus digest in: {row}");
assert!(row.contains("custom"), "no configuration in: {row}");
assert!(
row.contains("single-variable") || row.contains("confounded"),
"no pairing in: {row}"
);
assert!(row.contains("not-applicable"), "no split in: {row}");
}
}
#[test]
fn a_confounded_run_is_marked_in_its_own_row() {
let report = build(&[shipped_results()]).unwrap();
let md = render(&report, ReportFormat::Markdown);
let confounded: Vec<&str> = md
.lines()
.filter(|l| l.starts_with("| ") && l.contains("outguess"))
.collect();
assert!(!confounded.is_empty(), "no outguess rows found");
for row in confounded {
assert!(
row.contains("CONFOUNDED"),
"an outguess row does not say so: {row}"
);
}
}
#[test]
fn a_corpus_that_named_a_clean_half_it_does_not_ship_says_which_in_the_row() {
let dir = tempfile::tempdir().unwrap();
Doc {
pairing: "unverified".into(),
named_a_half_not_here: 7,
..Doc::default()
}
.write(dir.path(), "a.json");
let report = build(&[dir.path().to_path_buf()]).unwrap();
for format in [
ReportFormat::Text,
ReportFormat::Markdown,
ReportFormat::Csv,
] {
let text = flat(&render(&report, format));
assert!(
text.contains("7 pair(s) name a clean half this corpus does not contain"),
"{format:?} did not say the corpus named a half it does not ship: {text}"
);
}
}
#[test]
fn a_corpus_that_claimed_nothing_is_not_accused_of_withholding_a_clean_half() {
let dir = tempfile::tempdir().unwrap();
Doc {
pairing: "unverified".into(),
..Doc::default()
}
.write(dir.path(), "a.json");
let report = build(&[dir.path().to_path_buf()]).unwrap();
let text = flat(&render(&report, ReportFormat::Text));
assert!(text.contains("nothing could be compared"), "{text}");
assert!(!text.contains("does not contain"), "{text}");
}
#[test]
fn a_run_that_measured_nothing_says_so_in_its_own_row() {
let dir = tempfile::tempdir().unwrap();
Doc {
auc: 1.0,
measured: 0,
resumed: 10,
..Doc::default()
}
.write(dir.path(), "a.json");
let report = build(&[dir.path().to_path_buf()]).unwrap();
for format in [
ReportFormat::Text,
ReportFormat::Markdown,
ReportFormat::Csv,
] {
let text = flat(&render(&report, format));
assert!(
text.contains("NOTHING MEASURED HERE"),
"{format:?} published a resumed-only run as a measurement: {text}"
);
assert!(text.contains("all 10 answer(s)"), "{text}");
}
}
#[test]
fn a_partly_resumed_run_is_not_accused_of_measuring_nothing() {
let dir = tempfile::tempdir().unwrap();
Doc {
measured: 6,
resumed: 4,
..Doc::default()
}
.write(dir.path(), "a.json");
let report = build(&[dir.path().to_path_buf()]).unwrap();
let text = flat(&render(&report, ReportFormat::Text));
assert!(!text.contains("NOTHING MEASURED HERE"), "{text}");
}
#[test]
fn a_declaration_that_could_not_be_checked_survives_into_the_row() {
let dir = tempfile::tempdir().unwrap();
Doc {
trained_on: Some("some-other-corpus".into()),
contamination_check: "declared-but-uncheckable".into(),
..Doc::default()
}
.write(dir.path(), "a.json");
let report = build(&[dir.path().to_path_buf()]).unwrap();
for format in [
ReportFormat::Text,
ReportFormat::Markdown,
ReportFormat::Csv,
] {
let text = flat(&render(&report, format));
assert!(
text.contains("TRAINED ON some-other-corpus"),
"{format:?} dropped the declaration: {text}"
);
assert!(
text.contains("unknown rather than no"),
"{format:?} read an unchecked declaration as a clean one: {text}"
);
}
}
#[test]
fn a_checked_declaration_against_a_different_corpus_is_stated_quietly() {
let dir = tempfile::tempdir().unwrap();
Doc {
trained_on: Some("bossbase".into()),
contamination_check: "checked".into(),
..Doc::default()
}
.write(dir.path(), "a.json");
let report = build(&[dir.path().to_path_buf()]).unwrap();
let text = flat(&render(&report, ReportFormat::Text));
assert!(
text.contains("trained on bossbase, which is not this corpus"),
"{text}"
);
assert!(!text.contains("unknown rather than no"), "{text}");
}
#[test]
fn the_row_says_which_side_of_the_split_was_scored() {
let dir = tempfile::tempdir().unwrap();
Doc {
split: "by-cover".into(),
split_scored: Some("test".into()),
..Doc::default()
}
.write(dir.path(), "a.json");
let report = build(&[dir.path().to_path_buf()]).unwrap();
let text = flat(&render(&report, ReportFormat::Text));
assert!(text.contains("scored the test split only"), "{text}");
assert!(!text.contains("SCORED TRAIN AND TEST TOGETHER"), "{text}");
}
#[test]
fn a_run_over_both_halves_of_a_split_corpus_says_so() {
let dir = tempfile::tempdir().unwrap();
Doc {
split: "by-cover".into(),
..Doc::default()
}
.write(dir.path(), "a.json");
let report = build(&[dir.path().to_path_buf()]).unwrap();
let text = flat(&render(&report, ReportFormat::Text));
assert!(text.contains("SCORED TRAIN AND TEST TOGETHER"), "{text}");
}
#[test]
fn an_unverified_pairing_is_marked_in_the_row() {
let dir = tempfile::tempdir().unwrap();
Doc {
pairing: "unverified".into(),
..Doc::default()
}
.write(dir.path(), "a.json");
let report = build(&[dir.path().to_path_buf()]).unwrap();
for format in [
ReportFormat::Text,
ReportFormat::Markdown,
ReportFormat::Csv,
] {
assert!(
flat(&render(&report, format)).contains("PAIRING UNVERIFIED"),
"{format:?} hid an unverified pairing"
);
}
}
#[test]
fn a_by_file_split_is_marked_because_it_inflates_the_number() {
let dir = tempfile::tempdir().unwrap();
Doc {
split: "by-file".into(),
..Doc::default()
}
.write(dir.path(), "a.json");
let report = build(&[dir.path().to_path_buf()]).unwrap();
assert!(flat(&render(&report, ReportFormat::Text)).contains("SPLIT BY FILE"));
}
#[test]
fn unscored_images_and_contamination_reach_the_row() {
let dir = tempfile::tempdir().unwrap();
Doc {
n_error: 7,
trained_on: Some("corpus-a".into()),
..Doc::default()
}
.write(dir.path(), "a.json");
let report = build(&[dir.path().to_path_buf()]).unwrap();
let text = flat(&render(&report, ReportFormat::Text));
assert!(text.contains("7 image(s) unscored"), "{text}");
assert!(text.contains("TRAINED ON THIS CORPUS"), "{text}");
}
#[test]
fn a_detector_trained_on_a_different_corpus_is_not_flagged_as_contaminated() {
let dir = tempfile::tempdir().unwrap();
Doc {
trained_on: Some("somewhere-else".into()),
..Doc::default()
}
.write(dir.path(), "a.json");
let report = build(&[dir.path().to_path_buf()]).unwrap();
assert!(!flat(&render(&report, ReportFormat::Text)).contains("TRAINED ON THIS CORPUS"));
}
#[test]
fn two_corpora_never_share_a_table() {
let dir = tempfile::tempdir().unwrap();
Doc {
corpus: "one".into(),
..Doc::default()
}
.write(dir.path(), "a.json");
Doc {
corpus: "two".into(),
digest: format!("sha256:{}", "d".repeat(64)),
..Doc::default()
}
.write(dir.path(), "b.json");
let report = build(&[dir.path().to_path_buf()]).unwrap();
assert_eq!(report.groups.len(), 2);
assert!(!report.any_cross_group_comparison_is_valid());
assert!(flat(&render(&report, ReportFormat::Text))
.contains("Nothing may be read across a table boundary"));
}
#[test]
fn one_corpus_name_with_two_digests_is_two_tables() {
let dir = tempfile::tempdir().unwrap();
Doc::default().write(dir.path(), "a.json");
Doc {
digest: format!("sha256:{}", "e".repeat(64)),
..Doc::default()
}
.write(dir.path(), "b.json");
let report = build(&[dir.path().to_path_buf()]).unwrap();
assert_eq!(report.groups.len(), 2, "one name, two digests, one table");
}
#[test]
fn named_and_custom_never_interleave_and_named_leads() {
let dir = tempfile::tempdir().unwrap();
Doc {
configuration: "named".into(),
..Doc::default()
}
.write(dir.path(), "a.json");
Doc {
configuration: "custom".into(),
detector: "detector-b".into(),
..Doc::default()
}
.write(dir.path(), "b.json");
let report = build(&[dir.path().to_path_buf()]).unwrap();
assert_eq!(report.groups.len(), 2);
assert_eq!(report.groups[0].configuration, Configuration::Named);
assert_eq!(report.groups[1].configuration, Configuration::Custom);
for group in &report.groups {
assert_eq!(group.rows.len(), 1, "a group mixed the two");
}
let text = flat(&render(&report, ReportFormat::Text));
assert!(text.contains("Quote these figures beside nothing else"));
}
#[test]
fn a_malformed_json_file_is_named_rather_than_vanishing() {
let dir = tempfile::tempdir().unwrap();
Doc::default().write(dir.path(), "good.json");
std::fs::write(dir.path().join("bad.json"), "{not json").unwrap();
let report = build(&[dir.path().to_path_buf()]).unwrap();
assert_eq!(report.read, 1);
assert_eq!(report.skipped.len(), 1);
assert!(report.skipped[0].invalid_document);
assert_eq!(report.exit_code(), exit::SCHEMA_INVALID);
for format in [
ReportFormat::Text,
ReportFormat::Markdown,
ReportFormat::Csv,
] {
let text = render(&report, format);
assert!(text.contains("bad.json"), "{format:?} lost the bad file");
}
assert!(flat(&render(&report, ReportFormat::Text)).contains("THIS REPORT IS INCOMPLETE"));
assert!(render(&report, ReportFormat::Csv).contains("skipped_file"));
}
#[test]
fn valid_json_that_is_not_a_result_is_named_with_its_reason() {
let dir = tempfile::tempdir().unwrap();
Doc::default().write(dir.path(), "good.json");
std::fs::write(dir.path().join("other.json"), r#"{"hello":"world"}"#).unwrap();
let report = build(&[dir.path().to_path_buf()]).unwrap();
assert_eq!(report.skipped.len(), 1);
assert!(report.skipped[0]
.reason
.contains("not a result-v1 document"));
assert_eq!(report.exit_code(), exit::SCHEMA_INVALID);
}
#[test]
fn a_result_that_parses_but_does_not_validate_is_refused_from_the_table() {
let dir = tempfile::tempdir().unwrap();
let mut v = Doc::default().value();
v["metrics"]["auc"] = serde_json::json!(1.4);
std::fs::write(
dir.path().join("impossible.json"),
serde_json::to_string(&v).unwrap(),
)
.unwrap();
Doc::default().write(dir.path(), "good.json");
let report = build(&[dir.path().to_path_buf()]).unwrap();
assert_eq!(report.read, 1);
assert_eq!(report.skipped.len(), 1);
assert!(report.skipped[0].reason.contains("does not validate"));
assert!(!render(&report, ReportFormat::Markdown).contains("1.4000"));
}
#[test]
fn a_file_over_the_size_cap_is_skipped_with_the_cap_named() {
let dir = tempfile::tempdir().unwrap();
let big = "x".repeat((MAX_RESULT_BYTES + 1) as usize);
std::fs::write(dir.path().join("huge.json"), big).unwrap();
let report = build(&[dir.path().to_path_buf()]).unwrap();
assert!(report.groups.is_empty());
assert_eq!(report.skipped.len(), 1);
assert!(report.skipped[0].reason.contains("byte cap"));
assert_eq!(report.exit_code(), exit::FAILURE);
}
#[test]
fn an_empty_directory_refuses_rather_than_printing_an_empty_table() {
let dir = tempfile::tempdir().unwrap();
let err = build(&[dir.path().to_path_buf()]).unwrap_err();
assert!(matches!(err, ReportError::NothingFound { .. }));
assert_eq!(err.exit_code(), exit::USAGE);
assert!(err.to_string().contains("no table to print"));
}
#[test]
fn an_empty_input_set_refuses() {
let err = build(&[]).unwrap_err();
assert!(matches!(err, ReportError::NothingFound { .. }));
assert!(err.to_string().contains("nowhere") || err.to_string().contains("nothing under"));
}
#[test]
fn a_directory_of_non_json_files_refuses() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("README.md"), "not a result").unwrap();
let err = build(&[dir.path().to_path_buf()]).unwrap_err();
assert!(matches!(err, ReportError::NothingFound { .. }));
}
#[test]
fn a_missing_path_refuses_the_way_score_refuses_one() {
let err = build(&[PathBuf::from("/definitely/not/here")]).unwrap_err();
assert!(matches!(err, ReportError::PathUnusable { .. }));
assert_eq!(err.exit_code(), exit::PREFLIGHT_REFUSED);
let said = err.to_string();
assert!(said.contains("/definitely/not/here"), "{said}");
assert!(said.contains("nothing is there"), "{said}");
assert!(!said.contains("os error"), "{said}");
}
#[test]
fn a_named_file_without_a_json_extension_is_still_read() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("result.txt");
std::fs::write(
&path,
serde_json::to_string(&Doc::default().value()).unwrap(),
)
.unwrap();
let report = build(&[path]).unwrap();
assert_eq!(report.read, 1);
}
#[test]
fn a_tree_deeper_than_the_cap_refuses() {
let dir = tempfile::tempdir().unwrap();
let mut deep = dir.path().to_path_buf();
for i in 0..(MAX_DEPTH + 1) {
deep = deep.join(format!("d{i}"));
}
std::fs::create_dir_all(&deep).unwrap();
let err = build(&[dir.path().to_path_buf()]).unwrap_err();
assert!(matches!(err, ReportError::TooDeep { .. }));
assert_eq!(err.exit_code(), exit::PREFLIGHT_REFUSED);
}
#[test]
fn a_nested_directory_within_the_cap_is_walked() {
let dir = tempfile::tempdir().unwrap();
let nested = dir.path().join("a").join("b");
std::fs::create_dir_all(&nested).unwrap();
Doc::default().write(&nested, "r.json");
let report = build(&[dir.path().to_path_buf()]).unwrap();
assert_eq!(report.read, 1);
}
#[test]
fn two_runs_over_the_same_input_are_byte_identical() {
let dir = tempfile::tempdir().unwrap();
for (i, name) in ["z", "m", "a", "q"].iter().enumerate() {
Doc {
detector: format!("detector-{name}"),
auc: 0.5 + i as f64 / 100.0,
..Doc::default()
}
.write(dir.path(), &format!("{name}.json"));
}
for format in [
ReportFormat::Text,
ReportFormat::Markdown,
ReportFormat::Csv,
] {
let first = render(&build(&[dir.path().to_path_buf()]).unwrap(), format);
let second = render(&build(&[dir.path().to_path_buf()]).unwrap(), format);
assert_eq!(first, second, "{format:?} is not reproducible");
}
let a = render(&build(&[shipped_results()]).unwrap(), ReportFormat::Csv);
let b = render(&build(&[shipped_results()]).unwrap(), ReportFormat::Csv);
assert_eq!(a, b);
}
#[test]
fn a_path_named_twice_is_read_once() {
let dir = tempfile::tempdir().unwrap();
Doc::default().write(dir.path(), "a.json");
let report = build(&[dir.path().to_path_buf(), dir.path().to_path_buf()]).unwrap();
assert_eq!(report.read, 1);
}
#[test]
fn rows_are_ordered_by_arm_and_detector_rather_than_by_score() {
let dir = tempfile::tempdir().unwrap();
Doc {
detector: "zeta".into(),
auc: 0.99,
..Doc::default()
}
.write(dir.path(), "a.json");
Doc {
detector: "alpha".into(),
auc: 0.51,
..Doc::default()
}
.write(dir.path(), "b.json");
let report = build(&[dir.path().to_path_buf()]).unwrap();
let names: Vec<&str> = report.groups[0]
.rows
.iter()
.map(|r| r.detector.as_str())
.collect();
assert_eq!(names, ["alpha", "zeta"], "the table was ranked by score");
assert!(flat(&render(&report, ReportFormat::Text)).contains("never by score"));
}
#[test]
fn two_runs_of_one_detector_on_one_arm_are_marked_as_separate_measurements() {
let dir = tempfile::tempdir().unwrap();
Doc::default().write(dir.path(), "a.json");
Doc {
auc: 0.7,
..Doc::default()
}
.write(dir.path(), "b.json");
let report = build(&[dir.path().to_path_buf()]).unwrap();
assert_eq!(report.groups[0].rows.len(), 2);
let text = flat(&render(&report, ReportFormat::Text));
assert!(text.contains("one of 2 runs"), "{text}");
}
#[test]
fn a_corpus_with_no_digest_says_so_in_the_row_and_in_the_heading() {
let dir = tempfile::tempdir().unwrap();
Doc {
digest: String::new(),
..Doc::default()
}
.write(dir.path(), "a.json");
let report = build(&[dir.path().to_path_buf()]).unwrap();
let text = flat(&render(&report, ReportFormat::Text));
assert!(text.contains("NO CORPUS DIGEST"), "{text}");
assert!(text.contains("nobody can check which bytes"), "{text}");
assert!(render(&report, ReportFormat::Markdown).contains("carries no digest"));
}
#[test]
fn a_missing_point_on_the_curve_is_not_reported_rather_than_zero() {
let dir = tempfile::tempdir().unwrap();
let mut v = Doc::default().value();
v["metrics"]["tpr_at_fpr"] = serde_json::json!({"0.05": 0.3});
std::fs::write(
dir.path().join("a.json"),
serde_json::to_string(&v).unwrap(),
)
.unwrap();
let report = build(&[dir.path().to_path_buf()]).unwrap();
let text = flat(&render(&report, ReportFormat::Text));
assert!(text.contains("not reported"), "{text}");
assert!(
!text.contains("0.0000"),
"a missing point read as zero: {text}"
);
}
#[test]
fn a_rate_carries_its_unit_into_the_arm_cell() {
let dir = tempfile::tempdir().unwrap();
let mut v = Doc::default().value();
v["arm"]["rate"] = serde_json::json!({"value": 0.05, "unit": "capacity_fraction"});
std::fs::write(
dir.path().join("a.json"),
serde_json::to_string(&v).unwrap(),
)
.unwrap();
let mut w = Doc::default().value();
w["arm"].as_object_mut().unwrap().remove("rate");
w["arm"]["embedder"] = serde_json::json!("structural");
std::fs::write(
dir.path().join("b.json"),
serde_json::to_string(&w).unwrap(),
)
.unwrap();
let report = build(&[dir.path().to_path_buf()]).unwrap();
let text = flat(&render(&report, ReportFormat::Text));
assert!(text.contains("5.000% of capacity"), "{text}");
assert!(text.contains("structural"), "{text}");
assert!(
!text.contains("structural at"),
"a rateless arm invented one"
);
}
#[test]
fn the_nondeterminism_warning_names_the_plugin_it_is_about() {
let dir = tempfile::tempdir().unwrap();
let mut v = Doc::default().value();
v["provenance"]["plugins"] = serde_json::json!([
{"name": "steady", "image": format!("sha256:{}", "c".repeat(64)), "determinism": "exact", "pinned_by": "executable-hash", "isolation": "host"},
{"name": "wobbly", "image": format!("sha256:{}", "d".repeat(64)), "determinism": "nondeterministic", "pinned_by": "executable-hash", "isolation": "host"},
]);
std::fs::write(
dir.path().join("a.json"),
serde_json::to_string(&v).unwrap(),
)
.unwrap();
let text = flat(&render(
&build(&[dir.path().to_path_buf()]).unwrap(),
ReportFormat::Text,
));
assert!(text.contains("NONDETERMINISTIC"), "{text}");
assert!(
text.contains("wobbly"),
"the warning should name it: {text}"
);
assert!(
!text.contains("NONDETERMINISTIC: two runs need not agree (steady"),
"the deterministic plugin was accused: {text}"
);
}
#[test]
fn a_report_of_deterministic_plugins_carries_no_determinism_warning() {
let dir = tempfile::tempdir().unwrap();
let mut v = Doc::default().value();
v["provenance"]["plugins"] = serde_json::json!([
{"name": "steady", "image": format!("sha256:{}", "c".repeat(64)), "determinism": "exact", "pinned_by": "executable-hash", "isolation": "host"},
]);
std::fs::write(
dir.path().join("a.json"),
serde_json::to_string(&v).unwrap(),
)
.unwrap();
let text = flat(&render(
&build(&[dir.path().to_path_buf()]).unwrap(),
ReportFormat::Text,
));
assert!(!text.contains("NONDETERMINISTIC"), "{text}");
assert!(!text.contains("determinism unstated"), "{text}");
}
#[test]
fn nobody_having_checked_is_said_differently_from_having_checked_and_found_drift() {
let dir = tempfile::tempdir().unwrap();
let mut v = Doc::default().value();
v["provenance"]["plugins"] = serde_json::json!([
{"name": "unchecked", "image": format!("sha256:{}", "c".repeat(64)), "determinism": "unstated", "pinned_by": "executable-hash", "isolation": "host"},
]);
std::fs::write(
dir.path().join("a.json"),
serde_json::to_string(&v).unwrap(),
)
.unwrap();
let text = flat(&render(
&build(&[dir.path().to_path_buf()]).unwrap(),
ReportFormat::Text,
));
assert!(text.contains("determinism unstated"), "{text}");
assert!(text.contains("unchecked"), "{text}");
assert!(
!text.contains("NONDETERMINISTIC"),
"an unmeasured tool was called nondeterministic: {text}"
);
}
#[test]
fn plugins_that_disagree_are_said_to_be_mixed_rather_than_picked_between() {
let dir = tempfile::tempdir().unwrap();
let mut v = Doc::default().value();
v["provenance"]["plugins"] = serde_json::json!([
{"name": "a", "image": format!("sha256:{}", "c".repeat(64)), "determinism": "exact", "pinned_by": "executable-hash", "isolation": "host"},
{"name": "b", "image": format!("ghcr.io/x/y@sha256:{}", "c".repeat(64)), "determinism": "nondeterministic", "pinned_by": "image-digest", "isolation": "sandbox-no-network"},
]);
v["provenance"]["network_reachable"] = serde_json::json!(true);
std::fs::write(
dir.path().join("a.json"),
serde_json::to_string(&v).unwrap(),
)
.unwrap();
let report = build(&[dir.path().to_path_buf()]).unwrap();
assert_eq!(report.groups[0].rows[0].pinned_by, "mixed");
assert_eq!(report.groups[0].rows[0].isolation, "mixed");
assert!(flat(&render(&report, ReportFormat::Text)).contains("NONDETERMINISTIC"));
}
#[test]
fn a_network_reachable_run_says_so() {
let dir = tempfile::tempdir().unwrap();
let mut v = Doc::default().value();
v["provenance"]["network_reachable"] = serde_json::json!(true);
std::fs::write(
dir.path().join("a.json"),
serde_json::to_string(&v).unwrap(),
)
.unwrap();
let report = build(&[dir.path().to_path_buf()]).unwrap();
assert!(flat(&render(&report, ReportFormat::Text)).contains("reach the network"));
}
#[test]
fn a_self_reported_run_says_so() {
let dir = tempfile::tempdir().unwrap();
let mut v = Doc::default().value();
v["declarations"]["self_reported"] = serde_json::json!(true);
std::fs::write(
dir.path().join("a.json"),
serde_json::to_string(&v).unwrap(),
)
.unwrap();
let report = build(&[dir.path().to_path_buf()]).unwrap();
assert!(render(&report, ReportFormat::Csv).contains("self-reported"));
}
#[test]
fn a_row_with_nothing_flagged_does_not_claim_to_be_clean() {
let dir = tempfile::tempdir().unwrap();
Doc::default().write(dir.path(), "a.json");
let report = build(&[dir.path().to_path_buf()]).unwrap();
let text = flat(&render(&report, ReportFormat::Text));
assert!(text.contains("Nothing was flagged"), "{text}");
assert!(
!text.contains(" ok ") && !text.contains("clean run"),
"{text}"
);
assert!(render(&report, ReportFormat::Markdown).contains("| nothing flagged |"));
}
#[test]
fn the_text_conditions_list_only_the_rows_with_something_to_say() {
let report = build(&[shipped_results()]).unwrap();
let text = flat(&render(&report, ReportFormat::Text));
assert!(text.contains("Conditions, read before quoting"), "{text}");
let jpeg = group_named(&report, "round3-q95");
let quiet = jpeg.rows.iter().filter(|r| r.flags.is_empty()).count();
assert!(quiet > 0 && quiet < jpeg.rows.len(), "fixture has no mix");
assert!(
text.contains(&format!("the other {quiet} row(s) in this table")),
"{text}"
);
}
#[test]
fn a_table_with_nothing_flagged_says_so_rather_than_printing_nothing() {
let dir = tempfile::tempdir().unwrap();
for (i, detector) in ["detector-a", "detector-b"].iter().enumerate() {
Doc {
detector: (*detector).into(),
..Doc::default()
}
.write(dir.path(), &format!("{i}.json"));
}
let report = build(&[dir.path().to_path_buf()]).unwrap();
let group = group_named(&report, "corpus-a");
assert!(group.rows.iter().all(|r| r.flags.is_empty()));
let text = flat(&render(&report, ReportFormat::Text));
assert!(
text.contains(&format!(
"Nothing was flagged on any of the {} row(s)",
group.rows.len()
)),
"{text}"
);
}
#[test]
fn the_csv_quotes_a_field_that_would_otherwise_break_the_row() {
assert_eq!(csv_field("plain"), "plain");
assert_eq!(csv_field("a,b"), "\"a,b\"");
assert_eq!(csv_field("say \"hi\""), "\"say \"\"hi\"\"\"");
assert_eq!(csv_field("two\nlines"), "\"two\nlines\"");
}
#[test]
fn the_csv_has_one_field_per_column_on_every_row() {
let report = build(&[shipped_results()]).unwrap();
let csv = render(&report, ReportFormat::Csv);
let mut lines = csv.lines();
let columns = lines.next().unwrap().split(',').count();
for line in lines {
assert_eq!(
line.split(',').count(),
columns,
"row has the wrong number of fields: {line}"
);
}
}
#[test]
fn a_forged_document_is_named_as_skipped_rather_than_tabled() {
let dir = tempfile::tempdir().unwrap();
let path = write_sealed(dir.path(), "a.json", |_| {});
let mut v: serde_json::Value =
serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
v["metrics"]["auc"] = serde_json::json!(0.99);
std::fs::write(&path, serde_json::to_string_pretty(&v).unwrap()).unwrap();
let report = build(&[dir.path().to_path_buf()]).unwrap();
assert!(report.groups.is_empty(), "the forgery made a table");
assert_eq!(report.skipped.len(), 1, "{:?}", report.skipped);
assert!(report.skipped[0].invalid_document);
assert!(
report.skipped[0]
.reason
.contains("does not match its own content digest"),
"{:?}",
report.skipped[0]
);
assert_eq!(report.exit_code(), exit::SCHEMA_INVALID);
for format in [
ReportFormat::Text,
ReportFormat::Markdown,
ReportFormat::Csv,
] {
let out = render(&report, format);
assert!(!out.contains("0.9900"), "{format:?} tabled the forgery");
assert!(
flat(&out).contains("content digest"),
"{format:?} did not name it"
);
}
}
#[test]
fn a_sealed_document_nobody_touched_makes_an_ordinary_row() {
let dir = tempfile::tempdir().unwrap();
write_sealed(dir.path(), "a.json", |_| {});
let report = build(&[dir.path().to_path_buf()]).unwrap();
assert!(report.skipped.is_empty(), "{:?}", report.skipped);
assert_eq!(report.exit_code(), exit::OK);
}
#[test]
fn an_auc_outside_its_own_interval_never_reaches_a_table() {
let dir = tempfile::tempdir().unwrap();
write_with(dir.path(), "a.json", |v| {
v["metrics"]["auc"] = serde_json::json!(0.99);
v["metrics"]["auc_ci95"] = serde_json::json!([0.5, 0.5]);
});
let report = build(&[dir.path().to_path_buf()]).unwrap();
assert!(report.groups.is_empty());
assert_eq!(report.exit_code(), exit::SCHEMA_INVALID);
assert!(
report.skipped[0]
.reason
.contains("outside its own interval"),
"{:?}",
report.skipped[0]
);
}
#[test]
fn a_budget_the_corpus_could_not_express_is_said_in_the_cell() {
let dir = tempfile::tempdir().unwrap();
write_with(dir.path(), "a.json", |v| {
v["metrics"]["fpr_achieved"] = serde_json::json!({"0.01": 0.0, "0.1": 0.1});
});
let report = build(&[dir.path().to_path_buf()]).unwrap();
for format in [ReportFormat::Text, ReportFormat::Markdown] {
let out = flat(&render(&report, format));
assert!(out.contains("0.1100 at 0% FA"), "{format:?}: {out}");
assert!(out.contains("0.5500 "), "{format:?}: {out}");
assert!(!out.contains("0.5500 at"), "{format:?}: {out}");
assert!(out.contains("too few clean images"), "{format:?}: {out}");
}
}
#[test]
fn a_document_that_recorded_no_achieved_rates_renders_as_it_always_did() {
let dir = tempfile::tempdir().unwrap();
Doc::default().write(dir.path(), "a.json");
let report = build(&[dir.path().to_path_buf()]).unwrap();
for format in [ReportFormat::Text, ReportFormat::Markdown] {
let out = flat(&render(&report, format));
assert!(!out.contains(" FA"), "{format:?} invented a mark: {out}");
assert!(!out.contains("too few clean images"), "{format:?}: {out}");
}
}
#[test]
fn the_csv_gives_every_false_alarm_rate_its_own_column() {
let dir = tempfile::tempdir().unwrap();
write_with(dir.path(), "a.json", |v| {
v["metrics"]["tpr_at_fpr"] = serde_json::json!({"0.01": 0.11, "0.1": 0.55});
v["metrics"]["fpr_achieved"] = serde_json::json!({"0.01": 0.0, "0.1": 0.1});
});
let report = build(&[dir.path().to_path_buf()]).unwrap();
let csv = render(&report, ReportFormat::Csv);
let header: Vec<&str> = csv.lines().next().unwrap().split(',').collect();
for column in [
"tpr_at_fpr_0.01",
"tpr_at_fpr_0.01_achieved_fpr",
"tpr_at_fpr_0.1",
"tpr_at_fpr_0.1_achieved_fpr",
] {
assert!(header.contains(&column), "no {column} in {header:?}");
}
assert!(!header.contains(&"tpr_at_fpr"), "the packed cell survived");
let row: Vec<&str> = csv.lines().nth(1).unwrap().split(',').collect();
let at = |name: &str| row[header.iter().position(|h| *h == name).unwrap()];
assert_eq!(at("tpr_at_fpr_0.01"), "0.1100");
assert_eq!(at("tpr_at_fpr_0.01_achieved_fpr"), "0.0000");
assert_eq!(at("tpr_at_fpr_0.1"), "0.5500");
assert_eq!(at("tpr_at_fpr_0.1_achieved_fpr"), "0.1000");
}
#[test]
fn the_csv_columns_cover_the_rates_of_every_document_not_just_the_first() {
let dir = tempfile::tempdir().unwrap();
write_with(dir.path(), "a.json", |v| {
v["metrics"]["tpr_at_fpr"] = serde_json::json!({"0.01": 0.11});
});
write_with(dir.path(), "b.json", |v| {
v["subject"]["name"] = serde_json::json!("detector-b");
v["metrics"]["tpr_at_fpr"] = serde_json::json!({"0.05": 0.33});
});
let report = build(&[dir.path().to_path_buf()]).unwrap();
let csv = render(&report, ReportFormat::Csv);
let header: Vec<&str> = csv.lines().next().unwrap().split(',').collect();
assert!(header.contains(&"tpr_at_fpr_0.01"), "{header:?}");
assert!(header.contains(&"tpr_at_fpr_0.05"), "{header:?}");
assert!(
csv.contains("0.3300"),
"the second document's figure: {csv}"
);
let a_line = csv
.lines()
.find(|l| l.contains("detector-a"))
.expect("a row for detector-a");
let cells: Vec<&str> = a_line.split(',').collect();
let at = |name: &str| cells[header.iter().position(|h| *h == name).unwrap()];
assert_eq!(at("tpr_at_fpr_0.05"), "");
}
#[test]
fn a_csv_arm_line_leaves_the_curve_columns_empty() {
let dir = tempfile::tempdir().unwrap();
with_arms(dir.path(), two_arms());
let report = build(&[dir.path().to_path_buf()]).unwrap();
let csv = render(&report, ReportFormat::Csv);
let header: Vec<&str> = csv.lines().next().unwrap().split(',').collect();
let index = header
.iter()
.position(|h| *h == "tpr_at_fpr_0.01")
.expect("a column for the rate the fixture reports");
for line in csv.lines().filter(|l| l.starts_with("arm,")) {
assert_eq!(line.split(',').nth(index).unwrap(), "", "{line}");
}
}
#[test]
fn a_pipe_in_a_cell_cannot_shift_a_markdown_row() {
assert_eq!(escape_md("a|b"), "a\\|b");
assert_eq!(escape_md("a\nb"), "a b");
}
#[test]
fn abbreviating_a_digest_keeps_the_algorithm_and_refuses_to_invent_one() {
assert_eq!(abbreviate(""), "none");
assert_eq!(abbreviate("sha256:0123456789abcdef"), "sha256:01234567");
assert_eq!(abbreviate("short"), "short");
assert_eq!(abbreviate("0123456789"), "01234567");
assert_eq!(abbreviate("sha256:abc"), "sha256:abc");
}
#[test]
fn the_atomic_write_leaves_the_whole_file_and_no_part_file() {
let dir = tempfile::tempdir().unwrap();
let out = dir.path().join("report.md");
write_atomically(&out, "first").unwrap();
assert_eq!(std::fs::read_to_string(&out).unwrap(), "first");
write_atomically(&out, "second").unwrap();
assert_eq!(std::fs::read_to_string(&out).unwrap(), "second");
let leftovers: Vec<_> = std::fs::read_dir(dir.path())
.unwrap()
.map(|e| e.unwrap().file_name().to_string_lossy().into_owned())
.filter(|n| n.contains(".part-"))
.collect();
assert!(leftovers.is_empty(), "left behind: {leftovers:?}");
}
#[test]
fn an_unwritable_destination_fails_with_the_path_and_leaves_nothing() {
let dir = tempfile::tempdir().unwrap();
let out = dir.path().join("no-such-dir").join("report.md");
let err = write_atomically(&out, "body").unwrap_err();
assert!(matches!(err, ReportError::Write { .. }));
assert_eq!(err.exit_code(), exit::FAILURE);
assert!(err.to_string().contains("report.md"));
assert!(!dir.path().join("no-such-dir").exists());
}
#[test]
fn a_destination_with_no_parent_gets_a_temporary_file_beside_it() {
let bare = temp_path_for(Path::new("bare.md"));
assert_eq!(bare.parent(), Some(Path::new("")));
assert!(bare
.file_name()
.is_some_and(|n| n.to_string_lossy().starts_with(".bare.md.part-")));
let nested = temp_path_for(Path::new("/tmp/some/report.md"));
assert_eq!(nested.parent(), Some(Path::new("/tmp/some")));
assert!(nested
.file_name()
.is_some_and(|n| n.to_string_lossy().starts_with(".report.md.part-")));
}
#[test]
fn the_json_output_says_whether_the_table_is_complete() {
let dir = tempfile::tempdir().unwrap();
Doc::default().write(dir.path(), "a.json");
std::fs::write(dir.path().join("bad.json"), "{").unwrap();
let report = build(&[dir.path().to_path_buf()]).unwrap();
let json = to_json(&report);
assert_eq!(json["complete"], serde_json::json!(false));
assert_eq!(json["files_skipped"], serde_json::json!(1));
assert_eq!(json["results_read"], serde_json::json!(1));
assert_eq!(
json["groups"][0]["rows"][0]["conditions"]
.as_array()
.map(|a| a.len()),
Some(0)
);
assert!(json["groups"][0]["comparability"]
.as_str()
.is_some_and(|s| s.starts_with("NAMED:") || s.starts_with("CUSTOM:")));
assert!(json["groups"][0]["rows"][0]["corpus"]["digest"]
.as_str()
.is_some_and(|d| d.len() > 8));
}
#[test]
fn the_json_marks_a_single_table_as_internally_comparable() {
let dir = tempfile::tempdir().unwrap();
Doc::default().write(dir.path(), "a.json");
let report = build(&[dir.path().to_path_buf()]).unwrap();
assert_eq!(
to_json(&report)["cross_table_comparison_valid"],
serde_json::json!(true)
);
}
#[test]
fn wrapping_keeps_every_word_and_respects_the_column() {
let text = "one two three four five six seven eight nine ten";
let wrapped = wrap(text, 12);
for line in wrapped.lines() {
assert!(line.chars().count() <= 12, "too long: {line:?}");
}
assert_eq!(
wrapped.split_whitespace().collect::<Vec<_>>(),
text.split_whitespace().collect::<Vec<_>>()
);
assert_eq!(wrap("", 10), "");
assert_eq!(
wrap("antidisestablishmentarianism", 5),
"antidisestablishmentarianism"
);
}
#[test]
fn display_helpers_say_nowhere_rather_than_nothing() {
assert_eq!(display_list(&[]), "nowhere");
assert_eq!(display_paths(&[]), "");
assert_eq!(display_list(&["a".to_string(), "b".to_string()]), "a, b");
}
#[test]
fn tpr_lookup_matches_a_rate_however_it_was_written() {
let map = BTreeMap::from([("0.10".to_string(), 0.9), ("0.010".to_string(), 0.4)]);
assert_eq!(tpr_at(&map, 0.10), Some(0.9));
assert_eq!(tpr_at(&map, 0.01), Some(0.4));
assert_eq!(tpr_at(&map, 0.05), None);
}
#[test]
fn a_rate_written_two_ways_is_one_csv_column() {
let dir = tempfile::tempdir().unwrap();
write_with(dir.path(), "a.json", |v| {
v["metrics"]["tpr_at_fpr"] = serde_json::json!({"0.1": 0.5});
});
write_with(dir.path(), "b.json", |v| {
v["metrics"]["tpr_at_fpr"] = serde_json::json!({"0.10": 0.6});
});
let report = build(&[dir.path().to_path_buf()]).unwrap();
let header = render(&report, ReportFormat::Csv)
.lines()
.next()
.unwrap()
.to_string();
assert_eq!(header.matches("tpr_at_fpr_0.1,").count(), 1, "{header}");
}
#[test]
fn every_enum_value_renders_to_a_word_rather_than_a_debug_string() {
for d in [
Domain::Spatial,
Domain::Jpeg,
Domain::Structural,
Domain::Mixed,
Domain::Unstated,
] {
assert!(!domain_str(d).is_empty());
}
for p in [
Pairing::SingleVariable,
Pairing::Confounded,
Pairing::Unverified,
] {
assert!(!pairing_str(p).is_empty());
}
for s in [
SplitDiscipline::ByCover,
SplitDiscipline::ByFile,
SplitDiscipline::NotApplicable,
] {
assert!(!split_str(s).is_empty());
}
assert_eq!(source_str(CorpusSource::Fetched), "fetched");
assert_eq!(source_str(CorpusSource::Supplied), "supplied");
assert_eq!(unit_str(RateUnit::Bpp), "bpp");
assert_eq!(configuration_str(Configuration::Named), "named");
}
#[test]
fn an_unrecorded_plugin_list_is_named_as_unrecorded() {
let mut v = Doc::default().value();
v["provenance"]["plugins"] = serde_json::json!([]);
let parsed: Result1 = serde_json::from_value(v).unwrap();
let row = to_row(Path::new("x.json"), parsed);
assert_eq!(row.pinned_by, "unrecorded");
assert_eq!(row.isolation, "unrecorded");
}
#[test]
#[cfg(unix)]
fn a_symlinked_result_file_is_named_rather_than_skipped_silently() {
let dir = tempfile::tempdir().unwrap();
let elsewhere = dir.path().join("elsewhere");
std::fs::create_dir_all(&elsewhere).unwrap();
let real = Doc::default().write(&elsewhere, "real.json");
let here = dir.path().join("here");
std::fs::create_dir_all(&here).unwrap();
std::os::unix::fs::symlink(&real, here.join("linked.json")).unwrap();
Doc::default().write(&here, "ordinary.json");
let report = build(std::slice::from_ref(&here)).unwrap();
assert_eq!(report.read, 1);
assert_eq!(report.skipped.len(), 1);
assert!(report.skipped[0].reason.contains("symbolic link"));
assert_eq!(report.exit_code(), exit::FAILURE);
assert!(flat(&render(&report, ReportFormat::Text)).contains("linked.json"));
let named = build(&[here.join("linked.json")]).unwrap();
assert_eq!(named.read, 1);
assert!(named.skipped.is_empty());
}
#[test]
#[cfg(unix)]
fn a_symlink_that_is_not_a_json_file_is_not_reported_as_a_gap() {
let dir = tempfile::tempdir().unwrap();
Doc::default().write(dir.path(), "a.json");
std::fs::write(dir.path().join("notes.md"), "notes").unwrap();
std::os::unix::fs::symlink(dir.path().join("notes.md"), dir.path().join("link.md"))
.unwrap();
let report = build(&[dir.path().to_path_buf()]).unwrap();
assert!(report.skipped.is_empty(), "{:?}", report.skipped);
}
#[test]
fn the_size_cap_is_applied_to_the_bytes_actually_read() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(
dir.path().join("edge.json"),
"x".repeat(MAX_RESULT_BYTES as usize + 1),
)
.unwrap();
let report = build(&[dir.path().to_path_buf()]).unwrap();
assert!(report.skipped[0].reason.contains("byte cap"));
let at_cap = dir.path().join("at-cap.json");
std::fs::write(&at_cap, "x".repeat(MAX_RESULT_BYTES as usize)).unwrap();
let report = build(&[at_cap]).unwrap();
assert!(
report.skipped[0].reason.contains("not a result-v1"),
"{:?}",
report.skipped
);
}
fn with_arms(dir: &Path, arms: serde_json::Value) -> PathBuf {
let mut v = Doc::default().value();
v["metrics"]["per_arm"] = arms;
let path = dir.join("a.json");
std::fs::write(&path, serde_json::to_string(&v).unwrap()).unwrap();
path
}
fn two_arms() -> serde_json::Value {
serde_json::json!([
{"arm": "lsb-0400", "auc": 0.6944, "n_clean": 5, "n_stego": 5},
{"arm": "lsb-0100", "auc": 0.5, "auc_ci95": [0.4, 0.6], "n_clean": 5, "n_stego": 5},
])
}
#[test]
fn a_two_arm_document_breaks_down_into_both_arms_in_every_format() {
let dir = tempfile::tempdir().unwrap();
with_arms(dir.path(), two_arms());
let report = build(&[dir.path().to_path_buf()]).unwrap();
for format in [
ReportFormat::Text,
ReportFormat::Markdown,
ReportFormat::Csv,
] {
let text = flat(&render(&report, format));
assert!(text.contains("lsb-0100"), "{format:?} lost an arm: {text}");
assert!(text.contains("lsb-0400"), "{format:?} lost an arm: {text}");
assert!(text.contains("0.6944"), "{format:?} lost a figure: {text}");
assert!(text.contains("0.5000"), "{format:?} lost a figure: {text}");
}
assert!(flat(&render(&report, ReportFormat::Text)).contains("0.9000"));
}
#[test]
fn a_document_with_no_arms_gains_nothing_in_any_format() {
let dir = tempfile::tempdir().unwrap();
Doc::default().write(dir.path(), "a.json");
let report = build(&[dir.path().to_path_buf()]).unwrap();
assert!(arm_table(&report.groups[0]).is_none());
for format in [ReportFormat::Text, ReportFormat::Markdown] {
let text = flat(&render(&report, format));
assert!(!text.contains("Per arm"), "{format:?}: {text}");
assert!(!text.contains("clean/stego "), "{format:?}: {text}");
}
let csv = render(&report, ReportFormat::Csv);
assert!(
!csv.lines().any(|l| l.starts_with("arm,")),
"an arm line with no arms: {csv}"
);
}
#[test]
fn an_arm_interval_is_shown_where_there_is_one_and_not_invented_where_there_is_not() {
let dir = tempfile::tempdir().unwrap();
with_arms(dir.path(), two_arms());
let report = build(&[dir.path().to_path_buf()]).unwrap();
let text = flat(&render(&report, ReportFormat::Text));
assert!(text.contains("0.5000 [0.400, 0.600]"), "{text}");
assert!(
!text.contains("0.6944 ["),
"an interval was invented: {text}"
);
let csv = render(&report, ReportFormat::Csv);
let arm_lines: Vec<&str> = csv.lines().filter(|l| l.starts_with("arm,")).collect();
assert_eq!(arm_lines.len(), 2);
assert!(
arm_lines.iter().any(|l| l.contains("0.4000,0.6000")),
"{csv}"
);
assert!(
arm_lines.iter().any(|l| l.contains("0.6944,,")),
"the missing interval was filled in: {csv}"
);
let arms = &to_json(&report)["groups"][0]["rows"][0]["per_arm"];
assert!(arms[0]["auc_ci95"].is_array());
assert!(
arms[1].get("auc_ci95").is_none(),
"an absent interval became a value: {arms}"
);
}
#[test]
fn the_breakdown_is_ordered_by_arm_name_and_is_reproducible() {
let dir = tempfile::tempdir().unwrap();
with_arms(
dir.path(),
serde_json::json!([
{"arm": "wow-0200", "auc": 0.99, "n_clean": 5, "n_stego": 5},
{"arm": "lsb-0400", "auc": 0.6, "n_clean": 5, "n_stego": 5},
{"arm": "hugo-0100", "auc": 0.7, "n_clean": 5, "n_stego": 5},
]),
);
let report = build(&[dir.path().to_path_buf()]).unwrap();
let names: Vec<&str> = report.groups[0].rows[0]
.per_arm
.iter()
.map(|a| a.arm.as_str())
.collect();
assert_eq!(names, ["hugo-0100", "lsb-0400", "wow-0200"]);
for format in [
ReportFormat::Text,
ReportFormat::Markdown,
ReportFormat::Csv,
] {
let first = render(&build(&[dir.path().to_path_buf()]).unwrap(), format);
let second = render(&build(&[dir.path().to_path_buf()]).unwrap(), format);
assert_eq!(first, second, "{format:?} is not reproducible");
}
}
#[test]
fn the_json_carries_the_arm_figures_as_numbers_rather_than_as_a_rendered_string() {
let dir = tempfile::tempdir().unwrap();
with_arms(dir.path(), two_arms());
let report = build(&[dir.path().to_path_buf()]).unwrap();
let row = &to_json(&report)["groups"][0]["rows"][0];
assert_eq!(row["per_arm"][0]["arm"], serde_json::json!("lsb-0100"));
assert_eq!(row["per_arm"][0]["auc"], serde_json::json!(0.5));
assert_eq!(row["per_arm"][0]["n_clean"], serde_json::json!(5));
assert_eq!(row["per_arm"][1]["auc"], serde_json::json!(0.6944));
assert_eq!(row["per_arm"].as_array().map(|a| a.len()), Some(2));
}
#[test]
fn the_json_carries_an_empty_breakdown_as_an_empty_list() {
let dir = tempfile::tempdir().unwrap();
Doc::default().write(dir.path(), "a.json");
let report = build(&[dir.path().to_path_buf()]).unwrap();
assert_eq!(
to_json(&report)["groups"][0]["rows"][0]["per_arm"],
serde_json::json!([])
);
}
#[test]
fn a_single_arm_breakdown_is_rendered_if_a_document_carries_one() {
let dir = tempfile::tempdir().unwrap();
with_arms(
dir.path(),
serde_json::json!([{"arm": "only", "auc": 0.9, "n_clean": 5, "n_stego": 5}]),
);
let report = build(&[dir.path().to_path_buf()]).unwrap();
assert!(flat(&render(&report, ReportFormat::Text)).contains("only"));
}
fn many_arms(count: usize, prefix: &str) -> serde_json::Value {
serde_json::Value::Array(
(0..count)
.map(|i| {
serde_json::json!({
"arm": format!("{prefix}-{i:04}"),
"auc": 0.6,
"n_clean": 5,
"n_stego": 5,
})
})
.collect(),
)
}
#[test]
fn the_breakdown_stops_at_the_cap_and_names_what_it_left_out() {
let dir = tempfile::tempdir().unwrap();
for i in 0..4 {
let mut v = Doc::default().value();
v["subject"]["name"] = serde_json::json!(format!("detector-{i}"));
v["provenance"]["plugins"][0]["name"] = serde_json::json!(format!("detector-{i}"));
v["metrics"]["per_arm"] = many_arms(80, "lsb");
std::fs::write(
dir.path().join(format!("{i}.json")),
serde_json::to_string(&v).unwrap(),
)
.unwrap();
}
let report = build(&[dir.path().to_path_buf()]).unwrap();
let (lines, withheld) = arm_table(&report.groups[0]).unwrap();
assert_eq!(lines.len(), 160, "the cap cut a document in half");
assert_eq!(withheld, 2);
for format in [ReportFormat::Text, ReportFormat::Markdown] {
let text = flat(&render(&report, format));
assert!(
text.contains("2 further document(s) in this table"),
"{format:?}: {text}"
);
}
let csv = render(&report, ReportFormat::Csv);
assert_eq!(csv.lines().filter(|l| l.starts_with("arm,")).count(), 320);
let json = to_json(&report);
for row in json["groups"][0]["rows"].as_array().unwrap() {
assert_eq!(row["per_arm"].as_array().map(|a| a.len()), Some(80));
}
}
#[test]
fn one_document_larger_than_the_cap_is_still_printed_whole() {
let dir = tempfile::tempdir().unwrap();
with_arms(dir.path(), many_arms(MAX_ARM_LINES_SHOWN + 5, "lsb"));
let report = build(&[dir.path().to_path_buf()]).unwrap();
let (lines, withheld) = arm_table(&report.groups[0]).unwrap();
assert_eq!(lines.len(), MAX_ARM_LINES_SHOWN + 5);
assert_eq!(withheld, 0);
}
#[test]
fn a_csv_arm_line_carries_the_conditions_of_the_document_it_came_from() {
let dir = tempfile::tempdir().unwrap();
let mut v = Doc::default().value();
v["declarations"]["pairing"] = serde_json::json!("confounded");
v["metrics"]["per_arm"] = two_arms();
std::fs::write(
dir.path().join("a.json"),
serde_json::to_string(&v).unwrap(),
)
.unwrap();
let report = build(&[dir.path().to_path_buf()]).unwrap();
let csv = render(&report, ReportFormat::Csv);
let columns = csv.lines().next().unwrap().split(',').count();
for line in csv.lines().filter(|l| l.starts_with("arm,")) {
assert_eq!(line.split(',').count(), columns, "{line}");
assert!(line.contains("confounded"), "{line}");
assert!(line.contains("corpus-a"), "{line}");
assert!(line.contains("detector-a"), "{line}");
assert!(line.contains("CONFOUNDED"), "{line}");
}
}
#[test]
fn a_breakdown_never_crosses_a_corpus_boundary() {
let dir = tempfile::tempdir().unwrap();
for (name, digest, arm) in [("one", "d", "lsb-0100"), ("two", "e", "wow-0200")] {
let mut v = Doc::default().value();
v["corpus"]["name"] = serde_json::json!(name);
v["corpus"]["digest"] = serde_json::json!(format!("sha256:{}", digest.repeat(64)));
v["metrics"]["per_arm"] = serde_json::json!([
{"arm": arm, "auc": 0.6, "n_clean": 5, "n_stego": 5},
{"arm": "shared", "auc": 0.7, "n_clean": 5, "n_stego": 5},
]);
std::fs::write(
dir.path().join(format!("{name}.json")),
serde_json::to_string(&v).unwrap(),
)
.unwrap();
}
let report = build(&[dir.path().to_path_buf()]).unwrap();
assert_eq!(report.groups.len(), 2);
for group in &report.groups {
let (lines, _) = arm_table(group).unwrap();
assert_eq!(lines.len(), 2, "a breakdown gathered two corpora");
for line in &lines {
assert!(
line[1].starts_with(&group.corpus_name),
"an arm line does not name its corpus: {line:?}"
);
}
}
}
#[test]
fn an_auc_cell_brackets_an_interval_only_where_there_is_one() {
assert_eq!(auc_cell(0.5, None), "0.5000");
assert_eq!(auc_cell(0.5, Some([0.4, 0.6])), "0.5000 [0.400, 0.600]");
}
#[test]
#[cfg(unix)]
fn a_symlinked_directory_is_not_followed() {
{
let dir = tempfile::tempdir().unwrap();
let real = dir.path().join("real");
std::fs::create_dir_all(&real).unwrap();
Doc::default().write(&real, "a.json");
std::os::unix::fs::symlink(dir.path(), real.join("loop")).unwrap();
let report = build(&[dir.path().to_path_buf()]).unwrap();
assert_eq!(report.read, 1);
}
}
}