use sha2::{Digest, Sha256};
use std::collections::{BTreeMap, HashMap};
use std::io::{BufRead, BufReader};
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant};
use stegobench_core::corpus::CorpusEntry;
use stegobench_core::header::{self, Shape};
use stegobench_core::registry::{Entry, Kind};
use stegobench_core::result::{
AdapterRef, Arm, ArmMetrics, Configuration, CorpusRef, CorpusSource, Declarations, Determinism,
Domain, Host, Isolation, Metrics, Pairing, PinnedBy, PluginRef, Provenance, Rate, RateUnit,
Result1, SelfTest, SplitDiscipline, Subject, SubjectKind, RESULT_SCHEMA_ID,
};
use stegobench_core::samples::{Role, Sample, Samples};
use stegobench_plugin::runner::{self, JsonLines, Tally};
use stegobench_plugin::{Record, WorkItem};
pub struct Request<'a> {
pub corpus: &'a Path,
pub records: PathBuf,
pub timeout: Duration,
pub jobs: usize,
pub keep_raw: bool,
pub seed: Option<u64>,
pub fixtures: Option<&'a Path>,
pub limit: Option<u64>,
pub registered: Option<&'a CorpusEntry>,
pub trained_on: Option<&'a str>,
pub split: Option<&'a str>,
pub adapter_roots: &'a [PathBuf],
}
#[derive(Debug, thiserror::Error)]
pub enum ScoreError {
#[error("{0}")]
Corpus(#[from] stegobench_core::samples::SampleError),
#[error("{0}")]
Run(#[from] runner::RunError),
#[error("could not open the records file at {path}: {source}")]
Records {
path: String,
#[source]
source: std::io::Error,
},
#[error(
"{name} is registered as an embedder, and an embedder cannot be asked \
to tell two images apart. Pick a detector: `stegobench list detectors`"
)]
NotADetector { name: String },
#[error(
"the corpus at {path} puts {count} stego image(s) on a different side \
of the train and test split from the cover they were made from, for \
example: {examples}. A cover and its stego twin on opposite sides \
inflates every number computed from this corpus, and the inflation is \
invisible in the output, so this refuses rather than reporting a \
confident wrong answer"
)]
SplitLeaks {
path: String,
count: usize,
examples: String,
},
#[error(
"the directory at {path} is not {id}. That corpus is registered with \
the digest {want}, and these records come to {got}. Whatever the \
directory is called, it is not the corpus you named, and a number \
measured here would be filed under a name it does not belong to"
)]
NotThatCorpus {
path: String,
id: String,
want: String,
got: String,
},
#[error(
"{asked:?} is not a side of a split. The two halves are `train` and \
`test`"
)]
UnknownSplit { asked: String },
#[error(
"the corpus at {path} carries no train and test split, so there is \
no half to score. Its records state no `split`, and inventing one \
here would put a cover and its stego twin on opposite sides, which \
is the exact fault the split exists to prevent"
)]
NoSplitLabels { path: String },
#[error(
"the corpus at {path} holds {clean} clean and {stego} stego image(s), \
and a measurement needs both. A detector scored on one side of the \
question has not been measured, it has been asked a leading one"
)]
OneSided {
path: String,
clean: u64,
stego: u64,
},
#[error(
"{name} produced no usable answer for any of the {asked} image(s) in the \
corpus at {path}, which holds {clean} clean and {stego} stego, so there \
is nothing to compute a metric over. The corpus is not the problem here: \
every item was offered to the tool and every answer came back unusable. \
{why}\n\nCheck what the tool actually printed: `--keep-raw` writes its \
own output beside the records, and `stegobench doctor` runs the entry's \
self-test against the fixtures."
)]
NothingAnswered {
name: String,
path: String,
asked: u64,
clean: u64,
stego: u64,
why: String,
},
#[error(
"{name} answered about {answered} image(s), and no ROC AUC can be \
computed from those answers, so this run has no headline number. \
{why}\n\nA result-v1 document has to carry an AUC, and the value \
that would have been written is 0.5, which is exactly the figure \
that means a detector carries no information at all. Publishing it \
would be indistinguishable from publishing a measurement that came \
out at chance, so the run is refused instead and nothing is written."
)]
NoAuc {
name: String,
answered: usize,
why: String,
},
#[error(
"{id} is not the image its own record describes. The record states \
{want} and the file on disk is {got}. A run cannot be named over a \
corpus whose images and records disagree: the digest that names the \
corpus is taken over what the records SAY, so swapping the images \
underneath them leaves it unchanged"
)]
ImageChanged {
id: String,
want: String,
got: String,
},
}
impl ScoreError {
pub fn exit_code(&self) -> i32 {
use stegobench_core::exit;
match self {
ScoreError::NotADetector { .. } => exit::USAGE,
ScoreError::UnknownSplit { .. } => exit::USAGE,
ScoreError::NoSplitLabels { .. } => exit::PREFLIGHT_REFUSED,
ScoreError::NotThatCorpus { .. } | ScoreError::ImageChanged { .. } => {
exit::VERIFY_MISMATCH
}
ScoreError::SplitLeaks { .. } | ScoreError::OneSided { .. } => exit::PREFLIGHT_REFUSED,
ScoreError::NothingAnswered { .. } => exit::PLUGIN_FAILED,
ScoreError::Run(_) | ScoreError::NoAuc { .. } => exit::PLUGIN_FAILED,
ScoreError::Corpus(_) | ScoreError::Records { .. } => exit::FAILURE,
}
}
pub fn reason(&self) -> &'static str {
match self {
ScoreError::Corpus(_) => "corpus-unreadable",
ScoreError::Run(_) => "plugin-failed",
ScoreError::Records { .. } => "records-unreadable",
ScoreError::NotADetector { .. } => "not-a-detector",
ScoreError::UnknownSplit { .. } => "unknown-split",
ScoreError::NoSplitLabels { .. } => "no-split-labels",
ScoreError::NotThatCorpus { .. } => "not-that-corpus",
ScoreError::ImageChanged { .. } => "image-changed",
ScoreError::SplitLeaks { .. } => "split-leaks",
ScoreError::OneSided { .. } => "one-sided",
ScoreError::NothingAnswered { .. } => "nothing-answered",
ScoreError::NoAuc { .. } => "no-auc",
}
}
}
pub struct Prepared {
corpus: PathBuf,
limit: Option<u64>,
registered: Option<(String, String, Option<String>)>,
checks: Checks,
claim_holds: bool,
labels: Vec<bool>,
trained_on: Option<String>,
contaminated: bool,
contamination_check: stegobench_core::result::ContaminationCheck,
of_items: Option<u64>,
keep: Vec<bool>,
pub side: Option<Side>,
arm_names: Vec<String>,
arms: Vec<Option<u16>>,
started_utc: String,
preflight: Duration,
}
impl Prepared {
pub fn items(&self) -> u64 {
self.labels.len() as u64
}
pub fn preflight_seconds(&self) -> f64 {
self.preflight.as_secs_f64()
}
}
pub fn score<P>(
entry: &Entry,
request: &Request,
mut progress: P,
) -> Result<(Result1, Tally), ScoreError>
where
P: FnMut(&str),
{
refuse_embedder(entry)?;
let prepared = prepare(
request.corpus,
request.registered,
request.limit,
request.trained_on,
request.split,
&mut progress,
)?;
score_one(
entry,
&prepared,
&request.records,
How {
timeout: request.timeout,
jobs: request.jobs,
keep_raw: request.keep_raw,
seed: request.seed,
fixtures: request.fixtures,
adapter_roots: request.adapter_roots,
},
&mut progress,
)
}
fn refuse_embedder(entry: &Entry) -> Result<(), ScoreError> {
if entry.kind == Kind::Embedder {
return Err(ScoreError::NotADetector {
name: entry.name.clone(),
});
}
Ok(())
}
pub fn prepare<P>(
corpus: &Path,
registered: Option<&CorpusEntry>,
limit: Option<u64>,
trained_on: Option<&str>,
split: Option<&str>,
mut progress: P,
) -> Result<Prepared, ScoreError>
where
P: FnMut(&str),
{
let started = Instant::now();
let started_utc = now_utc();
let mut contaminated = false;
let mut contamination_check = stegobench_core::result::ContaminationCheck::NotDeclared;
let claim_holds = match registered {
None => false,
Some(entry) => match entry
.integrity
.as_ref()
.and_then(|i| i.records_sha256.as_deref())
{
None => {
progress(&format!(
"{} is registered but its entry declares no records \
digest, so there is nothing to check this directory \
against and the run is marked custom",
entry.id
));
false
}
Some(want) => {
let got = corpus_digest(corpus)?;
if got.as_deref() == Some(want) {
progress(
"the corpus matches the digest its registry entry \
declares; checking the images against their own \
records before naming the run",
);
let checked = verify_bytes(corpus, &mut progress)?;
progress(&format!(
"{checked} image(s) are the files their records \
describe"
));
true
} else {
return Err(ScoreError::NotThatCorpus {
path: corpus.display().to_string(),
id: entry.id.clone(),
want: want.to_string(),
got: got.unwrap_or_else(|| {
"nothing, because at least one record states no \
digest for its own image"
.into()
}),
});
}
}
},
};
let checks = check(corpus)?;
if !checks.split_leaks.is_empty() {
return Err(ScoreError::SplitLeaks {
path: corpus.display().to_string(),
count: checks.split_leaks.count,
examples: checks.split_leaks.examples(),
});
}
if let (Some(clean), Some(stego)) = (
checks.arm.clean_format.as_deref(),
Some(checks.arm.format.as_str()),
) {
if clean != stego && !stego.is_empty() {
progress(&format!(
"WARNING: every clean image here is {clean} and every stego image \
is {stego}. A detector that merely tells the two formats apart \
scores perfectly on this corpus without detecting anything, so a \
high number here is not evidence of steganalysis. The result \
records both formats"
));
}
}
match checks.pairing {
Pairing::Confounded => progress(&format!(
"WARNING: {} stego image(s) differ from their cover in more than \
the payload, for example: {}. This run measures that difference \
as well as the payload, and the result says so",
checks.pairing_breaks.count,
checks.pairing_breaks.examples()
)),
Pairing::Unverified if checks.unreadable > 0 => progress(&format!(
"the pairing rule could not be checked: {} stego image(s) name a \
clean half that is not in this corpus. Extract the clean arm they \
refer to alongside them and the rule becomes checkable; a record's \
own `clean` field names the one it wants. The result says \
unverified rather than claiming the rule held",
checks.unreadable
)),
Pairing::Unverified => progress(
"the pairing rule could not be checked: no stego image here names \
a cover or a clean half this could read alongside it. The result \
says unverified rather than claiming the rule held",
),
Pairing::SingleVariable => {
if checks.unreadable > 0 {
progress(&format!(
"{} of {} stego image(s) could not be compared with their \
cover, so the pairing check covered the rest",
checks.unreadable,
checks.unreadable + checks.compared
));
}
}
}
if let Some(trained) = trained_on {
let t = trained.trim();
let mut aliases: Vec<String> = vec![corpus_name(corpus)];
if let Some(e) = registered {
aliases.push(e.id.clone());
aliases.push(e.name.clone());
}
if let Some(base) = corpus.file_name().and_then(|s| s.to_str()) {
aliases.push(base.to_string());
}
let same = aliases
.iter()
.any(|a| a.trim().eq_ignore_ascii_case(t) && !a.trim().is_empty());
contaminated = same;
contamination_check = if registered.is_some() {
stegobench_core::result::ContaminationCheck::Checked
} else {
stegobench_core::result::ContaminationCheck::DeclaredButUncheckable
};
if same {
progress(&format!(
"WARNING: this detector is declared as trained on {trained}, \
which is the corpus it is being scored against. The number \
below measures what it memorised as well as what it detects, \
and it is not a figure to quote for {trained}"
));
} else if registered.is_none() {
progress(&format!(
"declared as trained on {trained}. This corpus is not \
identified, so that could not be checked against it: pass \
--corpus-id to say which registered corpus this directory \
holds and the check becomes real"
));
} else {
progress(&format!(
"declared as trained on {trained}, which is not the corpus \
being scored, and the result records it so a reader can \
judge that for themselves"
));
}
}
if claim_holds && limit.is_some() {
progress(
"this run scored part of the corpus, so it is marked custom \
rather than named: a prefix of a tier is not the tier",
);
}
let wanted = match split {
None => None,
Some(s) => match Side::parse(s) {
Some(side) => Some(side),
None => {
return Err(ScoreError::UnknownSplit {
asked: s.to_string(),
})
}
},
};
if wanted.is_some() && checks.sides.iter().all(Option::is_none) {
return Err(ScoreError::NoSplitLabels {
path: corpus.display().to_string(),
});
}
let mut labels = Vec::new();
let mut keep = Vec::new();
let mut of_items: Option<u64> = None;
let mut arm_names: Vec<String> = Vec::new();
let mut arms: Vec<Option<u16>> = Vec::new();
let mut rows = Samples::open(corpus)?.enumerate();
while let Some((i, sample)) = rows.next() {
let sample = sample?;
let take = match wanted {
None => true,
Some(side) => checks.sides.get(i).copied().flatten() == Some(side),
};
keep.push(take);
if take {
labels.push(sample.role == Role::Stego);
arms.push(intern(&mut arm_names, arm_name_of(&sample)));
if limit.is_some_and(|n| labels.len() as u64 >= n) {
let mut total = i as u64 + 1;
for (_, rest) in rows.by_ref() {
rest?;
total += 1;
}
of_items = Some(total);
break;
}
}
}
if let Some(side) = wanted {
let left_out = keep.iter().filter(|k| !**k).count();
progress(&format!(
"scoring the {} split only: {} image(s), {left_out} left out",
side.as_str(),
labels.len()
));
}
let stego = labels.iter().filter(|l| **l).count() as u64;
let clean = labels.len() as u64 - stego;
if clean == 0 || stego == 0 {
return Err(ScoreError::OneSided {
path: match wanted {
Some(side) => format!("{} ({} split)", corpus.display(), side.as_str()),
None => corpus.display().to_string(),
},
clean,
stego,
});
}
Ok(Prepared {
corpus: corpus.to_path_buf(),
limit,
of_items,
registered: registered.map(|e| (e.id.clone(), e.name.clone(), e.tier.clone())),
trained_on: trained_on.map(str::to_string),
contaminated,
contamination_check,
checks,
claim_holds,
labels,
keep,
side: wanted,
arm_names,
arms,
started_utc,
preflight: started.elapsed(),
})
}
#[derive(Clone, Copy)]
pub struct How<'a> {
pub timeout: Duration,
pub jobs: usize,
pub keep_raw: bool,
pub seed: Option<u64>,
pub fixtures: Option<&'a Path>,
pub adapter_roots: &'a [PathBuf],
}
pub fn score_one<P>(
entry: &Entry,
prepared: &Prepared,
records: &Path,
how: How<'_>,
mut progress: P,
) -> Result<(Result1, Tally), ScoreError>
where
P: FnMut(&str),
{
let How {
timeout,
jobs,
keep_raw,
seed,
fixtures,
adapter_roots,
} = how;
let selftest =
fixtures.map(
|dir| match stegobench_plugin::selftest::run(entry, dir, adapter_roots) {
stegobench_plugin::selftest::Verified::Passed => SelfTest::Passed,
stegobench_plugin::selftest::Verified::Failed(_) => SelfTest::Failed,
stegobench_plugin::selftest::Verified::Skipped(_) => SelfTest::Skipped,
stegobench_plugin::selftest::Verified::Answered(_) => SelfTest::Answered,
},
);
refuse_embedder(entry)?;
let file = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(records)
.map_err(|e| ScoreError::Records {
path: records.display().to_string(),
source: e,
})?;
let mut sink = JsonLines::new(file);
let mut feed = Feed::filtered(
&prepared.corpus,
prepared.limit,
prepared.side.map(|_| prepared.keep.clone()),
)?;
let raw_path = {
let name = records.file_name().unwrap_or_default().to_string_lossy();
let stem = name.strip_suffix(".records.jsonl").unwrap_or(&name);
records.with_file_name(format!("{stem}.raw.jsonl"))
};
if let Some(batch) = entry
.invoke
.as_ref()
.and_then(|i| i.batch)
.filter(|b| *b > 1)
{
progress(&format!(
"{} asks about {batch} image(s) per container, so the first answers \
arrive together rather than one at a time. Progress is reported \
every {}s while that happens, including when the count has not \
moved yet",
entry.name,
runner::HEARTBEAT.as_secs()
));
}
let own_started = Instant::now();
let total = prepared.items();
let declared_total = entry
.cost
.seconds_per_image
.map(|s| s * total.max(1) as f64);
let mut said_already = false;
let tally = runner::score(
runner::Run {
entry,
already: records,
timeout,
jobs,
adapter_roots,
raw: runner::RawPolicy {
path: Some(&raw_path),
every: keep_raw,
},
},
&mut feed,
&mut sink,
|t| {
progress(&format!("{} scored, {} errored", t.scored, t.errored));
if !said_already && t.scored >= REVISE_AFTER {
said_already = true;
if let Some(line) = revised_estimate(
t.scored,
total.saturating_sub(t.seen()),
total,
own_started.elapsed(),
declared_total,
) {
progress(&line);
}
}
},
)?;
if let Some(problem) = &tally.raw_problem {
progress(problem);
}
if let Some(e) = feed.fault {
return Err(ScoreError::Corpus(e));
}
let Joined {
scores,
labels,
arms,
errored,
answers_a_verdict,
} = join(records, &prepared.labels, &prepared.arms)?;
let n_stego = labels.iter().filter(|l| **l).count() as u64;
let n_clean = labels.len() as u64 - n_stego;
if n_clean == 0 || n_stego == 0 {
let corpus_sides = prepared.labels.iter().filter(|l| **l).count() as u64;
let corpus_clean = prepared.labels.len() as u64 - corpus_sides;
if scores.is_empty() && corpus_clean > 0 && corpus_sides > 0 {
return Err(ScoreError::NothingAnswered {
name: entry.name.clone(),
path: prepared.corpus.display().to_string(),
asked: prepared.labels.len() as u64,
clean: corpus_clean,
stego: corpus_sides,
why: match errored {
0 => "No record carried a score or a verdict at all.".to_string(),
n => format!("All {n} of them came back as failures."),
},
});
}
return Err(ScoreError::OneSided {
path: prepared.corpus.display().to_string(),
clean: n_clean,
stego: n_stego,
});
}
let checks = &prepared.checks;
let named = if prepared.claim_holds && prepared.limit.is_none() {
Configuration::Named
} else {
Configuration::Custom
};
let auc = match stegobench_metrics::roc_auc(&scores, &labels) {
Some(auc) => auc,
None => {
return Err(ScoreError::NoAuc {
name: entry.name.clone(),
answered: scores.len(),
why: why_no_auc(&scores, &labels, ""),
})
}
};
let auc_ci95 = stegobench_metrics::roc_auc_interval(&scores, &labels, stegobench_metrics::Z_95)
.map(|ci| [ci.low, ci.high]);
if answers_a_verdict {
progress(&format!(
"{} answers yes or no rather than giving a score, so every record \
here has `score: null` and a `verdict` instead. That is the answer, \
not a gap, which is why none of these count as unscored. It also \
means there is no threshold to choose for this detector: a run with \
it is already at its only operating point",
entry.name
));
}
for finding in stegobench_metrics::findings(&scores, &labels) {
progress(&match finding {
stegobench_metrics::Finding::EveryScoreIdentical { count } => format!(
"every one of the {count} answers from {} was identical, so this AUC is 0.5 \
by construction and not by measurement: it did not separate these images. \
A detector that found nothing and an adapter misreading its output look \
exactly like this, and only the tool's own output tells them apart",
entry.name
),
stegobench_metrics::Finding::ScoresRunBackwards { auc, inverted } => format!(
"{} scored {auc:.4}, which is below the 0.5 a coin flip gets. That \
usually means its scores run the wrong way round rather than that \
it cannot see anything: at {inverted:.4} it separates these images about as \
well inverted as it would upright. Check the adapter's sign before \
reading this as a measurement",
entry.name
),
});
}
let mut tpr_at_fpr = BTreeMap::new();
let mut fpr_achieved = BTreeMap::new();
let mut threshold_at_fpr = BTreeMap::new();
let mut unspendable: Vec<String> = Vec::new();
for fpr in [0.01, 0.05, 0.10] {
match stegobench_metrics::tpr_at_fpr(&scores, &labels, fpr) {
Some(tpr) => {
let key = format!("{fpr:.2}");
tpr_at_fpr.insert(key.clone(), tpr);
if let Some(point) = stegobench_metrics::operating_point(&scores, &labels, fpr) {
fpr_achieved.insert(key.clone(), point.achieved_fpr);
if let Some(threshold) = point.threshold {
threshold_at_fpr.insert(key, threshold);
}
if !point.budget_was_expressible(n_clean as usize) {
unspendable.push(format!("{:.0}%", fpr * 100.0));
}
}
}
None => {
return Err(ScoreError::NoAuc {
name: entry.name.clone(),
answered: scores.len(),
why: why_no_auc(
&scores,
&labels,
&format!(" at a false-alarm budget of {fpr:.2}"),
),
})
}
}
}
if !unspendable.is_empty() {
progress(&format!(
"{} clean image(s) cannot express a false-alarm budget finer than \
{:.1}%, so the figure(s) reported at {} are the figure at the \
nearest rate this corpus can actually show. The number is real; \
the heading over it is finer than the measurement. More clean \
images is the only fix",
n_clean,
stegobench_metrics::fpr_resolution(n_clean as usize) * 100.0,
unspendable.join(", ")
));
}
if tally.resumed > 0 && tally.scored == 0 {
progress(&format!(
"every one of the {} answer(s) came from the existing records, so \
nothing was scored. The run time and start time below are this \
command's, not the measurement's",
tally.resumed
));
} else if tally.resumed > 0 {
progress(&format!(
"{} answer(s) came from the existing records and {} were scored \
now, so the run time below covers this command rather than the \
whole measurement",
tally.resumed, tally.scored
));
}
let (version, pinned_by) = pinning(entry);
let adapter = adapter_ref(entry, adapter_roots);
let mut result = Result1 {
schema: RESULT_SCHEMA_ID.to_string(),
subject: Subject {
name: entry.name.clone(),
version: version.clone(),
kind: SubjectKind::Detector,
},
corpus: CorpusRef {
name: match &prepared.registered {
Some((_, name, _)) => name.clone(),
None => corpus_name(&prepared.corpus),
},
id: prepared.registered.as_ref().map(|(id, _, _)| id.clone()),
tier: prepared
.registered
.as_ref()
.and_then(|(_, _, tier)| tier.clone()),
source: CorpusSource::Supplied,
digest: checks.digest.clone().unwrap_or_default(),
pairs: labels.iter().filter(|&&stego| stego).count() as u64,
of_items: prepared.of_items,
split: prepared.side.map(|s| s.as_str().to_string()),
},
arm: checks.arm.clone(),
metrics: Metrics {
auc,
auc_ci95,
tpr_at_fpr,
fpr_achieved,
threshold_at_fpr,
verdict_rate: None,
n_clean,
n_stego,
n_error: errored,
per_arm: per_arm(&scores, &labels, &arms, &prepared.arm_names),
},
provenance: Provenance {
seed,
plugins: vec![PluginRef {
name: entry.name.clone(),
image: version,
determinism: entry.determinism.unwrap_or(Determinism::Unstated),
selftest,
adapter,
argv: entry.invoke.as_ref().map(|i| i.argv.clone()),
pinned_by,
isolation: isolation(entry),
}],
harness_version: env!("CARGO_PKG_VERSION").to_string(),
started_utc: prepared.started_utc.clone(),
elapsed_seconds: prepared.preflight.as_secs_f64() + own_started.elapsed().as_secs_f64(),
network_reachable: !runs_in_container(entry),
host: Some(host()),
records: None,
answers: stegobench_core::result::AnswerSource {
measured: tally.scored,
resumed: tally.resumed,
},
},
declarations: Declarations {
split_discipline: checks.split,
pairing: checks.pairing,
configuration: named,
trained_on: prepared.trained_on.clone(),
contaminated: prepared.contaminated,
contamination_check: prepared.contamination_check,
split_scored: prepared.side.map(|s| s.as_str().to_string()),
self_reported: true,
pairing_evidence: stegobench_core::result::PairingEvidence {
compared: checks.compared,
named_a_half_not_here: checks.unreadable,
differed: checks.pairing_breaks.count as u64,
},
},
content_digest: None,
};
result.provenance.records = stegobench_core::result::records_digest(records).ok();
result.seal();
Ok((result, tally))
}
fn why_no_auc(scores: &[f64], labels: &[bool], at: &str) -> String {
let refusal = crate::metrics::why_unrankable(scores, labels, at);
let here = match refusal.reason() {
"length-mismatch" => {
" Both are built from the same records in the same order here, so \
this is a bug in stegobench rather than anything you did. Please \
report it with this message."
}
"empty" => {
" Every image was put to this tool and none of them came back with \
a number, which is the tool or its parser rather than the corpus: \
`stegobench doctor` runs its self-test and says which."
}
"not-a-number" => {
" This is the detector's output, so the fix is with the tool or \
its parser: `stegobench doctor` runs its self-test."
}
"one-sided" => {
" Whatever the corpus holds, this tool produced a usable answer \
for only one of them."
}
_ => "",
};
format!("{refusal}{here}")
}
struct Feed {
samples: Samples,
limit: Option<u64>,
taken: u64,
seen: usize,
keep: Option<Vec<bool>>,
fault: Option<stegobench_core::samples::SampleError>,
}
impl Feed {
fn filtered(
corpus: &Path,
limit: Option<u64>,
keep: Option<Vec<bool>>,
) -> Result<Self, ScoreError> {
Ok(Self {
samples: Samples::open(corpus)?,
limit,
taken: 0,
seen: 0,
keep,
fault: None,
})
}
}
fn runs_in_container(entry: &Entry) -> bool {
entry.image.is_some() && !entry.invoke.as_ref().is_some_and(|i| i.host)
}
impl Iterator for Feed {
type Item = WorkItem;
fn next(&mut self) -> Option<WorkItem> {
loop {
if self.limit.is_some_and(|n| self.taken >= n) {
return None;
}
match self.samples.next()? {
Ok(sample) => {
let at = self.seen;
self.seen += 1;
if let Some(keep) = &self.keep {
match keep.get(at) {
Some(true) => {}
Some(false) => continue,
None => return None,
}
}
self.taken += 1;
return Some(WorkItem {
id: sample.id,
path: sample.image,
});
}
Err(e) => {
self.fault = Some(e);
return None;
}
}
}
}
}
const REVISE_AFTER: u64 = 16;
const MAX_EXAMPLES: usize = 3;
#[derive(Default)]
struct Violations {
count: usize,
examples: Vec<String>,
}
impl Violations {
fn note(&mut self, what: String) {
self.count += 1;
if self.examples.len() < MAX_EXAMPLES {
self.examples.push(what);
}
}
fn is_empty(&self) -> bool {
self.count == 0
}
fn examples(&self) -> String {
self.examples.join("; ")
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Side {
Train,
Test,
}
impl Side {
fn parse(s: &str) -> Option<Side> {
match s.trim().to_ascii_lowercase().as_str() {
"train" => Some(Side::Train),
"test" => Some(Side::Test),
_ => None,
}
}
pub fn as_str(self) -> &'static str {
match self {
Side::Train => "train",
Side::Test => "test",
}
}
}
struct Checks {
split: SplitDiscipline,
sides: Vec<Option<Side>>,
split_leaks: Violations,
pairing: Pairing,
pairing_breaks: Violations,
compared: u64,
unreadable: u64,
digest: Option<String>,
arm: Arm,
}
#[derive(Default)]
pub struct CorpusDigest {
hash: Sha256,
seen: u64,
every_record_declares_one: bool,
}
impl CorpusDigest {
fn new() -> Self {
Self {
hash: Sha256::new(),
seen: 0,
every_record_declares_one: true,
}
}
fn note(&mut self, sample: &Sample) {
self.seen += 1;
self.hash.update(sample.id.as_bytes());
self.hash.update(b"\0");
match &sample.digest {
Some(d) => self.hash.update(d.as_bytes()),
None => self.every_record_declares_one = false,
}
self.hash.update(b"\n");
}
fn finish(self) -> Option<String> {
(self.every_record_declares_one && self.seen > 0)
.then(|| format!("sha256:{:x}", self.hash.finalize()))
}
}
fn verify_bytes<P>(root: &Path, progress: &mut P) -> Result<u64, ScoreError>
where
P: FnMut(&str),
{
let mut checked = 0u64;
let mut skipped = 0u64;
let mut last_beat = Instant::now();
for sample in Samples::open(root)? {
let sample = sample?;
let Some(want) = sample.digest.as_deref() else {
skipped += 1;
continue;
};
let got = hash_file(&sample.image).map_err(|e| ScoreError::Records {
path: sample.image.display().to_string(),
source: e,
})?;
if got != want {
return Err(ScoreError::ImageChanged {
id: sample.id,
want: want.to_string(),
got,
});
}
checked += 1;
if last_beat.elapsed() >= HEARTBEAT {
progress(&format!("{checked} image(s) checked against their records"));
last_beat = Instant::now();
}
}
debug_assert_eq!(skipped, 0, "a digested corpus had a record with no digest");
Ok(checked)
}
const HEARTBEAT: Duration = Duration::from_secs(30);
fn hash_file(path: &Path) -> std::io::Result<String> {
use std::io::Read;
let mut file = std::fs::File::open(path)?;
let mut hasher = Sha256::new();
let mut buf = vec![0u8; 64 * 1024];
loop {
let n = file.read(&mut buf)?;
if n == 0 {
break;
}
hasher.update(&buf[..n]);
}
Ok(format!("{:x}", hasher.finalize()))
}
#[derive(Debug, Clone, PartialEq)]
pub struct Mismatch {
pub id: String,
pub claimed: String,
pub found: Result<String, String>,
}
const NAMED_MISMATCHES: usize = 10;
pub fn rehash_corpus(
root: &Path,
mut progress: impl FnMut(&str),
) -> Result<(Vec<Mismatch>, u64), ScoreError> {
let mut bad = Vec::new();
let mut read = 0u64;
let mut extra = 0u64;
let mut last = Instant::now();
for sample in Samples::open(root)? {
let sample = sample?;
let Some(claimed) = sample.digest.as_deref() else {
continue;
};
read += 1;
if last.elapsed() >= HEARTBEAT {
progress(&format!("rehashed {read} image(s) so far"));
last = Instant::now();
}
let found = hash_file(&sample.image).map_err(|e| e.to_string());
if found.as_deref() == Ok(claimed) {
continue;
}
if bad.len() < NAMED_MISMATCHES {
bad.push(Mismatch {
id: sample.id.clone(),
claimed: claimed.to_string(),
found,
});
} else {
extra += 1;
}
}
if extra > 0 {
progress(&format!(
"{} further image(s) also differ and are not listed",
extra
));
}
Ok((bad, read))
}
pub fn corpus_digest(root: &Path) -> Result<Option<String>, ScoreError> {
let mut digest = CorpusDigest::new();
for sample in Samples::open(root)? {
digest.note(&sample?);
}
Ok(digest.finish())
}
#[derive(Debug, Clone, PartialEq, Default)]
enum Uniform<T> {
#[default]
Nothing,
One(T),
Many,
}
impl<T: PartialEq> Uniform<T> {
fn note(&mut self, value: Option<T>) {
let Some(value) = value else { return };
match self {
Uniform::Nothing => *self = Uniform::One(value),
Uniform::One(seen) if *seen == value => {}
Uniform::One(_) => *self = Uniform::Many,
Uniform::Many => {}
}
}
fn one(&self) -> Option<&T> {
match self {
Uniform::One(v) => Some(v),
_ => None,
}
}
}
impl Uniform<String> {
fn or_say_why(&self) -> String {
match self {
Uniform::One(v) => v.clone(),
Uniform::Many => "mixed".into(),
Uniform::Nothing => "unstated".into(),
}
}
}
#[derive(Default)]
struct ArmFacts {
tool: Uniform<String>,
rate: Uniform<(String, f64)>,
domain: Uniform<String>,
format: Uniform<String>,
clean_format: Uniform<String>,
}
#[derive(Clone)]
struct Cover {
split: Option<String>,
shape: Option<Shape>,
}
fn check(corpus: &Path) -> Result<Checks, ScoreError> {
let mut covers: HashMap<String, Cover> = HashMap::new();
let mut clean_halves: HashMap<String, Cover> = HashMap::new();
let mut any_split = false;
let mut digest = CorpusDigest::new();
let mut facts = ArmFacts::default();
for sample in Samples::open(corpus)? {
let sample = sample?;
digest.note(&sample);
if sample.role == Role::Stego {
let arm = &sample.arm;
facts
.tool
.note(arm.tool.clone().or_else(|| arm.name.clone()));
facts.domain.note(arm.domain.clone());
facts.format.note(image_format(&sample.image));
facts
.rate
.note(arm.rate.zip(arm.rate_unit.clone()).map(|(r, u)| (u, r)));
}
if sample.role != Role::Clean {
continue;
}
facts.clean_format.note(image_format(&sample.image));
any_split |= sample.split.is_some();
let on_disk = sample
.image
.file_name()
.and_then(|n| n.to_str())
.map(str::to_string);
let declared = sample.declared_name.clone();
let shape = header::read(&sample.image).ok();
let cover = Cover {
split: sample.split.clone(),
shape,
};
if let Some(name) = declared {
covers.insert(name, cover.clone());
}
if let Some(name) = on_disk {
covers.entry(name).or_insert(cover.clone());
}
if let Some(digest) = sample.digest {
clean_halves.insert(digest, cover);
}
}
let mut checks = Checks {
split: SplitDiscipline::NotApplicable,
split_leaks: Violations::default(),
pairing: Pairing::Unverified,
pairing_breaks: Violations::default(),
compared: 0,
unreadable: 0,
sides: Vec::new(),
digest: digest.finish(),
arm: arm_of(&facts),
};
for sample in Samples::open(corpus)? {
let sample = sample?;
let side = match (&sample.role, &sample.cover) {
(Role::Clean, _) => sample.split.as_deref().and_then(Side::parse),
(_, Some(name)) => covers
.get(name)
.and_then(|c| c.split.as_deref())
.and_then(Side::parse)
.or_else(|| sample.split.as_deref().and_then(Side::parse)),
(_, None) => sample.split.as_deref().and_then(Side::parse),
};
checks.sides.push(side);
if sample.role != Role::Stego {
continue;
}
if let Some(cover_name) = &sample.cover {
if let Some(cover) = covers.get(cover_name) {
if let (Some(split), Some(cover_split)) = (&sample.split, &cover.split) {
any_split = true;
if split != cover_split {
checks.split_leaks.note(format!(
"{} is in the {split} split while its cover {cover_name} is \
in the {cover_split} split",
sample.id
));
}
}
}
}
let against = match &sample.clean_digest {
Some(digest) => match clean_halves.get(digest) {
Some(half) => Some((half, "its clean half".to_string())),
None => {
checks.unreadable += 1;
continue;
}
},
None => sample.cover.as_ref().and_then(|name| {
covers
.get(name)
.map(|cover| (cover, format!("its cover {name}")))
}),
};
let Some((against, label)) = against else {
continue;
};
match (against.shape, header::read(&sample.image).ok()) {
(Some(clean_shape), Some(stego_shape)) => match difference(clean_shape, stego_shape) {
None => checks.compared += 1,
Some(what) => {
checks.compared += 1;
checks
.pairing_breaks
.note(format!("{} and {label} {what}", sample.id));
}
},
_ => checks.unreadable += 1,
}
}
if any_split {
checks.split = SplitDiscipline::ByCover;
}
checks.pairing = if !checks.pairing_breaks.is_empty() {
Pairing::Confounded
} else if checks.compared > 0 {
Pairing::SingleVariable
} else {
Pairing::Unverified
};
Ok(checks)
}
fn arm_of(facts: &ArmFacts) -> Arm {
let rate = facts.rate.one().and_then(|(unit, value)| {
let unit = match unit.as_str() {
"bits per pixel" => RateUnit::Bpp,
"bits per non-zero AC coefficient" => return None,
u if u.contains("capacity") => RateUnit::CapacityFraction,
_ => return None,
};
Some(Rate {
value: *value,
unit,
})
});
let domain = match &facts.domain {
Uniform::One(d) => match d.as_str() {
"spatial" => Domain::Spatial,
"jpeg-dct" | "jpeg" => Domain::Jpeg,
"container" | "structural" => Domain::Structural,
_ => Domain::Unstated,
},
Uniform::Many => Domain::Mixed,
Uniform::Nothing => match facts.format.one().map(String::as_str) {
Some("jpeg") => Domain::Jpeg,
Some(_) => Domain::Spatial,
None => Domain::Unstated,
},
};
Arm {
embedder: facts.tool.or_say_why(),
rate,
domain,
format: facts.format.or_say_why(),
clean_format: facts.clean_format.one().cloned(),
}
}
fn image_format(path: &Path) -> Option<String> {
let ext = path.extension()?.to_str()?.to_ascii_lowercase();
Some(match ext.as_str() {
"jpg" | "jpeg" => "jpeg".into(),
"tif" | "tiff" => "tiff".into(),
_ => ext,
})
}
fn difference(cover: Shape, stego: Shape) -> Option<String> {
if cover.format != stego.format {
return Some(format!(
"are different formats: {} against {}",
cover.format.name(),
stego.format.name()
));
}
let (Some(c), Some(s)) = (cover.geometry, stego.geometry) else {
return None;
};
if (c.width, c.height) != (s.width, s.height) {
return Some(format!(
"are different sizes: {} by {} against {} by {}",
c.width, c.height, s.width, s.height
));
}
if c.depth != s.depth {
return Some(format!(
"have different bit depths: {} against {}",
c.depth, s.depth
));
}
if c.channels != s.channels {
return Some(format!(
"have different channel counts: {} against {}",
c.channels, s.channels
));
}
None
}
fn arm_name_of(sample: &stegobench_core::samples::Sample) -> Option<&str> {
if sample.role != Role::Stego {
return None;
}
sample.arm.name.as_deref()
}
fn intern(table: &mut Vec<String>, name: Option<&str>) -> Option<u16> {
let name = name?;
if let Some(i) = table.iter().position(|n| n == name) {
return u16::try_from(i).ok();
}
if table.len() >= u16::MAX as usize {
return None;
}
table.push(name.to_string());
u16::try_from(table.len() - 1).ok()
}
fn revised_estimate(
scored: u64,
left: u64,
total: u64,
elapsed: Duration,
declared_total: Option<f64>,
) -> Option<String> {
if scored == 0 {
return None;
}
let per_image = elapsed.as_secs_f64() / scored as f64;
let projected = per_image * left as f64;
let declared_left = declared_total.map(|d| d * (left as f64 / total.max(1) as f64));
let out_by = declared_left.map(|d| projected / d.max(1e-9));
if out_by.is_some_and(|r| (0.7..1.4).contains(&r)) {
return None;
}
let how_long = crate::human_duration(projected);
let measured = format!(
"measured on this machine: {per_image:.1}s an image over the first \
{scored} scored, so the {left} still to go will take about {how_long}"
);
Some(match out_by {
Some(ratio) if ratio > 1.0 => format!(
"{measured}, which is {ratio:.0}x what this detector's declared rate \
predicted. The declared figure is a guess made before the run; this \
one is what is happening"
),
Some(ratio) => format!(
"{measured}, which is {:.0}x faster than this detector's declared \
rate predicted",
1.0 / ratio.max(1e-9)
),
None => format!(
"{measured}. This detector declares no rate, so there was nothing to \
estimate from beforehand"
),
})
}
struct Joined {
scores: Vec<f64>,
labels: Vec<bool>,
arms: Vec<Option<u16>>,
errored: u64,
answers_a_verdict: bool,
}
fn join(records: &Path, labels: &[bool], arms: &[Option<u16>]) -> Result<Joined, ScoreError> {
let file = std::fs::File::open(records).map_err(|e| ScoreError::Records {
path: records.display().to_string(),
source: e,
})?;
let mut scores = Vec::new();
let mut errored = 0;
let mut kept = Vec::new();
let mut kept_arms = Vec::new();
let mut any_score = false;
for ((line, label), arm) in BufReader::new(file)
.lines()
.zip(labels.iter())
.zip(arms.iter().chain(std::iter::repeat(&None)))
{
let Ok(line) = line else { break };
if line.trim().is_empty() {
continue;
}
let Ok(record) = serde_json::from_str::<Record>(&line) else {
break;
};
if record.score.is_some() {
any_score = true;
}
match record
.score
.or(record.verdict.map(|v| if v { 1.0 } else { 0.0 }))
{
Some(s) => {
scores.push(s);
kept.push(*label);
kept_arms.push(*arm);
}
None => errored += 1,
}
}
Ok(Joined {
answers_a_verdict: !any_score && !scores.is_empty(),
scores,
labels: kept,
arms: kept_arms,
errored,
})
}
fn per_arm(
scores: &[f64],
labels: &[bool],
arms: &[Option<u16>],
names: &[String],
) -> Vec<ArmMetrics> {
let present: std::collections::BTreeSet<u16> = arms
.iter()
.zip(labels.iter())
.filter(|(_, stego)| **stego)
.filter_map(|(a, _)| *a)
.collect();
if present.len() < 2 {
return Vec::new();
}
let clean: Vec<f64> = scores
.iter()
.zip(labels.iter())
.filter(|(_, stego)| !**stego)
.map(|(s, _)| *s)
.collect();
let mut out = Vec::new();
for arm in present {
let mut s = clean.clone();
let mut l = vec![false; clean.len()];
for ((score, stego), a) in scores.iter().zip(labels.iter()).zip(arms.iter()) {
if *stego && *a == Some(arm) {
s.push(*score);
l.push(true);
}
}
let Some(auc) = stegobench_metrics::roc_auc(&s, &l) else {
continue;
};
let n_stego = l.iter().filter(|x| **x).count() as u64;
out.push(ArmMetrics {
arm: names
.get(arm as usize)
.cloned()
.unwrap_or_else(|| arm.to_string()),
auc,
auc_ci95: stegobench_metrics::roc_auc_interval(&s, &l, stegobench_metrics::Z_95)
.map(|ci| [ci.low, ci.high]),
n_clean: clean.len() as u64,
n_stego,
});
}
out
}
fn host() -> Host {
Host {
cores: std::thread::available_parallelism()
.ok()
.map(|n| n.get() as u32),
memory_gb: None,
os: std::env::consts::OS.to_string(),
arch: std::env::consts::ARCH.to_string(),
}
}
fn adapter_ref(entry: &Entry, roots: &[PathBuf]) -> Option<AdapterRef> {
let declared = entry.invoke.as_ref()?.adapter.as_ref()?;
let path = stegobench_plugin::adapter::resolve(declared, roots).ok()?;
let sha256 = stegobench_plugin::hash_file(&path).ok()?;
Some(AdapterRef {
declared: declared.clone(),
sha256,
})
}
fn pinning(entry: &Entry) -> (String, PinnedBy) {
if let Some(image) = &entry.image {
let pinned = if runs_in_container(entry) {
PinnedBy::ImageDigest
} else {
PinnedBy::Unpinned
};
return (image.reference.clone(), pinned);
}
match entry
.binary
.as_ref()
.and_then(|b| b.command.first())
.and_then(|program| stegobench_plugin::which(program))
.and_then(|path| stegobench_plugin::hash_file(&path).ok())
{
Some(hash) => (hash, PinnedBy::ExecutableHash),
None => ("unknown".into(), PinnedBy::Unpinned),
}
}
fn isolation(entry: &Entry) -> Isolation {
if runs_in_container(entry) {
return Isolation::SandboxNoNetwork;
}
if entry
.invoke
.as_ref()
.is_some_and(|i| i.host && i.endpoint_env.is_some())
{
return Isolation::RemoteService;
}
Isolation::Host
}
fn corpus_name(path: &Path) -> String {
path.file_name()
.and_then(|n| n.to_str())
.unwrap_or("corpus")
.to_string()
}
fn now_utc() -> String {
let secs = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0);
iso8601(secs)
}
fn iso8601(secs: u64) -> String {
let days = (secs / 86_400) as i64;
let rem = secs % 86_400;
let (h, m, s) = (rem / 3600, (rem % 3600) / 60, rem % 60);
let z = days + 719_468;
let era = z.div_euclid(146_097);
let doe = z.rem_euclid(146_097);
let yoe = (doe - doe / 1_460 + doe / 36_524 - doe / 146_096) / 365;
let y = yoe + era * 400;
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
let mp = (5 * doy + 2) / 153;
let d = doy - (153 * mp + 2) / 5 + 1;
let month = if mp < 10 { mp + 3 } else { mp - 9 };
let year = if month <= 2 { y + 1 } else { y };
format!("{year:04}-{month:02}-{d:02}T{h:02}:{m:02}:{s:02}Z")
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn the_timestamp_is_iso_8601_and_correct() {
assert_eq!(iso8601(0), "1970-01-01T00:00:00Z");
assert_eq!(iso8601(1_000_000_000), "2001-09-09T01:46:40Z");
assert_eq!(iso8601(1_709_164_800), "2024-02-29T00:00:00Z");
assert_eq!(iso8601(4_102_444_800), "2100-01-01T00:00:00Z");
}
fn png(width: u32, height: u32, depth: u8, colour: u8, padding: usize) -> Vec<u8> {
let mut v = b"\x89PNG\r\n\x1a\n".to_vec();
v.extend_from_slice(&13u32.to_be_bytes());
v.extend_from_slice(b"IHDR");
v.extend_from_slice(&width.to_be_bytes());
v.extend_from_slice(&height.to_be_bytes());
v.extend_from_slice(&[depth, colour, 0, 0, 0]);
v.extend_from_slice(&[0, 0, 0, 0]);
v.extend_from_slice(&vec![0u8; padding]);
v
}
fn corpus(root: &Path, clean: usize, stego: usize) {
std::fs::create_dir_all(root).expect("corpus");
for i in 0..clean {
std::fs::write(root.join(format!("c{i:03}.png")), png(32, 32, 8, 2, 0)).unwrap();
std::fs::write(
root.join(format!("c{i:03}.json")),
r#"{"role":"clean","sha256":"0"}"#,
)
.unwrap();
}
for i in 0..stego {
std::fs::write(root.join(format!("s{i:03}.png")), png(32, 32, 8, 2, 64)).unwrap();
std::fs::write(
root.join(format!("s{i:03}.json")),
r#"{"role":"stego","source_png":"c000.png","sha256":"0"}"#,
)
.unwrap();
}
}
fn request(dir: &Path, limit: Option<u64>) -> Request<'_> {
Request {
corpus: dir,
registered: None,
trained_on: None,
adapter_roots: &[],
records: dir.with_extension("records.jsonl"),
timeout: Duration::from_secs(5),
jobs: 1,
keep_raw: false,
seed: None,
fixtures: None,
limit,
split: None,
}
}
#[cfg(unix)]
#[test]
fn a_declaration_that_could_not_be_checked_says_so_rather_than_reading_as_clean() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
corpus(&root, 3, 3);
let entry = sizing_detector(tmp.path());
let mut req = request(&root, None);
req.trained_on = Some("some-other-corpus");
let result = score(&entry, &req, |_| {}).expect("scores").0;
assert!(
!result.declarations.contaminated,
"the names do not match, so this is not a proven contamination"
);
assert_eq!(
result.declarations.contamination_check,
stegobench_core::result::ContaminationCheck::DeclaredButUncheckable,
"`false` was published as though the check had run"
);
assert_eq!(
result.declarations.trained_on.as_deref(),
Some("some-other-corpus"),
"the declaration itself was dropped"
);
}
#[cfg(unix)]
#[test]
fn a_run_with_no_declaration_is_not_marked_uncheckable() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
corpus(&root, 3, 3);
let entry = sizing_detector(tmp.path());
let result = score(&entry, &request(&root, None), |_| {})
.expect("scores")
.0;
assert_eq!(
result.declarations.contamination_check,
stegobench_core::result::ContaminationCheck::NotDeclared
);
}
#[cfg(unix)]
#[test]
fn the_document_says_which_side_of_the_split_was_scored() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
split_corpus(&root, false);
let entry = sizing_detector(tmp.path());
let mut req = request(&root, None);
req.split = Some("test");
req.records = root.with_extension("test.records.jsonl");
let held_out = score(&entry, &req, |_| {}).expect("scores").0;
assert_eq!(held_out.declarations.split_scored.as_deref(), Some("test"));
let whole = score(&entry, &request(&root, None), |_| {})
.expect("scores")
.0;
assert_eq!(
whole.declarations.split_scored, None,
"a whole corpus run claimed a side"
);
assert_ne!(
held_out.declarations, whole.declarations,
"the two runs are still indistinguishable in the document"
);
}
#[cfg(unix)]
#[test]
fn a_corpus_whose_two_classes_are_different_formats_says_so() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
std::fs::create_dir_all(&root).unwrap();
for i in 0..3 {
std::fs::write(root.join(format!("c{i}.png")), png(64, 64, 8, 2, i)).unwrap();
std::fs::write(
root.join(format!("c{i}.json")),
format!(r#"{{"role":"clean","sha256":"{i}"}}"#),
)
.unwrap();
std::fs::write(root.join(format!("s{i}.jpg")), jpeg(64, 64)).unwrap();
std::fs::write(
root.join(format!("s{i}.json")),
format!(
r#"{{"role":"stego","sha256":"s{i}","arm":{{"tool":"steghide"}},"clean":"nowhere/{i}.jpg","clean_sha256":"absent{i}"}}"#
),
)
.unwrap();
}
let entry = sizing_detector(tmp.path());
let mut said = Vec::new();
let result = score(&entry, &request(&root, None), |m: &str| {
said.push(m.to_string())
})
.expect("scores")
.0;
assert_eq!(result.arm.clean_format.as_deref(), Some("png"));
assert_eq!(result.arm.format, "jpeg");
assert!(
said.iter()
.any(|m| m.contains("every clean image here is png and every stego image is jpeg")),
"nothing said the two classes are different formats: {said:?}"
);
assert!(
said.iter().any(|m| m.contains("not evidence of")),
"the warning did not say what it means: {said:?}"
);
}
#[cfg(unix)]
#[test]
fn a_corpus_of_one_format_throughout_is_not_warned_about() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
corpus(&root, 3, 3);
let entry = sizing_detector(tmp.path());
let mut said = Vec::new();
score(&entry, &request(&root, None), |m: &str| {
said.push(m.to_string())
})
.expect("scores");
assert!(
!said.iter().any(|m| m.contains("every clean image here is")),
"{said:?}"
);
}
#[test]
fn the_feed_hands_out_one_item_at_a_time_in_corpus_order() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
corpus(&root, 2, 2);
let feed = Feed::filtered(&root, None, None).expect("opens");
let ids: Vec<String> = feed.map(|w| w.id).collect();
assert_eq!(ids, ["c000", "c001", "s000", "s001"]);
}
#[test]
fn a_limit_stops_the_feed_early() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
corpus(&root, 4, 4);
let feed = Feed::filtered(&root, Some(3), None).expect("opens");
assert_eq!(feed.count(), 3);
}
#[test]
fn a_corpus_defect_stops_the_feed_and_is_kept_rather_than_read_as_the_end() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
corpus(&root, 2, 2);
std::fs::write(root.join("orphan.png"), b"\x89PNG\r\n\x1a\n").unwrap();
let mut feed = Feed::filtered(&root, None, None).expect("opens");
let seen = feed.by_ref().count();
assert!(
feed.fault.is_some(),
"a defect was read as the end, after {seen}"
);
}
#[cfg(unix)]
fn sizing_detector(dir: &Path) -> Entry {
use std::os::unix::fs::PermissionsExt;
let script = dir.join("size.sh");
std::fs::write(
&script,
"#!/bin/sh\n[ \"$1\" = \"--version\" ] && { echo v1; exit 0; }\n\
wc -c < \"$1\" | tr -d ' ' | awk '{print $1/1000}'\n",
)
.unwrap();
let mut perms = std::fs::metadata(&script).unwrap().permissions();
perms.set_mode(0o755);
std::fs::set_permissions(&script, perms).unwrap();
toml::from_str(&format!(
"name = \"sizer\"\nkind = \"detector\"\nlicence = \"MIT\"\n\
[binary]\ncommand = [{:?}]\nversion_args = [\"--version\"]\n\
[emits]\noutput = \"score\"\nhigher_means_stego = true\n\
[invoke]\nargv = [\"{{file}}\"]\nparser = \"number\"\n\
[selftest]\nmust_detect = \"fixtures/a.png\"\nmust_clear = \"fixtures/b.png\"\n",
script.display().to_string()
))
.expect("parses")
}
#[cfg(unix)]
#[test]
fn two_runs_over_the_same_corpus_agree_on_every_number() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
corpus(&root, 3, 3);
let entry = sizing_detector(tmp.path());
let first = score(&entry, &request(&root, None), |_| {})
.expect("first run")
.0;
let mut second_request = request(&root, None);
second_request.records = root.with_extension("second.jsonl");
let second = score(&entry, &second_request, |_| {})
.expect("second run")
.0;
assert_eq!(first.metrics, second.metrics);
assert_eq!(first.corpus, second.corpus);
assert_eq!(first.declarations, second.declarations);
assert_eq!(first.subject, second.subject);
}
#[cfg(unix)]
fn silent_detector(dir: &Path) -> Entry {
use std::os::unix::fs::PermissionsExt;
let script = dir.join("silent.sh");
std::fs::write(
&script,
"#!/bin/sh\n[ \"$1\" = \"--version\" ] && { echo v1; exit 0; }\nexit 0\n",
)
.unwrap();
let mut perms = std::fs::metadata(&script).unwrap().permissions();
perms.set_mode(0o755);
std::fs::set_permissions(&script, perms).unwrap();
toml::from_str(&format!(
"name = \"mute\"\nkind = \"detector\"\nlicence = \"MIT\"\n\
[binary]\ncommand = [{:?}]\nversion_args = [\"--version\"]\n\
[emits]\noutput = \"score\"\nhigher_means_stego = true\n\
[invoke]\nargv = [\"{{file}}\"]\nparser = \"number\"\n\
[selftest]\nmust_detect = \"fixtures/a.png\"\nmust_clear = \"fixtures/b.png\"\n",
script.display().to_string()
))
.expect("parses")
}
#[cfg(unix)]
#[test]
fn a_detector_that_answered_nothing_is_blamed_rather_than_the_corpus() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
corpus(&root, 3, 3);
let entry = silent_detector(tmp.path());
let err = score(&entry, &request(&root, None), |_| {}).expect_err("refused");
let said = err.to_string();
assert_eq!(err.reason(), "nothing-answered", "{said}");
assert!(
said.contains("3 clean and 3 stego"),
"the corpus was reported as empty: {said}"
);
assert!(
said.contains("The corpus is not the problem here"),
"nothing pointed at the tool: {said}"
);
assert!(said.contains("mute"), "the tool is not named: {said}");
assert!(
said.contains("--keep-raw"),
"no way to see what the tool printed: {said}"
);
}
#[cfg(unix)]
#[test]
fn a_corpus_with_only_one_side_is_still_the_thing_blamed() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
corpus(&root, 3, 0);
let entry = sizing_detector(tmp.path());
let err = score(&entry, &request(&root, None), |_| {}).expect_err("refused");
assert_eq!(err.reason(), "one-sided", "{err}");
}
#[cfg(unix)]
fn constant_detector(dir: &Path) -> Entry {
use std::os::unix::fs::PermissionsExt;
let script = dir.join("constant.sh");
std::fs::write(
&script,
"#!/bin/sh\n[ \"$1\" = \"--version\" ] && { echo v1; exit 0; }\necho 0.5\n",
)
.unwrap();
let mut perms = std::fs::metadata(&script).unwrap().permissions();
perms.set_mode(0o755);
std::fs::set_permissions(&script, perms).unwrap();
toml::from_str(&format!(
"name = \"constant\"\nkind = \"detector\"\nlicence = \"MIT\"\n\
[binary]\ncommand = [{:?}]\nversion_args = [\"--version\"]\n\
[emits]\noutput = \"score\"\nhigher_means_stego = true\n\
[invoke]\nargv = [\"{{file}}\"]\nparser = \"number\"\n\
[selftest]\nmust_detect = \"fixtures/a.png\"\nmust_clear = \"fixtures/b.png\"\n",
script.display().to_string()
))
.expect("parses")
}
#[test]
fn an_estimate_twelve_times_out_is_corrected_in_the_direction_it_is_wrong() {
let said = revised_estimate(
32,
521,
553,
Duration::from_secs_f64(570.0),
Some(1.4 * 553.0),
)
.expect("a 12x miss is worth saying");
assert!(said.contains("measured on this machine"), "{said}");
assert!(
said.contains("13x what this detector's declared rate predicted"),
"{said}"
);
assert!(
said.contains("2.6 hours"),
"the projection is missing: {said}"
);
}
#[test]
fn an_estimate_that_was_about_right_says_nothing_at_all() {
assert_eq!(
revised_estimate(
20,
80,
100,
Duration::from_secs_f64(20.0),
Some(1.0 * 100.0)
),
None,
"a run going to plan interrupted the reader to say so"
);
}
#[test]
fn a_run_faster_than_promised_is_also_worth_saying() {
let said = revised_estimate(
50,
50,
100,
Duration::from_secs_f64(5.0),
Some(10.0 * 100.0),
)
.expect("a 100x overestimate is worth saying");
assert!(said.contains("faster"), "{said}");
}
#[test]
fn a_detector_declaring_no_rate_gets_a_measurement_rather_than_a_comparison() {
let said = revised_estimate(16, 984, 1000, Duration::from_secs_f64(160.0), None)
.expect("a measurement stands on its own");
assert!(said.contains("declares no rate"), "{said}");
assert!(said.contains("2.7 hours"), "{said}");
}
#[test]
fn nothing_is_claimed_before_anything_has_been_scored() {
assert_eq!(
revised_estimate(0, 100, 100, Duration::from_secs_f64(30.0), Some(100.0)),
None,
"a rate was divided by zero images"
);
}
#[cfg(unix)]
fn verdict_detector(dir: &Path) -> Entry {
use std::os::unix::fs::PermissionsExt;
let script = dir.join("verdict.sh");
std::fs::write(
&script,
"#!/bin/sh\n[ \"$1\" = \"--version\" ] && { echo v1; exit 0; }\n\
case \"$1\" in *s*) echo \"[?] b1,lsb,bY .. text: hello\" ;; esac\n",
)
.unwrap();
let mut perms = std::fs::metadata(&script).unwrap().permissions();
perms.set_mode(0o755);
std::fs::set_permissions(&script, perms).unwrap();
toml::from_str(&format!(
"name = \"verdicter\"\nkind = \"detector\"\nlicence = \"MIT\"\n\
[binary]\ncommand = [{:?}]\nversion_args = [\"--version\"]\n\
[emits]\noutput = \"verdict\"\n\
[invoke]\nargv = [\"{{file}}\"]\nparser = \"zsteg\"\n\
[selftest]\nmust_detect = \"fixtures/a.png\"\nmust_clear = \"fixtures/b.png\"\n",
script.display().to_string()
))
.expect("parses")
}
#[cfg(unix)]
#[test]
fn a_verdict_only_detector_says_so_rather_than_reading_as_a_gap() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
corpus(&root, 3, 3);
let entry = verdict_detector(tmp.path());
let mut said = Vec::new();
let result = score(&entry, &request(&root, None), |m: &str| {
said.push(m.to_string())
})
.expect("a verdict detector still produces a result")
.0;
assert_eq!(result.metrics.n_error, 0, "a verdict is an answer");
assert!(
said.iter().any(|m| m.contains("not a gap")),
"nothing explained the null scores: {said:?}"
);
assert!(
said.iter().any(|m| m.contains("no threshold to choose")),
"nothing said a verdict detector has no cutoff: {said:?}"
);
assert!(
result.metrics.threshold_at_fpr.is_empty()
|| result
.metrics
.threshold_at_fpr
.values()
.all(|t| *t == 0.0 || *t == 1.0),
"a verdict detector's only cutoffs are its two answers: {:?}",
result.metrics.threshold_at_fpr
);
}
#[cfg(unix)]
#[test]
fn a_scoring_detector_is_not_told_it_answers_a_verdict() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
corpus(&root, 3, 3);
let entry = sizing_detector(tmp.path());
let mut said = Vec::new();
score(&entry, &request(&root, None), |m: &str| {
said.push(m.to_string())
})
.expect("scores");
assert!(
!said.iter().any(|m| m.contains("not a gap")),
"a scoring detector was told it answers a verdict: {said:?}"
);
}
#[cfg(unix)]
#[test]
fn one_answer_to_everything_is_named_rather_than_published_as_chance() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
corpus(&root, 3, 3);
let entry = constant_detector(tmp.path());
let mut said = Vec::new();
let result = score(&entry, &request(&root, None), |m: &str| {
said.push(m.to_string())
})
.expect("a constant detector still produces a result")
.0;
assert!(
(result.metrics.auc - 0.5).abs() < 1e-12,
"{:?}",
result.metrics
);
assert_eq!(result.metrics.n_error, 0, "it answered every image");
let warned = said.iter().any(|m| m.contains("identical"));
assert!(
warned,
"nothing said the answers were all the same: {said:?}"
);
assert!(
said.iter().any(|m| m.contains("by construction")),
"the warning does not say why the 0.5 is not a measurement: {said:?}"
);
}
#[cfg(unix)]
#[test]
fn a_declared_training_corpus_is_recorded_and_the_self_scored_case_is_warned_about() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
corpus(&root, 3, 3);
let entry = sizing_detector(tmp.path());
let registered = registered_corpus(None);
let mut elsewhere = request(&root, None);
elsewhere.registered = Some(®istered);
elsewhere.trained_on = Some("some-other-corpus");
let mut said = Vec::new();
let result = score(&entry, &elsewhere, |m: &str| said.push(m.to_string()))
.expect("a result")
.0;
assert_eq!(
result.declarations.trained_on.as_deref(),
Some("some-other-corpus"),
"the declaration did not reach the document"
);
assert!(
!said.iter().any(|m| m.contains("WARNING")),
"training on a different corpus is not a warning: {said:?}"
);
let root2 = tmp.path().join("corpus2");
corpus(&root2, 3, 3);
let mut itself = request(&root2, None);
itself.registered = Some(®istered);
itself.trained_on = Some("example");
let mut said = Vec::new();
let result = score(&entry, &itself, |m: &str| said.push(m.to_string()))
.expect("a result")
.0;
assert_eq!(result.declarations.trained_on.as_deref(), Some("example"));
assert!(
said.iter()
.any(|m| m.contains("WARNING") && m.contains("scored against")),
"scoring a detector on what it trained on has to say so: {said:?}"
);
}
#[cfg(unix)]
#[test]
fn a_detector_that_separates_the_images_is_not_warned_about() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
corpus(&root, 3, 3);
let entry = sizing_detector(tmp.path());
let mut said = Vec::new();
score(&entry, &request(&root, None), |m: &str| {
said.push(m.to_string())
})
.expect("run");
assert!(
!said.iter().any(|m| m.contains("identical")),
"warned about a detector that did separate the images: {said:?}"
);
}
#[cfg(unix)]
#[test]
fn a_resumed_run_reports_the_same_numbers_as_an_uninterrupted_one() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
corpus(&root, 3, 3);
let entry = sizing_detector(tmp.path());
let whole = score(&entry, &request(&root, None), |_| {})
.expect("whole")
.0;
let mut partial = request(&root, Some(4));
partial.records = root.with_extension("resumed.jsonl");
score(&entry, &partial, |_| {}).expect("partial");
let mut rest = request(&root, None);
rest.records = partial.records.clone();
let (resumed, tally) = score(&entry, &rest, |_| {}).expect("resumed");
assert_eq!(tally.resumed, 4, "the scored part should have been reused");
assert_eq!(whole.metrics, resumed.metrics);
}
fn split_corpus(root: &Path, leak: bool) {
std::fs::create_dir_all(root).expect("corpus");
for i in 0..3 {
let split = if i == 0 { "test" } else { "train" };
std::fs::write(root.join(format!("c{i}.png")), png(32, 32, 8, 2, 0)).unwrap();
std::fs::write(
root.join(format!("c{i}.json")),
format!(r#"{{"role":"clean","split":"{split}","sha256":"0"}}"#),
)
.unwrap();
let stego_split = if leak && i == 0 { "train" } else { split };
std::fs::write(root.join(format!("s{i}.png")), png(32, 32, 8, 2, 64)).unwrap();
std::fs::write(
root.join(format!("s{i}.json")),
format!(
r#"{{"role":"stego","source_png":"c{i}.png","split":"{stego_split}","sha256":"0"}}"#
),
)
.unwrap();
}
}
#[test]
fn a_corpus_that_keeps_twins_together_is_reported_as_by_cover() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
split_corpus(&root, false);
let checks = check(&root).expect("checked");
assert!(
checks.split_leaks.is_empty(),
"{}",
checks.split_leaks.examples()
);
assert_eq!(checks.split, SplitDiscipline::ByCover);
}
#[cfg(unix)]
#[test]
fn a_cover_split_from_its_twin_stops_the_run() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
split_corpus(&root, true);
let checks = check(&root).expect("checked");
assert_eq!(checks.split_leaks.count, 1);
assert!(
checks.split_leaks.examples().contains("c0.png"),
"{}",
checks.split_leaks.examples()
);
let entry = sizing_detector(tmp.path());
let err = score(&entry, &request(&root, None), |_| {}).expect_err("refused");
assert!(err.to_string().contains("different side"), "{err}");
}
#[cfg(unix)]
#[test]
fn scoring_the_test_half_leaves_the_train_half_out_and_says_which_it_used() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
split_corpus(&root, false);
let prepared = prepare(&root, None, None, None, Some("test"), |_| {}).expect("prepared");
assert_eq!(prepared.side, Some(Side::Test));
assert_eq!(prepared.items(), 2);
assert_eq!(prepared.keep, [true, false, false, true, false, false]);
let entry = sizing_detector(tmp.path());
let (doc, _) = score_one(
&entry,
&prepared,
&root.with_extension("t.jsonl"),
How {
timeout: Duration::from_secs(5),
jobs: 1,
keep_raw: false,
seed: None,
fixtures: None,
adapter_roots: &[],
},
|_| {},
)
.expect("scored");
assert_eq!(doc.corpus.split.as_deref(), Some("test"));
assert_eq!(doc.corpus.pairs, 1, "one stego image in the test half");
}
#[cfg(unix)]
#[test]
fn scoring_the_train_half_covers_the_samples_the_test_half_did_not() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
split_corpus(&root, false);
let test = prepare(&root, None, None, None, Some("test"), |_| {}).expect("test half");
let train = prepare(&root, None, None, None, Some("train"), |_| {}).expect("train half");
assert_eq!(train.side, Some(Side::Train));
assert_eq!(train.items(), 4);
for (a, b) in test.keep.iter().zip(train.keep.iter()) {
assert!(
!(*a && *b),
"a sample landed in both halves: {:?}",
test.keep
);
}
let both: Vec<bool> = test
.keep
.iter()
.zip(train.keep.iter())
.map(|(a, b)| *a || *b)
.collect();
assert!(both.iter().all(|x| *x), "a sample landed in neither half");
}
#[test]
fn two_arms_are_measured_separately_against_the_shared_clean_set() {
let names = vec!["weak-0100".to_string(), "loud-0400".to_string()];
let scores = vec![0.1, 0.2, 0.15, 0.25, 0.9, 0.95];
let labels = vec![false, false, true, true, true, true];
let arms = vec![None, None, Some(0), Some(0), Some(1), Some(1)];
let rows = per_arm(&scores, &labels, &arms, &names);
assert_eq!(rows.len(), 2, "{rows:?}");
let weak = rows.iter().find(|r| r.arm == "weak-0100").expect("weak");
let loud = rows.iter().find(|r| r.arm == "loud-0400").expect("loud");
assert_eq!(weak.n_clean, 2);
assert_eq!(weak.n_stego, 2);
assert_eq!(loud.n_clean, 2, "the clean set is shared, not divided");
assert_eq!(loud.n_stego, 2);
assert!(
loud.auc > weak.auc,
"the separable arm should score higher: {rows:?}"
);
assert_eq!(loud.auc, 1.0);
}
#[test]
fn one_arm_gets_no_breakdown_because_it_would_be_the_headline_twice() {
let names = vec!["only-0100".to_string()];
let scores = vec![0.1, 0.9];
let labels = vec![false, true];
let arms = vec![None, Some(0)];
assert!(per_arm(&scores, &labels, &arms, &names).is_empty());
}
#[test]
fn a_corpus_whose_records_name_no_arm_gets_no_breakdown() {
let scores = vec![0.1, 0.2, 0.9, 0.95];
let labels = vec![false, false, true, true];
let arms = vec![None, None, None, None];
assert!(per_arm(&scores, &labels, &arms, &[]).is_empty());
}
#[test]
fn an_arm_the_detector_answered_nothing_about_is_left_out_not_scored_zero() {
let names = vec!["a".to_string(), "b".to_string(), "c".to_string()];
let scores = vec![0.1, 0.9, 0.8];
let labels = vec![false, true, true];
let arms = vec![None, Some(0), Some(1)];
let rows = per_arm(&scores, &labels, &arms, &names);
assert_eq!(rows.len(), 2);
assert!(!rows.iter().any(|r| r.arm == "c"), "{rows:?}");
}
#[test]
fn the_arm_table_holds_one_entry_per_distinct_name_however_often_it_repeats() {
let mut table = Vec::new();
for _ in 0..1000 {
assert_eq!(intern(&mut table, Some("wow-0200")), Some(0));
assert_eq!(intern(&mut table, Some("hill-0400")), Some(1));
}
assert_eq!(table.len(), 2);
assert_eq!(intern(&mut table, None), None);
assert_eq!(table.len(), 2, "None must not become an arm");
}
#[cfg(unix)]
fn said(entry: &Entry, prepared: &Prepared, records: &Path) -> String {
let mut lines = Vec::new();
let _ = score_one(
entry,
prepared,
records,
How {
timeout: Duration::from_secs(5),
jobs: 1,
keep_raw: false,
seed: None,
fixtures: None,
adapter_roots: &[],
},
|m| lines.push(m.to_string()),
);
lines.join("\n")
}
#[cfg(unix)]
#[test]
fn a_budget_the_corpus_cannot_express_is_said_out_loud() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
corpus(&root, 3, 3);
let prepared = prepare(&root, None, None, None, None, |_| {}).expect("prepared");
let entry = sizing_detector(tmp.path());
let text = said(&entry, &prepared, &root.with_extension("u.jsonl"));
assert!(
text.contains("cannot express a false-alarm budget finer than"),
"{text}"
);
assert!(text.contains("1%"), "the budgets should be named: {text}");
assert!(
text.contains("The number is real"),
"it must not read as the figure being wrong: {text}"
);
}
#[cfg(unix)]
#[test]
fn a_corpus_large_enough_for_the_budget_is_not_warned_about() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
corpus(&root, 120, 120);
let prepared = prepare(&root, None, None, None, None, |_| {}).expect("prepared");
let entry = sizing_detector(tmp.path());
let text = said(&entry, &prepared, &root.with_extension("v.jsonl"));
assert!(
!text.contains("cannot express a false-alarm budget"),
"120 clean images can express 1%: {text}"
);
}
#[test]
fn a_budget_is_expressible_exactly_when_one_clean_image_fits_inside_it() {
assert!(stegobench_metrics::fpr_resolution(100) <= 0.01);
assert!(stegobench_metrics::fpr_resolution(99) > 0.01);
}
#[test]
fn a_whole_corpus_run_records_no_split_rather_than_guessing_one() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
split_corpus(&root, false);
let prepared = prepare(&root, None, None, None, None, |_| {}).expect("prepared");
assert_eq!(prepared.side, None);
assert_eq!(prepared.items(), 6);
}
#[test]
fn asking_for_a_half_of_a_corpus_that_has_none_is_refused() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
corpus(&root, 2, 2);
let err = prepare(&root, None, None, None, Some("test"), |_| {})
.err()
.expect("refused");
assert!(err.to_string().contains("no train and test split"), "{err}");
assert_eq!(err.exit_code(), 3);
}
#[test]
fn a_side_that_is_not_a_side_is_a_usage_error_naming_both_halves() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
split_corpus(&root, false);
let err = prepare(&root, None, None, None, Some("holdout"), |_| {})
.err()
.expect("refused");
let text = err.to_string();
assert!(text.contains("holdout"), "{text}");
assert!(text.contains("train") && text.contains("test"), "{text}");
assert_eq!(err.exit_code(), 2);
}
#[test]
fn a_half_holding_one_class_is_refused_and_the_message_names_the_half() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
std::fs::create_dir_all(&root).unwrap();
for (name, body) in [
("c0.json", r#"{"role":"clean","split":"test","sha256":"0"}"#),
(
"c1.json",
r#"{"role":"clean","split":"train","sha256":"0"}"#,
),
] {
std::fs::write(root.join(name), body).unwrap();
}
std::fs::write(root.join("c0.png"), png(32, 32, 8, 2, 0)).unwrap();
std::fs::write(root.join("c1.png"), png(32, 32, 8, 2, 0)).unwrap();
std::fs::write(root.join("s1.png"), png(32, 32, 8, 2, 64)).unwrap();
std::fs::write(
root.join("s1.json"),
r#"{"role":"stego","source_png":"c1.png","split":"train","sha256":"0"}"#,
)
.unwrap();
let err = prepare(&root, None, None, None, Some("test"), |_| {})
.err()
.expect("refused");
let text = err.to_string();
assert!(text.contains("test split"), "{text}");
assert!(text.contains("has not been measured"), "{text}");
}
#[test]
fn a_corpus_with_no_split_labels_says_not_applicable_rather_than_by_cover() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
corpus(&root, 2, 2);
let checks = check(&root).expect("checked");
assert!(checks.split_leaks.is_empty());
assert_eq!(checks.split, SplitDiscipline::NotApplicable);
}
#[test]
fn a_stego_row_inheriting_its_cover_split_is_not_a_violation() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
std::fs::create_dir_all(&root).unwrap();
std::fs::write(root.join("c0.png"), png(32, 32, 8, 2, 0)).unwrap();
std::fs::write(
root.join("c0.json"),
r#"{"role":"clean","split":"test","sha256":"0"}"#,
)
.unwrap();
std::fs::write(root.join("s0.png"), png(32, 32, 8, 2, 64)).unwrap();
std::fs::write(
root.join("s0.json"),
r#"{"role":"stego","source_png":"c0.png","sha256":"0"}"#,
)
.unwrap();
let checks = check(&root).expect("checked");
assert!(
checks.split_leaks.is_empty(),
"{}",
checks.split_leaks.examples()
);
assert_eq!(checks.split, SplitDiscipline::ByCover);
}
fn pair(root: &Path, stego_image: &[u8]) {
std::fs::create_dir_all(root).expect("corpus");
std::fs::write(root.join("c0.png"), png(64, 64, 8, 2, 0)).unwrap();
std::fs::write(root.join("c0.json"), r#"{"role":"clean","sha256":"0"}"#).unwrap();
std::fs::write(root.join("s0.png"), stego_image).unwrap();
std::fs::write(
root.join("s0.json"),
r#"{"role":"stego","source_png":"c0.png","sha256":"0"}"#,
)
.unwrap();
}
#[test]
fn a_stego_image_shaped_like_its_cover_is_reported_as_single_variable() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
pair(&root, &png(64, 64, 8, 2, 128));
let checks = check(&root).expect("checked");
assert!(checks.pairing_breaks.is_empty());
assert_eq!(checks.compared, 1);
assert_eq!(checks.pairing, Pairing::SingleVariable);
}
#[test]
fn a_stego_image_a_different_size_from_its_cover_is_confounded() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
pair(&root, &png(63, 64, 8, 2, 0));
let checks = check(&root).expect("checked");
assert_eq!(checks.pairing, Pairing::Confounded);
assert_eq!(checks.pairing_breaks.count, 1);
assert!(
checks.pairing_breaks.examples().contains("different sizes"),
"{}",
checks.pairing_breaks.examples()
);
}
#[test]
fn a_stego_image_in_another_format_is_confounded_and_says_which_two() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
let mut jpeg = vec![0xFF, 0xD8, 0xFF, 0xC0, 0x00, 0x11, 0x08];
jpeg.extend_from_slice(&64u16.to_be_bytes());
jpeg.extend_from_slice(&64u16.to_be_bytes());
jpeg.push(3);
jpeg.extend_from_slice(&[1, 0x11, 0, 2, 0x11, 0, 3, 0x11, 0]);
pair(&root, &jpeg);
let checks = check(&root).expect("checked");
assert_eq!(checks.pairing, Pairing::Confounded);
let seen = checks.pairing_breaks.examples();
assert!(seen.contains("PNG") && seen.contains("JPEG"), "{seen}");
}
#[test]
fn a_stego_image_of_another_bit_depth_or_channel_count_is_confounded() {
for stego in [png(64, 64, 16, 2, 0), png(64, 64, 8, 6, 0)] {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
pair(&root, &stego);
let checks = check(&root).expect("checked");
assert_eq!(checks.pairing, Pairing::Confounded);
}
}
fn jpeg(width: u16, height: u16) -> Vec<u8> {
let mut bytes = vec![0xFF, 0xD8, 0xFF, 0xC0, 0x00, 0x11, 0x08];
bytes.extend_from_slice(&height.to_be_bytes());
bytes.extend_from_slice(&width.to_be_bytes());
bytes.push(3);
bytes.extend_from_slice(&[1, 0x11, 0, 2, 0x11, 0, 3, 0x11, 0]);
bytes
}
fn jpeg_arm(root: &Path, stego_image: &[u8], clean_digest: Option<&str>) {
std::fs::create_dir_all(root).expect("corpus");
std::fs::write(root.join("c0.png"), png(64, 64, 8, 2, 0)).unwrap();
std::fs::write(
root.join("c0.json"),
r#"{"role":"clean","file":"c0.png","sha256":"covercrop"}"#,
)
.unwrap();
std::fs::write(root.join("k0.jpg"), jpeg(64, 64)).unwrap();
std::fs::write(
root.join("k0.json"),
r#"{"role":"clean","file":"clean_jpeg/00000.jpg","sha256":"cleanhalf"}"#,
)
.unwrap();
std::fs::write(root.join("s0.jpg"), stego_image).unwrap();
let clean = match clean_digest {
Some(digest) => format!(r#","clean_sha256":"{digest}""#),
None => String::new(),
};
std::fs::write(
root.join("s0.json"),
format!(r#"{{"role":"stego","source_png":"c0.png","sha256":"0"{clean}}}"#),
)
.unwrap();
}
#[test]
fn a_jpeg_arm_is_compared_against_its_clean_jpeg_half_and_not_against_the_cover_crop() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
jpeg_arm(&root, &jpeg(64, 64), Some("cleanhalf"));
let checks = check(&root).expect("checked");
assert!(
checks.pairing_breaks.is_empty(),
"{}",
checks.pairing_breaks.examples()
);
assert_eq!(checks.compared, 1);
assert_eq!(checks.unreadable, 0);
assert_eq!(checks.pairing, Pairing::SingleVariable);
}
#[test]
fn the_same_corpus_stating_no_clean_half_falls_back_to_the_cover_and_is_confounded() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
jpeg_arm(&root, &jpeg(64, 64), None);
let checks = check(&root).expect("checked");
assert_eq!(checks.pairing, Pairing::Confounded);
let seen = checks.pairing_breaks.examples();
assert!(seen.contains("its cover c0.png"), "{seen}");
}
#[test]
fn a_clean_half_the_corpus_names_but_does_not_ship_is_uncomparable_rather_than_confounded() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
jpeg_arm(&root, &jpeg(64, 64), Some("a-digest-no-row-here-states"));
let checks = check(&root).expect("checked");
assert_eq!(checks.compared, 0);
assert_eq!(checks.unreadable, 1);
assert!(checks.pairing_breaks.is_empty());
assert_eq!(checks.pairing, Pairing::Unverified);
}
#[test]
fn the_two_unverified_corpora_are_told_apart_by_the_counts_rather_than_the_verdict() {
let tmp = tempfile::tempdir().expect("tmp");
let dangling = tmp.path().join("dangling");
jpeg_arm(
&dangling,
&jpeg(64, 64),
Some("a-digest-no-row-here-states"),
);
let dangling = check(&dangling).expect("checked");
let silent = tmp.path().join("silent");
std::fs::create_dir_all(&silent).unwrap();
std::fs::write(silent.join("c0.png"), png(64, 64, 8, 2, 0)).unwrap();
std::fs::write(silent.join("c0.json"), r#"{"role":"clean","sha256":"a"}"#).unwrap();
std::fs::write(silent.join("s0.png"), png(64, 64, 8, 2, 9)).unwrap();
std::fs::write(silent.join("s0.json"), r#"{"role":"stego","sha256":"b"}"#).unwrap();
let silent = check(&silent).expect("checked");
assert_eq!(dangling.pairing, Pairing::Unverified);
assert_eq!(silent.pairing, Pairing::Unverified);
assert_eq!(
dangling.unreadable, 1,
"the dangling pointer was not counted"
);
assert_eq!(
silent.unreadable, 0,
"a corpus that claimed nothing was blamed"
);
}
#[test]
fn a_stego_image_differing_from_its_clean_half_is_still_confounded_and_says_so() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
jpeg_arm(&root, &jpeg(63, 64), Some("cleanhalf"));
let checks = check(&root).expect("checked");
assert_eq!(checks.pairing, Pairing::Confounded);
let seen = checks.pairing_breaks.examples();
assert!(seen.contains("its clean half"), "{seen}");
assert!(seen.contains("different sizes"), "{seen}");
}
#[test]
fn the_split_still_joins_on_the_cover_when_the_pairing_joins_on_the_digest() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
jpeg_arm(&root, &jpeg(64, 64), Some("cleanhalf"));
std::fs::write(
root.join("c0.json"),
r#"{"role":"clean","file":"c0.png","sha256":"covercrop","split":"test"}"#,
)
.unwrap();
std::fs::write(
root.join("s0.json"),
r#"{"role":"stego","source_png":"c0.png","sha256":"0","split":"train","clean_sha256":"cleanhalf"}"#,
)
.unwrap();
let checks = check(&root).expect("checked");
assert_eq!(checks.split_leaks.count, 1);
assert_eq!(checks.split, SplitDiscipline::ByCover);
assert_eq!(checks.pairing, Pairing::SingleVariable);
}
#[cfg(unix)]
#[test]
fn a_confounded_corpus_is_scored_and_the_document_says_so() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
std::fs::create_dir_all(&root).unwrap();
for i in 0..2 {
std::fs::write(root.join(format!("c{i}.png")), png(64, 64, 8, 2, i)).unwrap();
std::fs::write(
root.join(format!("c{i}.json")),
r#"{"role":"clean","sha256":"0"}"#,
)
.unwrap();
std::fs::write(root.join(format!("s{i}.png")), png(48, 48, 8, 2, 64 + i)).unwrap();
std::fs::write(
root.join(format!("s{i}.json")),
format!(r#"{{"role":"stego","source_png":"c{i}.png","sha256":"0"}}"#),
)
.unwrap();
}
let entry = sizing_detector(tmp.path());
let mut warnings = Vec::new();
let (result, _) = score(&entry, &request(&root, None), |line| {
warnings.push(line.to_string())
})
.expect("scored");
assert_eq!(result.declarations.pairing, Pairing::Confounded);
assert!(
warnings.iter().any(|w| w.starts_with("WARNING")),
"the run said nothing about it: {warnings:?}"
);
}
#[test]
fn a_corpus_nothing_can_be_compared_in_says_unverified() {
let tmp = tempfile::tempdir().expect("tmp");
let unreadable = tmp.path().join("unreadable");
std::fs::create_dir_all(&unreadable).unwrap();
std::fs::write(unreadable.join("c0.png"), b"\x89PNG\r\n\x1a\n").unwrap();
std::fs::write(
unreadable.join("c0.json"),
r#"{"role":"clean","sha256":"0"}"#,
)
.unwrap();
std::fs::write(unreadable.join("s0.png"), b"\x89PNG\r\n\x1a\n").unwrap();
std::fs::write(
unreadable.join("s0.json"),
r#"{"role":"stego","source_png":"c0.png","sha256":"0"}"#,
)
.unwrap();
let checks = check(&unreadable).expect("checked");
assert_eq!(checks.pairing, Pairing::Unverified);
assert_eq!(checks.unreadable, 1);
assert_eq!(checks.compared, 0);
let unlinked = tmp.path().join("unlinked");
std::fs::create_dir_all(&unlinked).unwrap();
std::fs::write(unlinked.join("c0.png"), png(64, 64, 8, 2, 0)).unwrap();
std::fs::write(unlinked.join("c0.json"), r#"{"role":"clean","sha256":"0"}"#).unwrap();
std::fs::write(unlinked.join("s0.png"), png(64, 64, 8, 2, 1)).unwrap();
std::fs::write(unlinked.join("s0.json"), r#"{"role":"stego","sha256":"0"}"#).unwrap();
let checks = check(&unlinked).expect("checked");
assert_eq!(checks.pairing, Pairing::Unverified);
assert_eq!(checks.compared, 0);
}
#[test]
fn a_format_this_cannot_measure_is_compared_as_far_as_its_format_and_no_further() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
std::fs::create_dir_all(&root).unwrap();
std::fs::write(root.join("c0.bmp"), b"BM\x40\x00\x40\x00").unwrap();
std::fs::write(root.join("c0.json"), r#"{"role":"clean","sha256":"0"}"#).unwrap();
std::fs::write(root.join("s0.bmp"), b"BM\x20\x00\x20\x00").unwrap();
std::fs::write(
root.join("s0.json"),
r#"{"role":"stego","source_png":"c0.bmp","sha256":"0"}"#,
)
.unwrap();
let checks = check(&root).expect("checked");
assert_eq!(checks.compared, 1);
assert_eq!(checks.pairing, Pairing::SingleVariable);
}
#[test]
fn the_count_of_violations_is_exact_while_the_examples_are_capped() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
std::fs::create_dir_all(&root).unwrap();
std::fs::write(root.join("c0.png"), png(64, 64, 8, 2, 0)).unwrap();
std::fs::write(root.join("c0.json"), r#"{"role":"clean","sha256":"0"}"#).unwrap();
for i in 0..12 {
std::fs::write(root.join(format!("s{i:02}.png")), png(48, 48, 8, 2, i)).unwrap();
std::fs::write(
root.join(format!("s{i:02}.json")),
r#"{"role":"stego","source_png":"c0.png","sha256":"0"}"#,
)
.unwrap();
}
let checks = check(&root).expect("checked");
assert_eq!(checks.pairing_breaks.count, 12);
assert_eq!(checks.pairing_breaks.examples.len(), MAX_EXAMPLES);
}
fn digested(root: &Path, labels: &[&str]) {
std::fs::create_dir_all(root).expect("corpus");
for (i, label) in labels.iter().enumerate() {
let role = if i == 0 { "clean" } else { "stego" };
let mut bytes = png(64, 64, 8, 2, i);
bytes.extend_from_slice(label.as_bytes());
let image = root.join(format!("i{i}.png"));
std::fs::write(&image, &bytes).unwrap();
let sha = if label.is_empty() {
String::new()
} else {
format!(r#","sha256":"{}""#, hash_file(&image).expect("hashed"))
};
std::fs::write(
root.join(format!("i{i}.json")),
format!(r#"{{"role":"{role}"{sha}}}"#),
)
.unwrap();
}
}
#[test]
fn the_same_corpus_in_two_places_gets_the_same_digest() {
let tmp = tempfile::tempdir().expect("tmp");
let here = tmp.path().join("here");
let there = tmp.path().join("somewhere/else/entirely");
digested(&here, &["aa", "bb", "cc"]);
digested(&there, &["aa", "bb", "cc"]);
let a = check(&here).expect("checked").digest;
let b = check(&there).expect("checked").digest;
assert!(a.is_some(), "no digest was computed");
assert_eq!(a, b);
}
#[test]
fn one_changed_record_changes_the_corpus_digest() {
let tmp = tempfile::tempdir().expect("tmp");
let a = tmp.path().join("a");
let b = tmp.path().join("b");
digested(&a, &["aa", "bb", "cc"]);
digested(&b, &["aa", "bb", "cd"]);
assert_ne!(
check(&a).expect("checked").digest,
check(&b).expect("checked").digest
);
}
#[test]
fn a_corpus_missing_one_digest_carries_none_at_all() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
digested(&root, &["aa", "", "cc"]);
assert_eq!(check(&root).expect("checked").digest, None);
}
#[cfg(unix)]
#[test]
fn the_digest_reaches_the_result_document() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
digested(&root, &["aa", "bb", "cc"]);
let entry = sizing_detector(tmp.path());
let (result, _) = score(&entry, &request(&root, None), |_| {}).expect("scored");
assert!(
result.corpus.digest.starts_with("sha256:"),
"{:?}",
result.corpus.digest
);
result.validate().expect("valid");
}
fn armed(root: &Path, extra: &str, extension: &str) {
std::fs::create_dir_all(root).expect("corpus");
std::fs::write(root.join("c0.png"), png(64, 64, 8, 2, 0)).unwrap();
std::fs::write(
root.join("c0.json"),
r#"{"role":"clean","tool":"clean","sha256":"aa"}"#,
)
.unwrap();
std::fs::write(root.join(format!("s0.{extension}")), png(64, 64, 8, 2, 8)).unwrap();
std::fs::write(
root.join("s0.json"),
format!(r#"{{"role":"stego","sha256":"bb"{extra}}}"#),
)
.unwrap();
}
#[test]
fn the_arm_is_read_off_the_corpus_rather_than_assumed() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
armed(
&root,
r#","tool":"wow","arm":"wow-0200","rate":0.2,"rate_unit":"bits per pixel","domain":"spatial""#,
"png",
);
let arm = check(&root).expect("checked").arm;
assert_eq!(arm.embedder, "wow");
assert_eq!(arm.domain, Domain::Spatial);
assert_eq!(arm.format, "png");
let rate = arm.rate.expect("a rate");
assert_eq!(rate.unit, RateUnit::Bpp);
assert!((rate.value - 0.2).abs() < f64::EPSILON);
}
#[test]
fn a_clean_row_does_not_make_every_corpus_a_mixed_one() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
armed(&root, r#","tool":"hugo","domain":"spatial""#, "png");
assert_eq!(check(&root).expect("checked").arm.embedder, "hugo");
}
#[test]
fn a_jpeg_arm_is_not_labelled_spatial_and_a_capacity_share_is_not_bpp() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
armed(
&root,
r#","tool":"outguess","rate":0.05,"rate_unit":"fraction of the capacity outguess reports","domain":"jpeg-dct""#,
"jpg",
);
let arm = check(&root).expect("checked").arm;
assert_eq!(arm.embedder, "outguess");
assert_eq!(arm.domain, Domain::Jpeg);
assert_eq!(arm.format, "jpeg", "jpg and jpeg are one format");
assert_eq!(arm.rate.expect("a rate").unit, RateUnit::CapacityFraction);
}
#[test]
fn a_rate_in_a_unit_this_cannot_carry_is_dropped_rather_than_relabelled() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
armed(
&root,
r#","tool":"juniward","rate":0.4,"rate_unit":"bits per non-zero AC coefficient","domain":"jpeg-dct""#,
"jpg",
);
let arm = check(&root).expect("checked").arm;
assert_eq!(arm.rate, None);
assert_eq!(arm.embedder, "juniward");
}
#[test]
fn a_corpus_of_several_arms_says_mixed_rather_than_picking_one() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
std::fs::create_dir_all(&root).unwrap();
std::fs::write(root.join("c0.png"), png(64, 64, 8, 2, 0)).unwrap();
std::fs::write(root.join("c0.json"), r#"{"role":"clean","sha256":"a"}"#).unwrap();
for (i, (tool, domain)) in [("wow", "spatial"), ("outguess", "jpeg-dct")]
.into_iter()
.enumerate()
{
std::fs::write(root.join(format!("s{i}.png")), png(64, 64, 8, 2, i + 1)).unwrap();
std::fs::write(
root.join(format!("s{i}.json")),
format!(r#"{{"role":"stego","sha256":"b","tool":"{tool}","domain":"{domain}"}}"#),
)
.unwrap();
}
let arm = check(&root).expect("checked").arm;
assert_eq!(arm.embedder, "mixed");
assert_eq!(arm.domain, Domain::Mixed);
}
#[test]
fn a_corpus_that_says_nothing_about_its_arm_says_unstated() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
armed(&root, "", "png");
let arm = check(&root).expect("checked").arm;
assert_eq!(arm.embedder, "unstated");
assert_eq!(arm.rate, None);
assert_eq!(arm.domain, Domain::Spatial);
}
#[test]
fn a_cover_renamed_by_the_packer_is_still_found_by_the_name_its_record_gives() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
std::fs::create_dir_all(&root).unwrap();
std::fs::write(root.join("000000.png"), png(64, 64, 8, 2, 0)).unwrap();
std::fs::write(
root.join("000000.json"),
r#"{"role":"clean","file":"09710.png","split":"test","sha256":"a"}"#,
)
.unwrap();
std::fs::write(root.join("000001.png"), png(48, 48, 8, 2, 0)).unwrap();
std::fs::write(
root.join("000001.json"),
r#"{"role":"stego","file":"000001.png","source_png":"09710.png","split":"train","sha256":"b"}"#,
)
.unwrap();
let checks = check(&root).expect("checked");
assert_eq!(
checks.pairing,
Pairing::Confounded,
"the cover was never found, so nothing was compared"
);
assert_eq!(checks.split_leaks.count, 1, "the split join did not fire");
}
#[test]
fn an_arm_reusing_a_basename_cannot_displace_the_cover_it_collides_with() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
std::fs::create_dir_all(root.join("cover")).unwrap();
std::fs::create_dir_all(root.join("clean-grey")).unwrap();
std::fs::write(root.join("cover/000000.png"), png(64, 64, 8, 2, 0)).unwrap();
std::fs::write(
root.join("cover/000000.json"),
r#"{"role":"clean","file":"09710.png","sha256":"a"}"#,
)
.unwrap();
std::fs::write(root.join("clean-grey/000000.png"), png(32, 32, 8, 2, 0)).unwrap();
std::fs::write(
root.join("clean-grey/000000.json"),
r#"{"role":"clean","file":"000000.png","sha256":"b"}"#,
)
.unwrap();
std::fs::write(root.join("s0.png"), png(64, 64, 8, 2, 7)).unwrap();
std::fs::write(
root.join("s0.json"),
r#"{"role":"stego","source_png":"09710.png","sha256":"c"}"#,
)
.unwrap();
let checks = check(&root).expect("checked");
assert_eq!(checks.compared, 1);
assert_eq!(
checks.pairing,
Pairing::SingleVariable,
"the stego row was compared against the wrong image: {}",
checks.pairing_breaks.examples()
);
}
#[test]
fn a_manifest_carrying_a_path_in_its_file_field_still_joins() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
std::fs::create_dir_all(&root).unwrap();
std::fs::write(root.join("000000.png"), png(64, 64, 8, 2, 0)).unwrap();
std::fs::write(
root.join("000000.json"),
r#"{"role":"clean","file":"covers/09710.png","sha256":"a"}"#,
)
.unwrap();
std::fs::write(root.join("000001.png"), png(64, 64, 8, 2, 9)).unwrap();
std::fs::write(
root.join("000001.json"),
r#"{"role":"stego","source_png":"09710.png","sha256":"b"}"#,
)
.unwrap();
let checks = check(&root).expect("checked");
assert_eq!(checks.compared, 1);
assert_eq!(checks.pairing, Pairing::SingleVariable);
}
#[cfg(unix)]
fn registered_corpus(declared: Option<&str>) -> CorpusEntry {
let integrity = match declared {
Some(d) => format!("[integrity]\nrecords_sha256 = \"{d}\"\n"),
None => String::new(),
};
toml::from_str(&format!(
"id = \"example\"\nname = \"Example Tier\"\ntier = \"nano\"\n\
description = \"For a test.\"\n{integrity}\
[licence]\nstatus = \"unverified\"\nredistribution = \"unknown\"\n\
redistribution_reason = \"A fixture.\"\n\
[obtain]\ninstructions = \"A fixture.\"\n\
[properties]\nbase_images = 1\nsize_note = \"A fixture.\"\n"
))
.expect("parses")
}
#[cfg(unix)]
#[test]
fn a_corpus_matching_the_digest_its_registry_entry_declares_is_named() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
digested(&root, &["aa", "bb", "cc"]);
let digest = corpus_digest(&root).expect("readable").expect("named");
let entry = registered_corpus(Some(&digest));
let mut request = request(&root, None);
request.registered = Some(&entry);
let (result, _) = score(&sizing_detector(tmp.path()), &request, |_| {}).expect("scored");
assert_eq!(result.declarations.configuration, Configuration::Named);
assert_eq!(result.corpus.name, "Example Tier");
assert_eq!(result.corpus.tier.as_deref(), Some("nano"));
result.validate().expect("valid");
}
#[cfg(unix)]
#[test]
fn the_expensive_corpus_work_happens_once_however_many_detectors_are_asked() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
digested(&root, &["aa", "bb", "cc"]);
let digest = corpus_digest(&root).expect("readable").expect("named");
let entry = registered_corpus(Some(&digest));
let mut lines: Vec<String> = Vec::new();
let prepared = prepare(&root, Some(&entry), None, None, None, |l: &str| {
lines.push(l.to_string())
})
.expect("prepared");
let verified = |lines: &[String]| {
lines
.iter()
.filter(|l| l.contains("are the files their records describe"))
.count()
};
assert_eq!(
verified(&lines),
1,
"the byte check should have run exactly once: {lines:?}"
);
let detector = sizing_detector(tmp.path());
let after = lines.len();
for name in ["first", "second"] {
let records = tmp.path().join(format!("{name}.jsonl"));
score_one(
&detector,
&prepared,
&records,
How {
timeout: Duration::from_secs(5),
jobs: 1,
keep_raw: false,
seed: None,
fixtures: None,
adapter_roots: &[],
},
|l: &str| lines.push(l.to_string()),
)
.unwrap_or_else(|e| panic!("{name} failed: {e}"));
}
assert_eq!(
verified(&lines),
1,
"a detector re-ran the byte check: {:?}",
&lines[after..]
);
let one = score_one(
&detector,
&prepared,
&tmp.path().join("third.jsonl"),
How {
timeout: Duration::from_secs(5),
jobs: 1,
keep_raw: false,
seed: None,
fixtures: None,
adapter_roots: &[],
},
|_| {},
)
.expect("third")
.0;
assert_eq!(one.provenance.started_utc, prepared.started_utc);
assert!(one.provenance.elapsed_seconds >= prepared.preflight_seconds());
assert_eq!(one.declarations.configuration, Configuration::Named);
}
#[test]
fn what_a_result_says_about_the_network_is_true_for_all_three_routes() {
let digest = "5".repeat(64);
let container: Entry = toml::from_str(&format!(
"name = \"c\"\nkind = \"detector\"\nlicence = \"MIT\"\n\
[image]\nreference = \"ghcr.io/x/y@sha256:{digest}\"\n\
[selftest]\nmust_detect = \"a.png\"\nmust_clear = \"b.png\"\n"
))
.expect("parses");
let local: Entry = toml::from_str(
"name = \"l\"\nkind = \"detector\"\nlicence = \"MIT\"\n\
[binary]\ncommand = [\"sh\"]\n\
[selftest]\nmust_detect = \"a.png\"\nmust_clear = \"b.png\"\n",
)
.expect("parses");
let service: Entry = toml::from_str(&format!(
"name = \"s\"\nkind = \"detector\"\nlicence = \"MIT\"\n\
[image]\nreference = \"ghcr.io/x/s@sha256:{digest}\"\n\
[invoke]\nhost = true\nadapter = \"a.py\"\nentrypoint = \"python3\"\n\
argv = [\"{{adapter}}\", \"{{file}}\"]\nparser = \"number\"\n\
[selftest]\nmust_detect = \"a.png\"\nmust_clear = \"b.png\"\n"
))
.expect("parses");
assert!(runs_in_container(&container));
assert!(!runs_in_container(&local));
assert!(
!runs_in_container(&service),
"an entry naming an image AND invoke.host does not run in that \
image, so a result claiming a sandbox and no network is false \
about both halves"
);
}
#[test]
fn the_pin_and_the_sandbox_are_answered_separately_for_every_entry_shape() {
let digest = "5".repeat(64);
let image_block = format!("[image]\nreference = \"ghcr.io/x/y@sha256:{digest}\"\n");
let tail = "[selftest]\nmust_detect = \"a.png\"\nmust_clear = \"b.png\"\n";
let adapter = "[invoke]\nhost = true\nadapter = \"a.py\"\n\
entrypoint = \"python3\"\nargv = [\"{adapter}\", \"{file}\"]\n\
parser = \"number\"\n";
let parse = |body: String| -> Entry { toml::from_str(&body).expect("parses") };
let head = |n: &str| format!("name = \"{n}\"\nkind = \"detector\"\nlicence = \"MIT\"\n");
let container = parse(format!("{}{image_block}{tail}", head("c")));
assert_eq!(pinning(&container).1, PinnedBy::ImageDigest);
assert_eq!(isolation(&container), Isolation::SandboxNoNetwork);
let local = parse(format!(
"{}[binary]\ncommand = [\"definitely-not-installed-xyzzy\"]\n{tail}",
head("l")
));
assert_eq!(pinning(&local).1, PinnedBy::Unpinned);
assert_eq!(isolation(&local), Isolation::Host);
let service = parse(format!(
"{}{image_block}{adapter}endpoint_env = \"X_ENDPOINT\"\n{tail}",
head("s")
));
assert_eq!(
pinning(&service).1,
PinnedBy::Unpinned,
"nothing in the run checked that the instance came from that image"
);
assert_eq!(pinning(&service).0, format!("ghcr.io/x/y@sha256:{digest}"));
assert_eq!(isolation(&service), Isolation::RemoteService);
let adapter_only = parse(format!("{}{image_block}{adapter}{tail}", head("a")));
assert_eq!(pinning(&adapter_only).1, PinnedBy::Unpinned);
assert_eq!(isolation(&adapter_only), Isolation::Host);
}
#[test]
fn the_two_fields_and_the_network_flag_never_contradict_each_other() {
for entry in [
"name = \"c\"\nkind = \"detector\"\nlicence = \"MIT\"\n\
[image]\nreference = \"ghcr.io/x/y@sha256:abc\"\n",
"name = \"s\"\nkind = \"detector\"\nlicence = \"MIT\"\n\
[image]\nreference = \"ghcr.io/x/y@sha256:abc\"\n\
[invoke]\nhost = true\nadapter = \"a.py\"\nentrypoint = \"python3\"\n\
argv = [\"{adapter}\", \"{file}\"]\nparser = \"number\"\n\
endpoint_env = \"X_ENDPOINT\"\n",
"name = \"l\"\nkind = \"detector\"\nlicence = \"MIT\"\n\
[binary]\ncommand = [\"sh\"]\n",
] {
let e: Entry = toml::from_str(&format!(
"{entry}[selftest]\nmust_detect = \"a.png\"\nmust_clear = \"b.png\"\n"
))
.expect("parses");
let sandboxed = isolation(&e) == Isolation::SandboxNoNetwork;
let network_reachable = !runs_in_container(&e);
assert_ne!(
sandboxed, network_reachable,
"{}: a sandbox with no network beside a reachable network is \
the contradiction this change exists to remove",
e.name
);
}
}
#[cfg(unix)]
#[test]
fn a_local_run_reports_the_network_as_reachable_in_the_document() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
corpus(&root, 2, 2);
let (result, _) =
score(&sizing_detector(tmp.path()), &request(&root, None), |_| {}).expect("scored");
assert!(
result.provenance.network_reachable,
"a program on this machine runs with this machine's network"
);
}
#[test]
fn every_way_a_run_can_fail_has_its_own_exit_code() {
use stegobench_core::exit;
let cases = [
(ScoreError::NotADetector { name: "x".into() }, exit::USAGE),
(
ScoreError::NotThatCorpus {
path: "p".into(),
id: "i".into(),
want: "a".into(),
got: "b".into(),
},
exit::VERIFY_MISMATCH,
),
(
ScoreError::SplitLeaks {
path: "p".into(),
count: 1,
examples: "e".into(),
},
exit::PREFLIGHT_REFUSED,
),
(
ScoreError::OneSided {
path: "p".into(),
clean: 1,
stego: 0,
},
exit::PREFLIGHT_REFUSED,
),
(
ScoreError::Records {
path: "p".into(),
source: std::io::Error::other("x"),
},
exit::FAILURE,
),
(
ScoreError::NoAuc {
name: "x".into(),
answered: 4,
why: "w".into(),
},
exit::PLUGIN_FAILED,
),
];
for (err, want) in cases {
assert_eq!(err.exit_code(), want, "{err}");
}
}
#[test]
fn a_run_with_no_computable_auc_refuses_rather_than_publishing_a_half() {
let err = ScoreError::NoAuc {
name: "zsteg".into(),
answered: 12,
why: why_no_auc(&[1.0, f64::NAN], &[false, true], ""),
};
let text = err.to_string();
assert_eq!(err.exit_code(), stegobench_core::exit::PLUGIN_FAILED);
assert!(text.contains("not numbers"), "{text}");
assert!(text.contains("0.5"), "{text}");
assert!(text.contains("nothing is written"), "{text}");
}
#[test]
fn every_reason_an_auc_cannot_be_computed_is_named_separately() {
let mismatch = why_no_auc(&[1.0, 2.0], &[true], "");
assert!(mismatch.contains("2 score(s)") && mismatch.contains("1 label(s)"));
assert!(
mismatch.contains("bug in stegobench"),
"a length mismatch is not the user's doing: {mismatch}"
);
let nan = why_no_auc(&[f64::NAN, 1.0], &[true, false], "");
assert!(
nan.contains("1 of the 2 answer(s) are not numbers"),
"{nan}"
);
assert!(
nan.contains("stegobench doctor"),
"a NaN from a detector needs a next step: {nan}"
);
let empty = why_no_auc(&[], &[], "");
assert!(
empty.contains("none of them came back with a number"),
"an empty answer set has to say the tool produced nothing: {empty}"
);
assert!(
empty.contains("stegobench doctor"),
"and where to look next: {empty}"
);
let one_sided = why_no_auc(&[1.0, 2.0], &[true, true], "");
assert!(one_sided.contains("0 clean and 2 stego"), "{one_sided}");
assert!(
one_sided.contains("this tool produced a usable answer"),
"the tool, not the corpus, is what is one-sided here: {one_sided}"
);
let residual = why_no_auc(&[1.0, 2.0], &[true, false], "");
assert!(residual.contains("bug in stegobench"), "{residual}");
let at_budget = why_no_auc(&[1.0, 2.0], &[true, false], " at a budget of 0.01");
assert!(at_budget.contains("at a budget of 0.01"), "{at_budget}");
}
#[test]
fn the_refusals_are_the_metrics_commands_own_words() {
for (scores, labels) in [
(vec![1.0, 2.0], vec![true]),
(vec![f64::NAN, 1.0], vec![true, false]),
(vec![1.0, 2.0], vec![true, true]),
(vec![], vec![]),
] {
let shared = crate::metrics::why_unrankable(&scores, &labels, "").to_string();
let here = why_no_auc(&scores, &labels, "");
assert!(
here.starts_with(&shared),
"{here}\ndoes not open with\n{shared}"
);
}
}
#[test]
fn the_conditions_this_diagnoses_are_the_ones_the_metric_actually_refuses() {
assert_eq!(stegobench_metrics::roc_auc(&[1.0, 2.0], &[true]), None);
assert_eq!(
stegobench_metrics::roc_auc(&[f64::NAN, 1.0], &[true, false]),
None
);
assert_eq!(
stegobench_metrics::roc_auc(&[1.0, 2.0], &[true, true]),
None
);
assert!(stegobench_metrics::roc_auc(&[1.0, 2.0], &[false, true]).is_some());
}
#[cfg(unix)]
#[test]
fn swapping_the_images_under_a_matching_manifest_is_caught() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
digested(&root, &["aa", "bb", "cc"]);
let digest = corpus_digest(&root).expect("readable").expect("named");
let entry = registered_corpus(Some(&digest));
std::fs::write(root.join("i1.png"), png(64, 64, 8, 2, 99)).unwrap();
assert_eq!(
corpus_digest(&root).expect("readable").as_deref(),
Some(digest.as_str()),
"the corpus digest noticed, which would make this test prove nothing"
);
let mut request = request(&root, None);
request.registered = Some(&entry);
let err = score(&sizing_detector(tmp.path()), &request, |_| {}).expect_err("refused");
assert!(
err.to_string().contains("is not the image its own record"),
"{err}"
);
assert_eq!(err.exit_code(), stegobench_core::exit::VERIFY_MISMATCH);
}
#[cfg(unix)]
#[test]
fn a_custom_run_does_not_pay_for_the_byte_check() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
digested(&root, &["aa", "bb", "cc"]);
std::fs::write(root.join("i1.png"), png(64, 64, 8, 2, 99)).unwrap();
let mut said = Vec::new();
let (result, _) = score(&sizing_detector(tmp.path()), &request(&root, None), |l| {
said.push(l.to_string())
})
.expect("scored");
assert_eq!(result.declarations.configuration, Configuration::Custom);
assert!(
!said.iter().any(|l| l.contains("their records describe")),
"a custom run paid for the check: {said:?}"
);
}
#[cfg(unix)]
#[test]
fn a_directory_that_is_not_the_corpus_it_claims_is_refused() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
digested(&root, &["aa", "bb", "cc"]);
let entry = registered_corpus(Some(&format!("sha256:{}", "0".repeat(64))));
let mut request = request(&root, None);
request.registered = Some(&entry);
let err = score(&sizing_detector(tmp.path()), &request, |_| {}).expect_err("refused");
assert!(err.to_string().contains("is not example"), "{err}");
}
#[cfg(unix)]
#[test]
fn naming_a_corpus_whose_entry_declares_no_digest_stays_custom() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
digested(&root, &["aa", "bb", "cc"]);
let entry = registered_corpus(None);
let mut request = request(&root, None);
request.registered = Some(&entry);
let mut said = Vec::new();
let (result, _) = score(&sizing_detector(tmp.path()), &request, |l| {
said.push(l.to_string())
})
.expect("scored");
assert_eq!(result.declarations.configuration, Configuration::Custom);
assert!(
said.iter()
.any(|l| l.contains("declares no records digest")),
"it downgraded silently: {said:?}"
);
assert_eq!(result.corpus.name, "Example Tier");
}
#[cfg(unix)]
#[test]
fn a_partial_run_over_a_matching_corpus_is_still_custom() {
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
digested(&root, &["aa", "bb", "cc", "dd"]);
let digest = corpus_digest(&root).expect("readable").expect("named");
let entry = registered_corpus(Some(&digest));
let mut request = request(&root, Some(3));
request.registered = Some(&entry);
let mut said = Vec::new();
let (result, _) = score(&sizing_detector(tmp.path()), &request, |l| {
said.push(l.to_string())
})
.expect("scored");
assert_eq!(result.declarations.configuration, Configuration::Custom);
assert!(
said.iter().any(|l| l.contains("prefix of a tier")),
"{said:?}"
);
}
#[test]
fn an_embedder_is_refused_rather_than_scored() {
let entry: Entry = toml::from_str(
"name = \"x\"\nkind = \"embedder\"\nlicence = \"MIT\"\n\
[image]\nreference = \"a@sha256:b\"\nsize_mb = 1\nbundled = true\n\
[selftest]\nmust_detect = \"fixtures/a.png\"\nmust_clear = \"fixtures/b.png\"\n",
)
.expect("parses");
let tmp = tempfile::tempdir().expect("tmp");
let root = tmp.path().join("corpus");
corpus(&root, 1, 1);
let err = score(&entry, &request(&root, None), |_| {}).expect_err("refused");
assert!(err.to_string().contains("cannot be asked"), "{err}");
}
#[test]
fn every_score_error_has_its_own_stable_word() {
let every: Vec<ScoreError> = vec![
ScoreError::Corpus(stegobench_core::samples::SampleError::TooManySamples),
ScoreError::Run(runner::RunError::BadRecord {
path: "p".into(),
reason: "r".into(),
}),
ScoreError::Records {
path: "p".into(),
source: std::io::Error::other("x"),
},
ScoreError::NotADetector { name: "n".into() },
ScoreError::SplitLeaks {
path: "p".into(),
count: 1,
examples: "e".into(),
},
ScoreError::NotThatCorpus {
path: "p".into(),
id: "i".into(),
want: "w".into(),
got: "g".into(),
},
ScoreError::UnknownSplit {
asked: "sideways".into(),
},
ScoreError::NoSplitLabels { path: "p".into() },
ScoreError::OneSided {
path: "p".into(),
clean: 1,
stego: 0,
},
ScoreError::NoAuc {
name: "n".into(),
answered: 1,
why: "w".into(),
},
ScoreError::ImageChanged {
id: "i".into(),
want: "w".into(),
got: "g".into(),
},
];
let mut seen = std::collections::BTreeSet::new();
for e in &every {
let word = e.reason();
assert!(!word.is_empty(), "a variant answered with nothing");
assert!(
word.chars().all(|c| c.is_ascii_lowercase() || c == '-'),
"{word:?} is not the kebab-case vocabulary the others use"
);
assert!(seen.insert(word), "{word:?} is claimed by two variants");
}
assert_eq!(seen.len(), every.len());
}
}