use std::path::{Path, PathBuf};
use std::time::Duration;
use stegobench_core::registry::{Entry, Kind};
use stegobench_plugin::{runner, Record, WorkItem};
#[derive(Debug, Clone, PartialEq)]
pub enum Answer {
Score(f64),
Verdict(bool),
Failed(String),
}
fn format_of(path: &Path) -> Option<String> {
use std::io::Read;
let mut head = [0u8; 12];
let read = std::fs::File::open(path)
.and_then(|mut f| f.read(&mut head))
.ok()?;
let head = &head[..read];
let starts = |magic: &[u8]| head.starts_with(magic);
if starts(b"\x89PNG\r\n\x1a\n") {
return Some("png".into());
}
if starts(&[0xFF, 0xD8, 0xFF]) {
return Some("jpeg".into());
}
if starts(b"BM") {
return Some("bmp".into());
}
if starts(b"GIF8") {
return Some("gif".into());
}
if starts(b"II*\x00") || starts(b"MM\x00*") {
return Some("tiff".into());
}
if head.len() >= 12 && starts(b"RIFF") && &head[8..12] == b"WEBP" {
return Some("webp".into());
}
if starts(b"P5") || starts(b"P6") {
return Some("pnm".into());
}
None
}
fn reads_format(entry: &stegobench_core::registry::Entry, format: Option<&String>) -> bool {
if entry.accepts.formats.is_empty() {
return true;
}
let Some(format) = format else {
return true;
};
entry.accepts.formats.iter().any(|declared| {
let declared = declared.to_ascii_lowercase();
declared == *format || (declared == "jpg" && format == "jpeg")
})
}
impl Answer {
fn cell(&self) -> String {
match self {
Answer::Score(s) => format!("{s:.4}"),
Answer::Verdict(true) => "stego".to_string(),
Answer::Verdict(false) => "clean".to_string(),
Answer::Failed(_) => "failed".to_string(),
}
}
}
#[derive(Debug, Clone)]
pub struct Column {
pub detector: String,
pub unavailable: Option<String>,
pub answers: Vec<Answer>,
}
#[derive(Debug, Clone)]
pub struct Examination {
pub images: Vec<PathBuf>,
pub columns: Vec<Column>,
pub undeclared: Vec<(String, String, usize)>,
}
#[derive(Debug, thiserror::Error)]
pub enum ExamineError {
#[error(
"no detector called {name} is registered. `stegobench list detectors` \
shows the ones that are"
)]
NotRegistered { name: String },
#[error(
"{name} is registered as an embedder, which hides a payload in an \
image rather than answering questions about one. `stegobench list \
detectors` shows what can answer"
)]
NotADetector { name: String },
#[error("{} is not a file or a directory here", path.display())]
NotAFile { path: PathBuf },
#[error("could not read the directory {}: {source}", path.display())]
UnreadableDirectory {
path: PathBuf,
#[source]
source: std::io::Error,
},
#[error(
"{} holds no image files, so there is nothing to ask about. Name the \
images themselves if they are somewhere else, or check the extension: \
a file this does not recognise as an image is skipped rather than \
guessed at",
path.display()
)]
NoImagesInDirectory { path: PathBuf },
#[error(
"{} holds more than {most} images. That is more than this will start a \
detector for on one command, because a run of that size belongs in \
`score` against a labelled corpus where it can be resumed. Name fewer \
images, or a subdirectory",
path.display()
)]
TooManyImages { path: PathBuf, most: usize },
#[error("the same detector was asked for twice: {name}")]
AskedTwice { name: String },
#[error(
"name at least one detector with --detector, because nothing here \
guesses which tool you meant. `stegobench list detectors` names \
every one registered"
)]
NoDetectorNamed,
#[error("could not make a scratch directory for the run: {source}")]
NoScratch {
#[source]
source: std::io::Error,
},
#[error(transparent)]
Run(#[from] runner::RunError),
}
impl ExamineError {
pub fn exit_code(&self) -> i32 {
use stegobench_core::exit;
match self {
ExamineError::NotRegistered { .. }
| ExamineError::NotADetector { .. }
| ExamineError::AskedTwice { .. }
| ExamineError::NoDetectorNamed => exit::USAGE,
ExamineError::NotAFile { .. }
| ExamineError::NoImagesInDirectory { .. }
| ExamineError::TooManyImages { .. } => exit::PREFLIGHT_REFUSED,
ExamineError::UnreadableDirectory { .. } => exit::FAILURE,
ExamineError::NoScratch { .. } => exit::FAILURE,
ExamineError::Run(_) => exit::PLUGIN_FAILED,
}
}
pub fn reason(&self) -> &'static str {
match self {
ExamineError::NotRegistered { .. } => "not-registered",
ExamineError::NotADetector { .. } => "not-a-detector",
ExamineError::AskedTwice { .. } => "asked-twice",
ExamineError::NoDetectorNamed => "no-detector-named",
ExamineError::NotAFile { .. } => "not-a-file",
ExamineError::UnreadableDirectory { .. } => "unreadable-directory",
ExamineError::NoImagesInDirectory { .. } => "no-images-there",
ExamineError::TooManyImages { .. } => "too-many-images",
ExamineError::NoScratch { .. } => "no-scratch",
ExamineError::Run(_) => "run-failed",
}
}
}
pub struct Request<'a> {
pub detectors: &'a [String],
pub images: &'a [PathBuf],
pub timeout: Duration,
pub jobs: usize,
pub adapter_roots: &'a [PathBuf],
pub raw: Option<&'a Path>,
}
struct Collect(Vec<Record>);
impl runner::Sink for Collect {
fn write(&mut self, record: &Record) -> std::io::Result<()> {
self.0.push(record.clone());
Ok(())
}
}
const UNAVAILABLE: &str = "unavailable";
const MOST_IMAGES_IN_A_DIRECTORY: usize = 10_000;
fn images_under(path: &Path) -> Result<Vec<PathBuf>, ExamineError> {
if path.is_file() {
return Ok(vec![path.to_path_buf()]);
}
if !path.is_dir() {
return Err(ExamineError::NotAFile {
path: path.to_path_buf(),
});
}
let entries = std::fs::read_dir(path).map_err(|source| ExamineError::UnreadableDirectory {
path: path.to_path_buf(),
source,
})?;
let mut found = Vec::new();
for entry in entries {
let entry = entry.map_err(|source| ExamineError::UnreadableDirectory {
path: path.to_path_buf(),
source,
})?;
let candidate = entry.path();
if !candidate.is_file() {
continue;
}
let looks_right = candidate
.extension()
.and_then(|e| e.to_str())
.map(|e| e.to_ascii_lowercase())
.is_some_and(|e| crate::LOOKS_LIKE_AN_IMAGE.contains(&e.as_str()));
if looks_right {
found.push(candidate);
}
if found.len() > MOST_IMAGES_IN_A_DIRECTORY {
return Err(ExamineError::TooManyImages {
path: path.to_path_buf(),
most: MOST_IMAGES_IN_A_DIRECTORY,
});
}
}
if found.is_empty() {
return Err(ExamineError::NoImagesInDirectory {
path: path.to_path_buf(),
});
}
found.sort();
Ok(found)
}
fn preflight<'r>(
registry: &'r stegobench_core::registry::Registry,
request: &Request<'_>,
) -> Result<Vec<&'r Entry>, ExamineError> {
if request.detectors.is_empty() {
return Err(ExamineError::NoDetectorNamed);
}
let mut entries = Vec::new();
let mut seen: Vec<&str> = Vec::new();
for name in request.detectors {
if seen.contains(&name.as_str()) {
return Err(ExamineError::AskedTwice { name: name.clone() });
}
seen.push(name);
let Some(entry) = registry.entries.get(name) else {
return Err(ExamineError::NotRegistered { name: name.clone() });
};
if entry.kind != Kind::Detector {
return Err(ExamineError::NotADetector { name: name.clone() });
}
entries.push(entry);
}
Ok(entries)
}
fn expand(named: &[PathBuf]) -> Result<Vec<PathBuf>, ExamineError> {
let mut images: Vec<PathBuf> = Vec::new();
for path in named {
for image in images_under(path)? {
let canonical = image.canonicalize().unwrap_or_else(|_| image.clone());
if !images
.iter()
.any(|seen| seen.canonicalize().unwrap_or_else(|_| seen.clone()) == canonical)
{
images.push(image);
}
}
}
Ok(images)
}
pub fn run<P>(
registry: &stegobench_core::registry::Registry,
request: Request<'_>,
mut progress: P,
) -> Result<Examination, ExamineError>
where
P: FnMut(&str),
{
let entries = preflight(registry, &request)?;
let images = expand(request.images)?;
let scratch = tempfile::tempdir().map_err(|source| ExamineError::NoScratch { source })?;
let mut columns = Vec::with_capacity(entries.len());
let formats: Vec<Option<String>> = images.iter().map(|p| format_of(p)).collect();
let mut undeclared: Vec<(String, String, usize)> = Vec::new();
for entry in entries {
if let Some(why) = crate::unavailable_reason(entry, request.adapter_roots) {
let why = crate::shorten_digests(&why);
progress(&format!("{} {why}, so it answered nothing", entry.name));
columns.push(Column {
detector: entry.name.clone(),
unavailable: Some(why),
answers: Vec::new(),
});
continue;
}
let records = scratch.path().join(format!("{}.records.jsonl", entry.name));
let outside: Vec<&PathBuf> = images
.iter()
.zip(&formats)
.filter(|(_, f)| !reads_format(entry, f.as_ref()))
.map(|(path, _)| path)
.collect();
let items: Vec<WorkItem> = images
.iter()
.map(|path| WorkItem {
id: path.display().to_string(),
path: path.clone(),
})
.collect();
if outside.is_empty() {
progress(&format!("{} over {} image(s)", entry.name, images.len()));
} else {
progress(&format!(
"{} over {} image(s); it declares {} and {} of them are not",
entry.name,
images.len(),
entry.accepts.formats.join(", "),
outside.len()
));
}
undeclared.push((
entry.name.clone(),
entry.accepts.formats.join(", "),
outside.len(),
));
let items_asked = items.len();
let mut sink = Collect(Vec::with_capacity(items_asked));
let tally = runner::score(
runner::Run {
entry,
already: &records,
timeout: request.timeout,
jobs: request.jobs,
adapter_roots: request.adapter_roots,
raw: runner::RawPolicy {
path: request.raw,
every: request.raw.is_some(),
},
},
items,
&mut sink,
|t| progress(&heartbeat(&entry.name, t.seen(), items_asked, t.errored)),
)?;
if let Some(problem) = &tally.raw_problem {
progress(problem);
}
columns.push(Column {
detector: entry.name.clone(),
unavailable: None,
answers: sink.0.iter().map(answer_of).collect(),
});
}
Ok(Examination {
images,
columns,
undeclared: undeclared.into_iter().filter(|(_, _, n)| *n > 0).collect(),
})
}
fn heartbeat(name: &str, answered: u64, total: usize, errored: u64) -> String {
format!("still working: {name}, {answered} of {total} answered, {errored} errored")
}
fn digest_of(path: &Path) -> Option<String> {
use sha2::{Digest, Sha256};
let mut file = std::fs::File::open(path).ok()?;
let mut hasher = Sha256::new();
std::io::copy(&mut file, &mut hasher).ok()?;
Some(format!("{:x}", hasher.finalize()))
}
fn answer_of(record: &Record) -> Answer {
match (record.score, record.verdict, &record.error) {
(Some(s), _, _) => Answer::Score(s),
(None, Some(v), _) => Answer::Verdict(v),
(None, None, Some(e)) => Answer::Failed(e.clone()),
(None, None, None) => Answer::Failed("no answer and no reason".to_string()),
}
}
impl Examination {
pub fn complete(&self) -> bool {
self.columns.iter().all(|c| {
c.unavailable.is_none()
&& c.answers.len() == self.images.len()
&& !c.answers.iter().any(|a| matches!(a, Answer::Failed(_)))
})
}
pub fn exit_code(&self) -> i32 {
use stegobench_core::exit;
if self.columns.iter().any(|c| {
c.answers.iter().any(|a| matches!(a, Answer::Failed(_)))
|| (c.unavailable.is_none() && c.answers.len() != self.images.len())
}) {
return exit::PLUGIN_FAILED;
}
let unavailable = self
.columns
.iter()
.filter(|c| c.unavailable.is_some())
.count();
if unavailable > 0 && unavailable == self.columns.len() {
return exit::ENVIRONMENT_UNFIT;
}
if unavailable > 0 {
return exit::PREFLIGHT_REFUSED;
}
exit::OK
}
fn row_labels(images: &[PathBuf]) -> Vec<String> {
let base = |p: &PathBuf| {
p.file_name()
.map(|n| n.to_string_lossy().to_string())
.unwrap_or_else(|| p.display().to_string())
};
let mut labels: Vec<String> = images.iter().map(base).collect();
let colliding = |labels: &[String]| -> Vec<bool> {
labels
.iter()
.map(|l| labels.iter().filter(|o| *o == l).count() > 1)
.collect()
};
for (i, _) in colliding(&labels)
.into_iter()
.enumerate()
.filter(|(_, c)| *c)
.collect::<Vec<_>>()
{
let path = &images[i];
labels[i] = match path.parent().and_then(|p| p.file_name()) {
Some(dir) => format!("{}/{}", dir.to_string_lossy(), base(path)),
None => path.display().to_string(),
};
}
for (i, _) in colliding(&labels)
.into_iter()
.enumerate()
.filter(|(_, c)| *c)
.collect::<Vec<_>>()
{
labels[i] = images[i].display().to_string();
}
labels
}
pub fn table(&self) -> String {
let mut name_width = "image".len();
let labels = Self::row_labels(&self.images);
for label in &labels {
name_width = name_width.max(label.chars().count());
}
let mut widths = Vec::with_capacity(self.columns.len());
for column in &self.columns {
let mut w = column.detector.chars().count();
if column.unavailable.is_some() {
w = w.max(UNAVAILABLE.len());
}
for answer in &column.answers {
w = w.max(answer.cell().chars().count());
}
widths.push(w);
}
let mut out = String::new();
let mut line = format!("{:<name_width$}", "image");
for (column, width) in self.columns.iter().zip(&widths) {
line.push_str(&format!(" {:<width$}", column.detector, width = width));
}
out.push_str(line.trim_end());
out.push('\n');
for (row, label) in labels.iter().enumerate() {
let mut line = format!("{label:<name_width$}");
for (column, width) in self.columns.iter().zip(&widths) {
let cell = match (&column.unavailable, column.answers.get(row)) {
(Some(_), _) => UNAVAILABLE.to_string(),
(None, Some(answer)) => answer.cell(),
(None, None) => "failed".to_string(),
};
line.push_str(&format!(" {cell:<width$}"));
}
out.push_str(line.trim_end());
out.push('\n');
}
out
}
pub fn answered_anything(&self) -> bool {
self.columns
.iter()
.any(|c| c.answers.iter().any(|a| !matches!(a, Answer::Failed(_))))
}
fn one_answer_for_everything(&self) -> Vec<&str> {
if self.images.len() < 2 {
return Vec::new();
}
self.columns
.iter()
.filter(|c| c.unavailable.is_none())
.filter(|c| {
let mut answered = c
.answers
.iter()
.filter(|a| !matches!(a, Answer::Failed(_)))
.map(|a| a.cell());
match answered.next() {
Some(first) => {
c.answers.len() == self.images.len()
&& answered.clone().count() + 1 == self.images.len()
&& answered.all(|a| a == first)
}
None => false,
}
})
.map(|c| c.detector.as_str())
.collect()
}
pub fn footnotes(&self) -> Vec<String> {
let mut notes = Vec::new();
if self.answered_anything() {
notes.push(
"NOT A MEASUREMENT. No image here was labelled, so nothing above is an \
accuracy, and a figure from it is not quotable. `stegobench help scope` \
says why, and `stegobench score` is what produces a number you can defend."
.to_string(),
);
} else {
notes.push(
"NOTHING RAN, so the table above says nothing about these images. Not \
that they are clean: that nobody looked."
.to_string(),
);
}
if self
.columns
.iter()
.filter(|c| c.unavailable.is_none())
.count()
> 1
{
notes.push(
"Each column is on its own scale. A higher number means more suspicious \
to that one detector, and two detectors' numbers do not compare with \
each other."
.to_string(),
);
}
for column in &self.columns {
if let Some(why) = &column.unavailable {
notes.push(format!("{}: {why}", column.detector));
}
}
for column in &self.columns {
let failures = column
.answers
.iter()
.filter(|a| matches!(a, Answer::Failed(_)))
.count();
if failures > 0 {
notes.push(format!(
"{}: answered nothing about {failures} of {} image(s). \
`--raw <FILE>` keeps what it printed.",
column.detector,
self.images.len()
));
}
}
for (detector, declares, count) in &self.undeclared {
notes.push(format!(
"{detector} reads {declares}, and {count} of these {} image(s) \
are not that. Its answers about those are outside what its \
own entry claims it supports, so treat them as a pointer to \
look rather than as a result.",
self.images.len()
));
}
let flat = self.one_answer_for_everything();
if !flat.is_empty() {
notes.push(format!(
"{} gave one answer to all {} image(s). That is what finding \
nothing looks like, and it is also what a misread output \
looks like; the table cannot tell you which. `--raw <FILE>` \
keeps what the tool printed, which can.",
flat.join(", "),
self.images.len()
));
}
notes
}
pub fn to_json(&self) -> serde_json::Value {
let images: Vec<serde_json::Value> = self
.images
.iter()
.map(|p| {
serde_json::json!({
"path": p.display().to_string(),
"sha256": digest_of(p),
})
})
.collect();
let columns: Vec<serde_json::Value> = self
.columns
.iter()
.map(|column| {
let answers: Vec<serde_json::Value> = column
.answers
.iter()
.map(|a| match a {
Answer::Score(s) => serde_json::json!({ "score": s }),
Answer::Verdict(v) => serde_json::json!({ "verdict": v }),
Answer::Failed(why) => serde_json::json!({ "error": why }),
})
.collect();
serde_json::json!({
"detector": column.detector,
"unavailable": column.unavailable,
"answers": answers,
})
})
.collect();
serde_json::json!({
"kind": "examination",
"measurement": false,
"tool": env!("CARGO_PKG_NAME"),
"tool_version": env!("CARGO_PKG_VERSION"),
"images": images,
"columns": columns,
"undeclared_formats": self
.undeclared
.iter()
.map(|(detector, declares, count)| {
serde_json::json!({
"detector": detector,
"declares": declares,
"images_outside": count,
})
})
.collect::<Vec<_>>(),
})
}
}
#[cfg(test)]
mod tests {
use super::*;
fn examination(columns: Vec<Column>, images: usize) -> Examination {
Examination {
images: (0..images)
.map(|i| PathBuf::from(format!("img{i}.png")))
.collect(),
columns,
undeclared: Vec::new(),
}
}
fn answered(detector: &str, answers: Vec<Answer>) -> Column {
Column {
detector: detector.to_string(),
unavailable: None,
answers,
}
}
#[test]
fn the_heartbeat_names_the_detector_and_both_counts() {
let line = heartbeat("zsteg", 30, 40, 2);
assert!(line.contains("zsteg"), "{line}");
assert!(line.contains("30 of 40"), "{line}");
assert!(line.contains("2 errored"), "{line}");
}
#[test]
fn a_score_and_a_verdict_render_differently() {
assert_eq!(Answer::Score(0.5).cell(), "0.5000");
assert_eq!(Answer::Verdict(true).cell(), "stego");
assert_eq!(Answer::Verdict(false).cell(), "clean");
assert_eq!(Answer::Failed("why".into()).cell(), "failed");
}
#[test]
fn a_record_becomes_the_answer_it_carries() {
let base = Record {
id: "a".into(),
score: None,
verdict: None,
error: None,
elapsed_ms: None,
};
assert_eq!(
answer_of(&Record {
score: Some(0.25),
..base.clone()
}),
Answer::Score(0.25)
);
assert_eq!(
answer_of(&Record {
verdict: Some(true),
..base.clone()
}),
Answer::Verdict(true)
);
assert_eq!(
answer_of(&Record {
error: Some("boom".into()),
..base.clone()
}),
Answer::Failed("boom".into())
);
assert_eq!(
answer_of(&base),
Answer::Failed("no answer and no reason".into())
);
}
#[test]
fn a_score_wins_over_a_verdict_in_the_same_record() {
let both = Record {
id: "a".into(),
score: Some(0.9),
verdict: Some(false),
error: None,
elapsed_ms: None,
};
assert_eq!(answer_of(&both), Answer::Score(0.9));
}
#[test]
fn every_detector_answering_exits_zero() {
let e = examination(
vec![
answered("alfa", vec![Answer::Score(0.1), Answer::Score(0.2)]),
answered("bravo", vec![Answer::Verdict(true), Answer::Verdict(false)]),
],
2,
);
assert!(e.complete());
assert_eq!(e.exit_code(), stegobench_core::exit::OK);
}
#[test]
fn a_detector_that_is_not_here_is_a_refusal_rather_than_a_failure() {
let e = examination(
vec![
answered("alfa", vec![Answer::Score(0.1)]),
Column {
detector: "bravo".to_string(),
unavailable: Some("no container runtime".to_string()),
answers: Vec::new(),
},
],
1,
);
assert!(!e.complete());
assert_eq!(e.exit_code(), stegobench_core::exit::PREFLIGHT_REFUSED);
assert!(e
.footnotes()
.iter()
.any(|n| n.contains("no container runtime")));
}
#[test]
fn a_detector_that_answered_nothing_is_a_failure() {
let e = examination(
vec![answered(
"alfa",
vec![Answer::Score(0.1), Answer::Failed("timed out".into())],
)],
2,
);
assert!(!e.complete());
assert_eq!(e.exit_code(), stegobench_core::exit::PLUGIN_FAILED);
}
#[test]
fn a_short_column_is_a_failure_even_with_no_error_recorded() {
let e = examination(vec![answered("alfa", vec![Answer::Score(0.1)])], 3);
assert!(!e.complete());
assert_eq!(e.exit_code(), stegobench_core::exit::PLUGIN_FAILED);
}
#[test]
fn nothing_being_installed_is_told_apart_from_something_being_installed() {
let missing = |name: &str| Column {
detector: name.to_string(),
unavailable: Some("not installed".to_string()),
answers: Vec::new(),
};
let some = examination(
vec![answered("alfa", vec![Answer::Score(0.1)]), missing("bravo")],
1,
);
assert_eq!(some.exit_code(), stegobench_core::exit::PREFLIGHT_REFUSED);
let none = examination(vec![missing("alfa"), missing("bravo")], 1);
assert_eq!(none.exit_code(), stegobench_core::exit::ENVIRONMENT_UNFIT);
assert_ne!(some.exit_code(), none.exit_code());
}
#[test]
fn a_failure_outranks_an_unavailable_detector_in_the_exit_code() {
let e = examination(
vec![
answered("alfa", vec![Answer::Failed("boom".into())]),
Column {
detector: "bravo".to_string(),
unavailable: Some("not installed".to_string()),
answers: Vec::new(),
},
],
1,
);
assert_eq!(e.exit_code(), stegobench_core::exit::PLUGIN_FAILED);
}
#[test]
fn two_images_sharing_a_basename_get_rows_a_reader_can_tell_apart() {
let e = Examination {
images: vec![
PathBuf::from("corpus/covers/000000.png"),
PathBuf::from("corpus/lsb-0400/000000.png"),
PathBuf::from("corpus/covers/000001.png"),
],
columns: vec![answered(
"alfa",
vec![Answer::Score(0.1), Answer::Score(0.2), Answer::Score(0.3)],
)],
undeclared: Vec::new(),
};
let table = e.table();
assert!(
table.contains("covers/000000.png") && table.contains("lsb-0400/000000.png"),
"BOTH sides of the collision have to be qualified, or the reader \
still cannot tell which row is which:\n{table}"
);
assert!(
table
.lines()
.any(|l| l.starts_with("000001.png") || l.starts_with("000001.png ")),
"a name that collided with nothing was lengthened anyway:\n{table}"
);
}
#[test]
fn one_answer_for_every_image_is_called_out_under_the_table() {
let flat = examination(
vec![answered(
"zsteg",
vec![Answer::Verdict(false), Answer::Verdict(false)],
)],
2,
);
let note = flat
.footnotes()
.into_iter()
.find(|n| n.contains("one answer to all"))
.expect("a column with one answer in every row is called out");
assert!(note.contains("zsteg"), "{note}");
assert!(
note.contains("--raw"),
"the note has to say what to do next"
);
let varied = examination(
vec![answered(
"zsteg",
vec![Answer::Verdict(false), Answer::Verdict(true)],
)],
2,
);
assert!(
!varied
.footnotes()
.iter()
.any(|n| n.contains("one answer to all")),
"a column with two different answers was called out as flat"
);
let alone = examination(vec![answered("zsteg", vec![Answer::Verdict(false)])], 1);
assert!(
!alone
.footnotes()
.iter()
.any(|n| n.contains("one answer to all")),
"a single image examination carried the flat-column note"
);
}
#[test]
fn an_answer_outside_a_detector_s_declared_formats_is_caveated_not_hidden() {
let e = Examination {
images: vec![
PathBuf::from("EXH-003-scan.png"),
PathBuf::from("EXH-005-attachment.jpg"),
],
columns: vec![answered(
"zsteg",
vec![Answer::Verdict(true), Answer::Verdict(true)],
)],
undeclared: vec![("zsteg".to_string(), "png, bmp".to_string(), 1)],
};
let table = e.table();
assert!(
table.lines().filter(|l| l.contains("stego")).count() >= 2,
"the detection was dropped instead of caveated:\n{table}"
);
let note = e
.footnotes()
.into_iter()
.find(|n| n.contains("png, bmp"))
.expect("the caveat names what the detector actually declares");
assert!(note.contains("zsteg"), "{note}");
assert!(
note.contains("1 of these 2"),
"it has to say how many were outside: {note}"
);
let clean = Examination {
images: vec![PathBuf::from("a.png")],
columns: vec![answered("zsteg", vec![Answer::Verdict(false)])],
undeclared: Vec::new(),
};
assert!(
!clean.footnotes().iter().any(|n| n.contains("outside what")),
"a run entirely inside the declarations carried the caveat anyway"
);
}
#[test]
fn the_json_record_names_every_image_by_digest_and_says_what_produced_it() {
let dir = tempfile::tempdir().expect("tmp");
let image = dir.path().join("EXH-001.png");
std::fs::write(&image, b"\x89PNG\r\n\x1a\nexhibit bytes").expect("written");
let e = Examination {
images: vec![image.clone()],
columns: vec![answered("zsteg", vec![Answer::Verdict(false)])],
undeclared: Vec::new(),
};
let json = e.to_json();
let first = &json["images"][0];
assert_eq!(
first["path"].as_str().expect("a path"),
image.display().to_string()
);
let expected = {
use sha2::{Digest, Sha256};
let mut h = Sha256::new();
h.update(std::fs::read(&image).expect("read"));
format!("{:x}", h.finalize())
};
assert_eq!(
first["sha256"].as_str().expect("a digest"),
expected,
"the record has to name the bytes, not the filename"
);
assert_eq!(json["tool_version"], env!("CARGO_PKG_VERSION"));
let gone = Examination {
images: vec![dir.path().join("never-existed.png")],
columns: vec![answered("zsteg", vec![Answer::Verdict(false)])],
undeclared: Vec::new(),
};
assert!(
gone.to_json()["images"][0]["sha256"].is_null(),
"an unreadable exhibit must be null, not missing"
);
}
#[test]
fn the_table_aligns_on_the_longest_name_and_cell() {
let e = Examination {
images: vec![
PathBuf::from("/some/where/a-long-filename.png"),
PathBuf::from("/some/where/b.png"),
],
columns: vec![answered(
"aletheia-spa",
vec![Answer::Score(0.1234), Answer::Verdict(true)],
)],
undeclared: Vec::new(),
};
let table = e.table();
let lines: Vec<&str> = table.lines().collect();
assert_eq!(lines.len(), 3);
assert!(lines[1].starts_with("a-long-filename.png"));
assert!(lines[2].starts_with("b.png"));
let at = |line: &str| line.find("0.1234").or_else(|| line.find("stego"));
assert_eq!(at(lines[1]), at(lines[2]));
for line in &lines {
assert_eq!(*line, line.trim_end(), "trailing space on {line:?}");
}
}
#[test]
fn the_not_a_measurement_note_is_always_there() {
let e = examination(vec![answered("alfa", vec![Answer::Score(0.1)])], 1);
let notes = e.footnotes();
assert!(notes[0].contains("NOT A MEASUREMENT"));
assert!(notes[0].contains("not quotable"));
assert!(!notes.iter().any(|n| n.contains("own scale")));
}
#[test]
fn a_table_where_nothing_ran_says_so_instead_of_quoting_the_caveat() {
let e = examination(
vec![Column {
detector: "alfa".to_string(),
unavailable: Some("not pulled".to_string()),
answers: Vec::new(),
}],
2,
);
assert!(!e.answered_anything());
let notes = e.footnotes();
assert!(notes[0].contains("NOTHING RAN"), "{notes:?}");
assert!(notes[0].contains("Not that they are clean"), "{notes:?}");
assert!(
!notes.iter().any(|n| n.contains("NOT A MEASUREMENT")),
"a caveat about a number nobody produced: {notes:?}"
);
}
#[test]
fn a_table_with_one_real_answer_still_carries_the_caveat() {
let e = examination(
vec![
answered("alfa", vec![Answer::Score(0.1), Answer::Score(0.2)]),
Column {
detector: "bravo".to_string(),
unavailable: Some("not pulled".to_string()),
answers: Vec::new(),
},
],
2,
);
assert!(e.answered_anything());
assert!(e.footnotes()[0].contains("NOT A MEASUREMENT"));
}
#[test]
fn errors_alone_do_not_count_as_having_answered() {
let e = examination(
vec![answered("alfa", vec![Answer::Failed("boom".into())])],
1,
);
assert!(!e.answered_anything());
assert!(e.footnotes()[0].contains("NOTHING RAN"));
}
#[test]
fn naming_no_detector_is_our_refusal_rather_than_claps() {
let registry = stegobench_core::registry::Registry::default();
let error = run(
®istry,
Request {
detectors: &[],
images: &[PathBuf::from("some-image.png")],
timeout: Duration::from_secs(1),
jobs: 1,
adapter_roots: &[],
raw: None,
},
|_| {},
)
.expect_err("naming no detector is refused");
assert!(
matches!(error, ExamineError::NoDetectorNamed),
"refused for the wrong reason: {error}"
);
assert_eq!(error.exit_code(), stegobench_core::exit::USAGE);
assert_eq!(error.reason(), "no-detector-named");
let said = error.to_string();
assert!(said.contains("--detector"), "{said}");
assert!(said.contains("list detectors"), "{said}");
assert!(
!said.contains("required arguments"),
"still clap's wording: {said}"
);
}
#[test]
fn the_missing_detector_is_named_before_a_missing_file_is() {
let registry = stegobench_core::registry::Registry::default();
let error = run(
®istry,
Request {
detectors: &[],
images: &[PathBuf::from("/no/such/file.png")],
timeout: Duration::from_secs(1),
jobs: 1,
adapter_roots: &[],
raw: None,
},
|_| {},
)
.expect_err("refused");
assert_eq!(error.reason(), "no-detector-named", "{error}");
}
#[test]
fn two_detectors_are_warned_not_to_be_compared_directly() {
let e = examination(
vec![
answered("alfa", vec![Answer::Score(0.1)]),
answered("bravo", vec![Answer::Score(0.9)]),
],
1,
);
assert!(e.footnotes().iter().any(|n| n.contains("own scale")));
}
#[test]
fn one_detector_plus_an_unavailable_one_is_not_two_scales() {
let e = examination(
vec![
answered("alfa", vec![Answer::Score(0.1)]),
Column {
detector: "bravo".to_string(),
unavailable: Some("not installed".to_string()),
answers: Vec::new(),
},
],
1,
);
assert!(!e.footnotes().iter().any(|n| n.contains("own scale")));
}
#[test]
fn the_json_says_it_is_not_a_measurement() {
let e = examination(vec![answered("alfa", vec![Answer::Score(0.1)])], 1);
let json = e.to_json();
assert_eq!(json["kind"], "examination");
assert_eq!(json["measurement"], false);
assert!(json.get("metrics").is_none());
assert!(json.get("auc").is_none());
}
#[test]
fn every_error_carries_its_own_exit_code_and_stable_word() {
use stegobench_core::exit;
let cases = [
(
ExamineError::NotRegistered { name: "x".into() },
exit::USAGE,
"not-registered",
),
(
ExamineError::NotADetector { name: "x".into() },
exit::USAGE,
"not-a-detector",
),
(
ExamineError::AskedTwice { name: "x".into() },
exit::USAGE,
"asked-twice",
),
(
ExamineError::NotAFile {
path: PathBuf::from("x"),
},
exit::PREFLIGHT_REFUSED,
"not-a-file",
),
];
for (error, code, reason) in cases {
assert_eq!(error.exit_code(), code, "{error}");
assert_eq!(error.reason(), reason);
assert!(!error.to_string().is_empty());
}
}
#[test]
fn asking_for_a_detector_twice_is_refused_rather_than_run_twice() {
let e = ExamineError::AskedTwice {
name: "zsteg".into(),
};
assert!(e.to_string().contains("twice"));
assert_eq!(e.exit_code(), stegobench_core::exit::USAGE);
}
}
#[cfg(test)]
mod expansion {
use super::*;
fn tree(files: &[&str]) -> tempfile::TempDir {
let dir = tempfile::tempdir().expect("tmp");
for name in files {
let path = dir.path().join(name);
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent).expect("mkdir");
}
std::fs::write(&path, b"x").expect("write");
}
dir
}
#[test]
fn a_named_file_is_itself() {
let dir = tree(&["a.png"]);
let file = dir.path().join("a.png");
assert_eq!(images_under(&file).expect("found"), vec![file.clone()]);
}
#[test]
fn a_named_directory_becomes_the_images_in_it_sorted() {
let dir = tree(&["c.png", "a.jpg", "b.bmp", "notes.txt", "run.sh"]);
let found = images_under(dir.path()).expect("found");
let names: Vec<String> = found
.iter()
.map(|p| p.file_name().unwrap().to_string_lossy().to_string())
.collect();
assert_eq!(names, vec!["a.jpg", "b.bmp", "c.png"]);
}
#[test]
fn an_extension_in_capitals_is_still_an_image() {
let dir = tree(&["PHOTO.JPG", "other.PNG"]);
assert_eq!(images_under(dir.path()).expect("found").len(), 2);
}
#[test]
fn a_subdirectory_is_not_walked_into() {
let dir = tree(&["top.png", "deeper/inside.png"]);
let found = images_under(dir.path()).expect("found");
assert_eq!(found.len(), 1);
assert!(found[0].ends_with("top.png"));
}
#[test]
fn a_directory_with_no_images_is_refused_rather_than_examined_as_empty() {
let dir = tree(&["notes.txt"]);
let e = images_under(dir.path()).expect_err("refused");
assert_eq!(e.reason(), "no-images-there");
assert_eq!(e.exit_code(), stegobench_core::exit::PREFLIGHT_REFUSED);
}
#[test]
fn a_path_that_is_neither_is_the_same_refusal_as_before() {
let dir = tree(&[]);
let e = images_under(&dir.path().join("nothing-here.png")).expect_err("refused");
assert_eq!(e.reason(), "not-a-file");
}
#[test]
fn naming_a_file_and_its_directory_asks_about_it_once() {
let dir = tree(&["a.png", "b.png"]);
let images =
expand(&[dir.path().to_path_buf(), dir.path().join("a.png")]).expect("expanded");
assert_eq!(images.len(), 2, "{images:?}");
}
#[test]
fn the_same_file_named_twice_is_asked_about_once() {
let dir = tree(&["a.png"]);
let file = dir.path().join("a.png");
let images = expand(&[file.clone(), file]).expect("expanded");
assert_eq!(images.len(), 1);
}
#[test]
fn two_directories_are_both_expanded_in_the_order_they_were_named() {
let first = tree(&["z.png"]);
let second = tree(&["a.png"]);
let images =
expand(&[first.path().to_path_buf(), second.path().to_path_buf()]).expect("expanded");
assert!(images[0].ends_with("z.png"), "{images:?}");
assert!(images[1].ends_with("a.png"), "{images:?}");
}
#[test]
fn one_bad_path_refuses_the_whole_command_rather_than_examining_the_rest() {
let dir = tree(&["a.png"]);
let e = expand(&[dir.path().to_path_buf(), PathBuf::from("/no/such/place")])
.expect_err("refused");
assert_eq!(e.reason(), "not-a-file");
}
}