use std::io::{IsTerminal, Write};
use std::path::{Path, PathBuf};
use clap::{CommandFactory, Parser};
use stegobench_cli::cli::{Cli, Command, ReportFormat};
use stegobench_cli::examine;
use stegobench_cli::fetch;
use stegobench_cli::fixtures;
use stegobench_cli::help_topics;
use stegobench_cli::metrics;
use stegobench_cli::needs;
use stegobench_cli::plain_io as plain;
use stegobench_cli::registry;
use stegobench_cli::registry::Resolved;
use stegobench_cli::report;
use stegobench_cli::score;
use stegobench_cli::STARTER_ID;
use stegobench_core::registry::{Kind, Registry};
use stegobench_core::{exit, ManifestV1, Result1, RunV1};
use stegobench_plugin::availability::Presence;
use stegobench_plugin::{availability, selftest, Verified};
#[cfg_attr(test, derive(Debug))]
struct Output {
json: serde_json::Value,
human: String,
code: i32,
payload_on_stdout: bool,
}
impl Output {
fn ok(json: serde_json::Value, human: impl Into<String>) -> Self {
Output {
json,
human: human.into(),
code: exit::OK,
payload_on_stdout: false,
}
}
fn err(code: i32, human: impl Into<String>) -> Self {
let human = human.into();
Output {
json: serde_json::json!({ "ok": false, "error": human }),
human,
code,
payload_on_stdout: false,
}
}
fn err_because(code: i32, reason: &str, human: impl Into<String>) -> Self {
let mut out = Output::err(code, human);
out.json["reason"] = serde_json::json!(reason);
out
}
}
fn score_refusal(e: &score::ScoreError, limit: Option<u64>) -> Output {
let why = e.to_string();
let human = match (e, limit) {
(score::ScoreError::OneSided { .. }, Some(n)) => format!(
"{why}.\n--limit {n} takes the FIRST {n} item(s) in corpus order, \
and a corpus lists its covers before its stego arms, so a small \
limit reaches clean images only. Raise it past the covers, drop \
it and score the whole corpus, or use --split test for a smaller \
whole."
),
_ => why,
};
Output::err_because(e.exit_code(), e.reason(), human)
}
const KNOWN_SCHEMAS: &[&str] = &["result-v1", "run-v1", "manifest-v1"];
fn schema_value(name: &str) -> Option<serde_json::Value> {
let schema = match name {
"result-v1" => serde_json::to_value(schemars::schema_for!(Result1)),
"run-v1" => serde_json::to_value(schemars::schema_for!(RunV1)),
"manifest-v1" => serde_json::to_value(schemars::schema_for!(ManifestV1)),
_ => return None,
};
Some(schema.expect("a generated schema serialises"))
}
fn cmd_schema(name: &str) -> Output {
if name == "all" {
let map: serde_json::Map<String, serde_json::Value> = KNOWN_SCHEMAS
.iter()
.map(|n| (n.to_string(), schema_value(n).expect("known schema")))
.collect();
return Output::ok(
serde_json::Value::Object(map),
"all schemas written to stdout",
);
}
match schema_value(name) {
Some(v) => Output::ok(v, format!("{name} schema written to stdout")),
None => Output::err(
exit::USAGE,
format!(
"unknown schema {name:?}. Known schemas: {}, or `all`",
KNOWN_SCHEMAS.join(", ")
),
),
}
}
fn with_registry(cli: &Cli, f: impl FnOnce(&Resolved) -> Output) -> Output {
match registry::resolve(cli.registry.as_deref()) {
Ok(resolved) => f(&resolved),
Err(e) => Output::err(e.exit_code(), e.to_string()),
}
}
fn corpora_block(reg: &Registry, where_from: &str) -> String {
if reg.corpora.is_empty() {
return format!("no corpora registered in {where_from}");
}
let rows = reg.corpora.values().map(|c| c.cells()).collect::<Vec<_>>();
let mut text = stegobench_core::table::align(&rows).join("\n");
let publishable = reg
.corpora
.values()
.filter(|c| c.licence.redistribution.allows_publishing())
.count();
text.push_str(&format!(
"\n\n{} corpora, {publishable} republishable. Using one is not \
publishing it; `describe <id>` prints the terms.",
reg.corpora.len()
));
text
}
fn cmd_list(resolved: &Resolved, kind: &str) -> Output {
let reg = &resolved.registry;
let where_from = match resolved.source.path() {
Some(p) => p.display().to_string(),
None => "built in".to_string(),
};
let corpora_from = match resolved.source.path() {
Some(p) => format!("{}/corpora", p.display()),
None => where_from.clone(),
};
if kind == "corpora" {
return Output::ok(
serde_json::json!({
"corpora": reg.corpora.values().collect::<Vec<_>>(),
"count": reg.corpora.len(),
"registry": resolved.to_json(),
}),
format!(
"{}\n\n{}",
corpora_block(reg, &corpora_from),
resolved.source.line()
),
);
}
let wanted: Vec<_> = match kind {
"detectors" => reg.of_kind(Kind::Detector),
"embedders" => reg.of_kind(Kind::Embedder),
"all" => reg.entries.values().collect(),
other => {
return Output::err(
exit::USAGE,
format!("unknown kind {other:?}. Known: detectors, embedders, corpora, all"),
)
}
};
let rows = wanted.iter().map(|e| e.cells()).collect::<Vec<_>>();
let aligned = stegobench_core::table::align(&rows);
let mut human = if kind == "all" {
let mut out: Vec<String> = Vec::new();
for (k, heading) in [(Kind::Detector, "DETECTORS"), (Kind::Embedder, "EMBEDDERS")] {
let group: Vec<&String> = wanted
.iter()
.zip(&aligned)
.filter(|(e, _)| e.kind == k)
.map(|(_, line)| line)
.collect();
if group.is_empty() {
continue;
}
if !out.is_empty() {
out.push(String::new());
}
out.push(format!("{heading} ({})", group.len()));
out.extend(group.into_iter().cloned());
}
out.join("\n")
} else {
aligned.join("\n")
};
if human.is_empty() {
human = if reg.entries.is_empty() {
format!("no tools registered in {where_from}")
} else {
format!(
"none of the {} tool(s) in {where_from} is a {}. `list all` \
prints every one.",
reg.entries.len(),
kind.trim_end_matches('s')
)
};
} else {
human.push_str(
"\n\ncontainer pinned by image digest, sandboxed, no network. \
Needs a container runtime.\nlocal a program you installed, \
pinned by the hash of the file that ran. No sandbox, your \
network.",
);
let services = wanted
.iter()
.filter(|e| e.invoke.as_ref().is_some_and(|i| i.host))
.map(|e| e.name.as_str())
.collect::<Vec<_>>();
if !services.is_empty() {
human.push_str(&format!(
"\n\nSERVICE, not sandboxed, and needs the network: {}. The \
image names the subject; an adapter here reaches an instance \
you started. Nothing checks that instance was built from that \
image, so a result of it reads `unpinned`.",
services.join(", ")
));
}
human.push_str("\n\n`stegobench doctor` says what each one still needs.");
if wanted.iter().any(|e| e.kind == Kind::Detector) {
if let Some(way) = stegobench_cli::smallest_way_in(&resolved.registry) {
human.push_str(&format!("\n\n{way}"));
}
}
let f = stegobench_core::registry::Registry::footprint_of(wanted.iter().copied());
human.push_str(&format!(
"\n\n{} tools in {} images. {} MB small enough to bundle, {:.1} GB \
larger and fetched on demand.",
f.tools,
f.unique_images,
f.bundled_mb,
f.on_demand_mb as f64 / 1024.0
));
}
let mut json = serde_json::json!({
"tools": wanted,
"footprint": stegobench_core::registry::Registry::footprint_of(wanted.iter().copied()),
"registry": resolved.to_json(),
});
if kind == "all" {
human.push_str(&format!(
"\n\nCORPORA ({})\n{}",
reg.corpora.len(),
corpora_block(reg, &corpora_from)
));
json["corpora"] = serde_json::json!(reg.corpora.values().collect::<Vec<_>>());
}
human.push_str(&format!("\n\n{}", resolved.source.line()));
Output::ok(json, human)
}
fn cmd_embed(
resolved: &Resolved,
embedder: &str,
cover: &Path,
payload: &Path,
out: &Path,
passphrase: Option<&str>,
no_verify: bool,
) -> Output {
let reg = &resolved.registry;
let Some(entry) = reg.entries.get(embedder) else {
return Output::err(
exit::USAGE,
format!(
"no embedder called {embedder} is registered. `stegobench \
list embedders` shows the ones that are"
),
);
};
if entry.kind != Kind::Embedder {
return Output::err(
exit::USAGE,
format!(
"{embedder} is registered as a detector, which answers \
questions about an image rather than hiding anything in \
one. `stegobench list embedders` shows what can embed"
),
);
}
for (what, path) in [("cover", cover), ("payload", payload)] {
if !path.is_file() {
return Output::err(
exit::PREFLIGHT_REFUSED,
format!("the {what} {} is not a file here", path.display()),
);
}
}
match stegobench_plugin::embed::run(entry, cover, payload, out, passphrase, !no_verify) {
Err(e) if e.contains("declares no way to embed") => Output::err(exit::PREFLIGHT_REFUSED, e),
Err(e) if e.starts_with("could not stage") || e.starts_with("no scratch") => {
Output::err(exit::PREFLIGHT_REFUSED, e)
}
Err(e) => Output::err(exit::PLUGIN_FAILED, e),
Ok(done) => {
let checked = match &done.recovered {
Some(Ok(())) => "the payload was extracted again and matches byte for byte",
Some(Err(_)) => "the payload did NOT survive",
None => "the payload was NOT checked, so nothing here says it is really in there",
};
let json = serde_json::json!({
"embedder": entry.name,
"cover": cover.display().to_string(),
"stego": done.stego.display().to_string(),
"bytes": done.bytes,
"verified": match &done.recovered {
Some(Ok(())) => serde_json::json!(true),
Some(Err(e)) => serde_json::json!({ "ok": false, "reason": e }),
None => serde_json::Value::Null,
},
});
if let Some(Err(e)) = &done.recovered {
let mut bad = Output::err(
exit::VERIFY_MISMATCH,
format!(
"{} wrote {} ({} bytes) and {e}.\nThe file is there \
and what is in it is not what you gave it, so it is \
not a stego image of that payload.",
entry.name,
done.stego.display(),
done.bytes
),
);
bad.json = json;
return bad;
}
Output::ok(
json,
format!(
"{} wrote {} ({} bytes), and {checked}.\nOne image is a \
demonstration. A corpus needs every pair written from \
one source through one code path: see `stegobench help \
pairing`.",
entry.name,
done.stego.display(),
done.bytes
),
)
}
}
}
fn cmd_describe(resolved: &Resolved, name: &str, toml_only: bool) -> Output {
let reg = &resolved.registry;
if let Some(e) = reg.entries.get(name) {
let needs = needs::of_tool(
e,
&availability::check(e, resolved.adapter_roots()),
resolved.adapter_roots(),
);
let mut json = serde_json::to_value(e).unwrap_or(serde_json::Value::Null);
if let Some(map) = json.as_object_mut() {
map.insert("needs".into(), needs.to_json());
}
if toml_only {
return raw_toml(e, json, &e.name);
}
let durable: &[PathBuf] = match resolved.source.path() {
Some(_) => resolved.adapter_roots(),
None => &[],
};
return Output::ok(
json,
describe_block(&needs, e.name.as_str(), "tool", &tool_facts(e, durable)),
);
}
if let Some(c) = reg.corpora.get(name) {
let needs = needs::of_corpus(c);
let mut json = serde_json::to_value(c).unwrap_or(serde_json::Value::Null);
if let Some(map) = json.as_object_mut() {
map.insert("needs".into(), needs.to_json());
}
if toml_only {
return raw_toml(c, json, &c.id);
}
return Output::ok(
json,
describe_block(&needs, c.id.as_str(), "corpus", &corpus_facts(c)),
);
}
let known: Vec<_> = reg
.entries
.keys()
.chain(reg.corpora.keys())
.cloned()
.collect();
let mut human = if Path::new(name).exists() {
format!(
"{name:?} is a path on this machine, and `describe` takes a \
registered name or corpus id rather than a path. There is nothing \
registered under that name.\nTo measure a directory of samples:\n \
stegobench score --corpus {name} --detector <name>\n"
)
} else {
let close = near_registered_names(reg, name);
let mut head = format!("nothing registered as {name:?}.");
if !close.is_empty() {
head.push_str(&format!(" Did you mean: {}?", close.join(", ")));
}
head.push('\n');
head
};
human.push_str(&format!("Known tools and corpora: {}", known.join(", ")));
Output::err(exit::USAGE, human)
}
fn raw_toml<T: serde::Serialize>(entry: &T, json: serde_json::Value, name: &str) -> Output {
let rendered = toml::Value::try_from(entry).map(|mut value| {
if let Some(table) = value.as_table_mut() {
table.remove(stegobench_core::registry::MAY_BE_SERVED_KEY);
}
value
});
match rendered.and_then(|v| toml::to_string_pretty(&v)) {
Ok(text) => {
let mut out = Output::ok(json, text);
out.payload_on_stdout = true;
out
}
Err(e) => Output::err(
exit::FAILURE,
format!(
"the registered entry for {name} could not be written back as \
TOML: {e}. `stegobench describe {name}` prints the summary, \
and `--json` prints the same entry as JSON."
),
),
}
}
fn readiness_word(needs: &needs::Needs, kind: &str, name: &str) -> String {
if kind == "corpus" && needs.readiness == needs::Readiness::Unknown {
return if name == STARTER_ID {
"included in this binary".to_string()
} else {
"bring your own copy".to_string()
};
}
if needs.readiness == needs::Readiness::Ready {
return "stegobench can run it".to_string();
}
if needs.readiness == needs::Readiness::CannotBeDriven {
return "stegobench cannot drive it".to_string();
}
needs.readiness.word().to_string()
}
fn describe_block(
needs: &needs::Needs,
name: &str,
kind: &str,
facts: &[(&str, String)],
) -> String {
let width = facts.iter().map(|(k, _)| k.len()).max().unwrap_or(0);
let mut text = format!("{name} [{}]\n", readiness_word(needs, kind, name));
for (label, value) in facts {
let mut lines = value.lines();
let first = lines.next().unwrap_or("");
text.push_str(&format!("{label:<width$} {first}\n", width = width));
for line in lines {
text.push_str(&format!("{:<width$} {line}\n", "", width = width));
}
}
let steps = needs.block();
if steps.is_empty() {
text.push_str(&format!(
"\nNothing needed. `stegobench doctor` says whether this {kind} \
works.\n"
));
} else {
let heading = match needs.readiness {
needs::Readiness::CannotRunHere | needs::Readiness::CannotBeDriven => {
"Why you can't use it here:"
}
_ => "Needs from you:",
};
text.push_str(&format!("\n{heading}\n{steps}\n"));
}
text.push_str(&format!(
"\nThe registered entry in full: stegobench describe {name} --toml"
));
text
}
fn shell_word(word: &str) -> String {
let safe = |c: char| c.is_ascii_alphanumeric() || "_@%+=:,./-{}".contains(c);
if !word.is_empty() && word.chars().all(safe) {
return word.to_string();
}
format!("'{}'", word.replace('\'', r"'\''"))
}
fn runnable_command(
e: &stegobench_core::registry::Entry,
adapter_roots: &[PathBuf],
) -> Option<String> {
let invoke = e.invoke.as_ref();
let argv = invoke.map(|i| i.argv.as_slice()).unwrap_or(&[]);
let host = invoke.is_some_and(|i| i.host);
if host || e.image.is_none() {
if let Some(bin) = e.binary.as_ref().filter(|b| !b.command.is_empty()) {
let mut words: Vec<String> = bin.command.iter().map(|w| shell_word(w)).collect();
words.extend(argv.iter().map(|a| shell_word(a)));
return Some(words.join(" "));
}
let (Some(i), Some(rel)) = (invoke, invoke.and_then(|i| i.adapter.as_ref())) else {
return None;
};
let adapter = stegobench_plugin::adapter::resolve(rel, adapter_roots).ok()?;
let program = i
.entrypoint
.clone()
.unwrap_or_else(|| stegobench_plugin::DEFAULT_HOST_ENTRYPOINT.to_string());
let mut words = vec![shell_word(&program)];
for arg in argv {
words.push(shell_word(
&arg.replace("{adapter}", &adapter.display().to_string()),
));
}
return Some(words.join(" "));
}
let image = e.image.as_ref()?;
let i = invoke?;
let mut words: Vec<String> = ["docker", "run", "--rm"]
.iter()
.map(|s| s.to_string())
.collect();
if i.writable_workdir {
words.push("--user".into());
words.push("$(id -u):$(id -g)".into());
}
for flag in stegobench_plugin::selftest::SANDBOX {
words.push((*flag).into());
}
words.push("-v".into());
words.push(if i.writable_workdir {
"\"$PWD\":/work".into()
} else {
"\"$PWD/{file}\":/work/{file}:ro".into()
});
let mut adapter_inner = String::new();
if let Some(rel) = &i.adapter {
let abs = stegobench_plugin::adapter::resolve(rel, adapter_roots).ok()?;
let base = abs.file_name()?.to_str()?;
adapter_inner = format!("/adapter/{base}");
words.push("-v".into());
words.push(shell_word(&format!("{}:{adapter_inner}:ro", abs.display())));
}
for kv in &i.env {
words.push("-e".into());
words.push(shell_word(kv));
}
if let Some(ep) = &i.entrypoint {
words.push("--entrypoint".into());
words.push(shell_word(ep));
}
words.push(shell_word(&image.reference));
for arg in argv {
words.push(shell_word(
&arg.replace("{file}", "/work/{file}")
.replace("{adapter}", &adapter_inner),
));
}
Some(words.join(" "))
}
fn tool_facts(
e: &stegobench_core::registry::Entry,
adapter_roots: &[PathBuf],
) -> Vec<(&'static str, String)> {
let mut facts = vec![(
"Kind",
match e.kind {
Kind::Detector => "detector".to_string(),
Kind::Embedder => "embedder".to_string(),
},
)];
facts.push(("Licence", e.licence.clone()));
if let Some(r) = e.redistribution {
facts.push((
"Republish",
format!(
"{}{}",
stegobench_core::corpus::redistribution_label(r),
match e.redistribution_reason.as_deref().map(str::trim) {
Some(why) if !why.is_empty() => format!(". {why}"),
_ => String::new(),
}
),
));
}
let runs = match (&e.image, &e.binary) {
(Some(i), _) => {
let sandbox = if e.invoke.as_ref().is_some_and(|i| i.host) {
"a service this only talks to, so its isolation is not ours to state"
} else if i.needs_network {
"a container, with network"
} else {
"a container, sandboxed with no network"
};
format!("{} ({sandbox})", i.reference)
}
(_, Some(b)) => format!(
"{} (a program you installed, no sandbox, your network)",
b.command.first().map(String::as_str).unwrap_or("a binary")
),
_ => "nothing declared, so it cannot be run".to_string(),
};
facts.push(("Runs as", runs));
match runnable_command(e, adapter_roots) {
Some(line) => {
let containerised = e.image.is_some() && !e.invoke.as_ref().is_some_and(|i| i.host);
let note = if !line.contains("{file}") {
"It reads every image in the directory you run it from."
} else if containerised {
"{file} is your image's filename, and this runs from the directory it is in."
} else {
"{file} is the path to your image."
};
facts.push(("Run it", format!("{line}\n{note}")));
}
None => {
if let Some(adapter) = e.invoke.as_ref().and_then(|i| i.adapter.as_ref()) {
facts.push((
"Run it",
format!(
"no command to paste: this one needs {adapter}, and \
the copy inside this binary lasts only as long as a \
run.\nWith a checkout of the repository: stegobench \
describe {} --registry plugins/registry",
e.name
),
));
}
}
}
facts.push((
"Answers",
if !e.emits.declared() {
"not declared: this entry has no [emits] block".to_string()
} else {
match e.emits.output {
stegobench_core::registry::Output::Score => {
if e.emits.higher_means_stego {
"a score, higher means more like stego".to_string()
} else {
"a score, lower means more like stego".to_string()
}
}
stegobench_core::registry::Output::Verdict => {
"a verdict, yes or no, so it gives one point and not a curve".to_string()
}
}
},
));
if !e.accepts.formats.is_empty() {
facts.push(("Reads", e.accepts.formats.join(", ")));
}
if let Some(s) = e.cost.seconds_per_image {
facts.push(("Costs", format!("about {s} seconds an image")));
}
if let Some(u) = &e.upstream {
facts.push(("Upstream", u.clone()));
}
if let Some(n) = &e.notes {
facts.push(("Notes", n.clone()));
}
facts
}
fn corpus_facts(c: &stegobench_core::corpus::CorpusEntry) -> Vec<(&'static str, String)> {
use stegobench_core::corpus::{LicenceStatus, Redistribution};
let mut facts = vec![("Name", c.name.clone()), ("What", c.description.clone())];
if let Some(t) = &c.tier {
facts.push(("Tier", t.clone()));
}
let licence = match c.licence.status {
LicenceStatus::Verified => {
let named = c.licence.spdx.clone().unwrap_or_else(|| "verified".into());
let inferred = if c.licence.spdx_version_inferred {
", version inferred rather than stated"
} else {
""
};
match &c.licence.verified_on {
Some(day) => format!("{named}, verified_on {day}{inferred}"),
None => format!("{named}, verified{inferred}"),
}
}
LicenceStatus::Unverified => {
"UNVERIFIED: nobody has established the terms, so treat it as granting nothing"
.to_string()
}
LicenceStatus::NoneGranted => {
"NO LICENCE: there is no grant, which is not the same as none being found".to_string()
}
};
facts.push(("Licence", licence));
if let Some(src) = &c.licence.source {
facts.push(("Read from", src.clone()));
}
facts.push((
"Republish",
format!(
"{}: {}",
match c.licence.redistribution {
Redistribution::Permitted => "permitted",
Redistribution::Forbidden => "forbidden",
Redistribution::Unknown => "unknown, which any gate here treats as no",
Redistribution::MirroredByDecision =>
"mirrored by decision, with no grant behind it",
},
c.licence.redistribution_reason
),
));
let size = match (c.properties.base_images, c.properties.total_images) {
(Some(b), Some(t)) => format!("{b} covers, {t} images in all"),
(Some(b), None) => format!("{b} covers"),
(None, Some(t)) => format!("{t} images"),
(None, None) => match (&c.properties.size_mb, &c.properties.size_note) {
(Some(mb), _) => format!("{mb} MB"),
(None, Some(note)) => note.clone(),
(None, None) => "size unestablished".to_string(),
},
};
facts.push(("Size", size));
if let Some(paired) = c.properties.paired {
facts.push((
"Paired",
if paired {
"yes, every stego image has its clean twin".to_string()
} else {
"NO, so it cannot answer the question this benchmark asks".to_string()
},
));
}
if c.demonstration {
facts.push((
"Demonstration",
"yes: enough to prove a detector runs, nowhere near enough to measure one".to_string(),
));
}
if let Some(cite) = &c.citation {
facts.push(("Cite as", cite.clone()));
}
if let Some(n) = &c.notes {
facts.push(("Notes", n.clone()));
}
facts
}
fn is_sha256_hex(text: &str) -> bool {
text.len() == 64 && text.bytes().all(|b| b.is_ascii_hexdigit())
}
const VERSION_MAX_CHARS: usize = 40;
fn version_column(version: &str) -> String {
let v = version.trim();
let plausible = !v.is_empty()
&& v.chars().count() <= VERSION_MAX_CHARS
&& !v.chars().any(char::is_control)
&& v.chars().any(|c| c.is_ascii_digit());
if plausible {
v.to_string()
} else {
"version probe failed".to_string()
}
}
fn pin_columns(pin: &str) -> String {
if let Some((version, hash)) = pin.rsplit_once(' ') {
if let Some(short) = short_sha256(hash) {
return format!("{} {short}", version_column(version));
}
}
short_sha256(pin).unwrap_or_else(|| pin.chars().take(23).collect())
}
fn short_sha256(text: &str) -> Option<String> {
let hex = text.strip_prefix("sha256:")?;
is_sha256_hex(hex).then(|| format!("sha256:{}", &hex[..12]))
}
fn doctor_row(check: &availability::Availability, drivable: bool, width: usize) -> String {
let state = match &check.presence {
Presence::Present { .. } if !drivable => {
"undrivable nothing in its entry says what to launch".to_string()
}
Presence::Present { pin } => format!("present {}", pin_columns(pin)),
Presence::Absent { reason } => format!("MISSING {reason}"),
Presence::Unknown { reason } => format!("unknown {reason}"),
Presence::Unsupported { reason } => format!("n/a {reason}"),
};
let verified = match check.verified {
Some(true) => " verified",
Some(false) => " SELF TEST FAILED",
None => " not verified",
};
let secrets = if check.missing_secrets.is_empty() {
String::new()
} else {
format!(" needs {}", check.missing_secrets.join(", "))
};
format!(
"{:<width$} {state}{verified}{secrets}",
check.name,
width = width
)
}
fn cmd_doctor(
resolved: &Resolved,
fixtures: Option<&fixtures::Fixtures>,
strict: bool,
registry_reach: bool,
) -> Output {
let reg = &resolved.registry;
let no_selftest = fixtures.is_none();
let mut rows = Vec::new();
let (mut missing, mut broken, mut passed, mut skipped, mut answered) = (0, 0, 0, 0, 0);
let mut unsupported = 0;
let mut undetermined = 0;
let mut present = 0;
let (mut unfetchable, mut unasked) = (0, 0);
let width = reg
.entries
.values()
.map(|e| e.name.chars().count())
.max()
.unwrap_or(0)
+ 1;
for entry in reg.entries.values() {
let mut check = availability::check(entry, resolved.adapter_roots());
let verdict = match fixtures {
Some(f) if check.presence.is_present() => {
selftest::run(entry, f.dir(), resolved.adapter_roots())
}
_ => Verified::Skipped("not attempted".into()),
};
check.verified = match &verdict {
Verified::Passed => Some(true),
Verified::Failed(_) => Some(false),
Verified::Skipped(_) | Verified::Answered(_) => None,
};
match &check.presence {
Presence::Unsupported { .. } => unsupported += 1,
Presence::Unknown { .. } => undetermined += 1,
p if !p.is_present() => missing += 1,
_ => present += 1,
}
match &verdict {
Verified::Passed => passed += 1,
Verified::Failed(_) => broken += 1,
Verified::Answered(_) => answered += 1,
Verified::Skipped(_) => skipped += 1,
}
let row = doctor_row(&check, entry.can_be_driven(), width);
let detail = match &verdict {
Verified::Failed(why) => format!("{row} ({why})"),
Verified::Answered(why) => {
format!("{} ({why})", row.replace("not verified", "responded "))
}
Verified::Skipped(_) if !entry.can_be_driven() => row,
Verified::Skipped(why) if !no_selftest && check.presence.is_present() => {
format!("{row} ({why})")
}
_ => row,
};
let detail = match (registry_reach, &entry.image) {
(true, Some(image)) if needs::runs_in_container(entry) => {
let said = match stegobench_plugin::registry_reach(&image.reference) {
stegobench_plugin::Reach::Fetchable => "fetchable".to_string(),
stegobench_plugin::Reach::Absent { reason } => {
unfetchable += 1;
format!("NOT FETCHABLE: {reason}")
}
stegobench_plugin::Reach::NotRun { reason } => {
unasked += 1;
format!("not asked: {reason}")
}
};
format!("{detail}\n registry {said}")
}
_ => detail,
};
let needs = needs::of_tool(entry, &check, resolved.adapter_roots());
rows.push((check, verdict, detail, needs, entry.kind));
}
let mut human: Vec<String> = Vec::new();
human.push(resolved.source.line());
human.push(match fixtures {
Some(f) => f.line(),
None => "fixtures not read, because the self-tests were not run".to_string(),
});
let detectors = rows.iter().filter(|r| r.4 == Kind::Detector).count();
let embedders = rows.len() - detectors;
for (kind, heading, count) in [
(Kind::Detector, "DETECTORS", detectors),
(Kind::Embedder, "EMBEDDERS", embedders),
] {
if count == 0 {
continue;
}
human.push(String::new());
human.push(format!("{heading} ({count})"));
for (_, _, detail, needs, _) in rows.iter().filter(|r| r.4 == kind) {
human.push(detail.clone());
let block = needs.block();
if !block.is_empty() {
for line in block.lines() {
human.push(format!(" {line}"));
}
}
}
}
let needing = rows
.iter()
.filter(|(_, _, _, n, _)| n.readiness == needs::Readiness::NeedsYou)
.count();
human.push(String::new());
human.push(format!(
"{} tool(s): {detectors} detector(s), {embedders} embedder(s).",
rows.len()
));
human.push(format!(
"On this machine: {present} installed, {missing} not installed, \
{undetermined} undetermined, {unsupported} cannot run here."
));
human.push(format!(
"Stegobench's own self-test: {passed} passed, {answered} responded, \
{broken} failed, {skipped} not run. Passing says stegobench can \
drive it, not that you can run it yourself."
));
if undetermined > 0 {
human.push(format!(
"{undetermined} undetermined: nothing to install would settle it. \
Its line says what it needs."
));
}
if unsupported > 0 {
human.push(format!(
"{unsupported} cannot run on {} at all, so nothing to install would change it.",
std::env::consts::OS
));
}
if answered > 0 {
human.push(
"responded: it answered without settling either fixture, so nothing is proved.".into(),
);
}
if skipped > 0 {
human.push("not run is not the same as working; each line says why.".into());
}
if needing > 0 {
human.push(format!(
"{needing} need something from you, indented under each."
));
}
let json = serde_json::json!({
"registry": resolved.to_json(),
"fixtures": fixtures.map(|f| serde_json::json!({
"source": f.source().tag(),
"path": f.dir().display().to_string(),
})),
"checked": rows.len(),
"detectors": detectors,
"embedders": embedders,
"needing_action": needing,
"present": present,
"verified": passed,
"broken": broken,
"missing": missing,
"unsupported_here": unsupported,
"undetermined": undetermined,
"not_checked": skipped,
"responded": answered,
"tools": rows.iter().map(|(c, v, d, n, k)| serde_json::json!({
"name": c.name,
"kind": match k {
Kind::Detector => "detector",
Kind::Embedder => "embedder",
},
"present": c.presence.is_present(),
"verified": c.verified,
"status": match v {
Verified::Passed => "passed",
Verified::Failed(_) => "failed",
Verified::Answered(_) => "responded",
Verified::Skipped(_) => "not_checked",
},
"detail": d,
"missing_secrets": c.missing_secrets,
"needs": n.to_json(),
})).collect::<Vec<_>>(),
});
let mut out = Output::ok(json, human.join("\n"));
out.payload_on_stdout = true;
let strictly_short = strict && (missing > 0 || undetermined > 0);
let usable = passed + answered;
let unfit = if no_selftest {
present == 0 || strictly_short
} else {
broken > 0 || usable == 0 || strictly_short
};
if unfit {
out.code = exit::ENVIRONMENT_UNFIT;
}
if out.code == exit::ENVIRONMENT_UNFIT {
let fresh = present == 0 && broken == 0 && !strictly_short;
let why = if !no_selftest && broken > 0 {
format!("{broken} installed tool(s) failed their own self-test")
} else if fresh {
"nothing is installed here yet, so there is nothing to measure \
with. That is the normal state of a fresh install rather than a \
fault on this machine"
.to_string()
} else if no_selftest && present == 0 {
"no tool here is installed, so nothing could be measured".to_string()
} else if !no_selftest && usable == 0 {
"no tool here is usable, so nothing could be measured".to_string()
} else {
"--strict was given and something is missing or undetermined".to_string()
};
out.human.push_str(&format!("\n\nUNFIT: {why}."));
if fresh {
if let Some(way) = stegobench_cli::smallest_way_in(&resolved.registry) {
out.human.push_str(&format!("\n\n{way}"));
}
}
} else if no_selftest {
out.human.push_str(&format!(
"\n\nFIT: {present} tool(s) installed. The self-tests were \
skipped, so none of them is proved to work."
));
} else if missing > 0 || undetermined > 0 {
out.human.push_str(
"\n\nFIT: what is not installed is listed, not counted against \
you. `--strict` counts it.",
);
}
if registry_reach && (unfetchable > 0 || unasked > 0) {
if unfetchable > 0 {
out.human.push_str(&format!(
"\n\nNOT REPRODUCIBLE ELSEWHERE: {unfetchable} pinned image(s) \
are not in their registry. A result naming one pins the bytes \
that ran here and gives a reader no way to obtain them."
));
}
if unasked > 0 {
out.human.push_str(&format!(
"\n\n{unasked} image(s) could not be asked about. That is not \
the same as their being absent, and each line says why."
));
}
}
out
}
fn cmd_validate(file: &Path) -> Output {
let text = match std::fs::read_to_string(file) {
Ok(t) => t,
Err(e) => {
return Output::err(
exit::FAILURE,
format!("cannot read {}: {e}", file.display()),
)
}
};
let sniff: serde_json::Value = match serde_json::from_str(&text) {
Ok(v) => v,
Err(e) => {
return Output::err(
exit::SCHEMA_INVALID,
format!("{} is not valid JSON: {e}", file.display()),
)
}
};
let schema_field = sniff.get("schema").and_then(|v| v.as_str());
macro_rules! validate_as {
($ty:ty, $label:literal) => {
validate_as!($ty, $label, |_| None)
};
($ty:ty, $label:literal, $seal:expr) => {{
let parsed: $ty = match serde_json::from_str(&text) {
Ok(v) => v,
Err(e) => {
return Output::err(
exit::SCHEMA_INVALID,
format!("{} is not a {} document: {e}", file.display(), $label),
)
}
};
match parsed.validate() {
Ok(()) => {
let mut json = serde_json::json!({ "ok": true, "schema": parsed.schema });
let mut human = format!("{} is a valid {} document", file.display(), $label);
let seal: Option<(&'static str, serde_json::Value)> = ($seal)(&parsed);
if let Some((line, detail)) = seal {
json["seal"] = detail;
human.push('\n');
human.push_str(line);
}
Output::ok(json, human)
}
Err(problems) => {
let mut out = Output::err(
exit::SCHEMA_INVALID,
format!(
"{} is not valid:\n {}",
file.display(),
problems.join("\n ")
),
);
out.json = serde_json::json!({ "ok": false, "problems": problems });
out
}
}
}};
}
let seal_note = |r: &Result1| -> Option<(&'static str, serde_json::Value)> {
Some(if r.content_digest.is_some() {
(
"Its seal leaves out started_utc and elapsed_seconds, so a \
changed timestamp wouldn't show up here.",
serde_json::json!({
"sealed": true,
"excluded": ["provenance.started_utc", "provenance.elapsed_seconds"],
}),
)
} else {
(
"It carries no content digest, so nothing here checked whether \
it changed after it was written.",
serde_json::json!({ "sealed": false }),
)
})
};
match schema_field {
Some(s) if s.starts_with("stegobench/result-v") => {
validate_as!(Result1, "result-v1", seal_note)
}
Some(s) if s.starts_with("stegobench/run-v") => validate_as!(RunV1, "run-v1"),
Some(s) if s.starts_with("stegobench/manifest-v") => {
validate_as!(ManifestV1, "manifest-v1")
}
Some(other) => Output::err(
exit::SCHEMA_INVALID,
format!(
"{} declares schema {other:?}, which this version does not \
know. Known: {}",
file.display(),
KNOWN_SCHEMAS.join(", ")
),
),
None => Output::err(
exit::SCHEMA_INVALID,
format!(
"{} has no \"schema\" field, so it cannot be dispatched to a \
validator. Known: {}",
file.display(),
KNOWN_SCHEMAS.join(", ")
),
),
}
}
fn cmd_verify(file: &Path, corpus: &Path, records: Option<&Path>, shallow: bool) -> Output {
let text = match std::fs::read_to_string(file) {
Ok(t) => t,
Err(e) => {
return Output::err(
exit::FAILURE,
format!("cannot read {}: {e}", file.display()),
)
}
};
let result: Result1 = match serde_json::from_str(&text) {
Ok(v) => v,
Err(e) => {
return Output::err(
exit::SCHEMA_INVALID,
format!(
"{} is not a result-v1 document: {e}. `stegobench validate \
{}` says what is wrong with it",
file.display(),
file.display()
),
)
}
};
if let Some(claimed) = result.content_digest.as_deref() {
if claimed != result.compute_content_digest() {
return Output::err(
exit::VERIFY_MISMATCH,
format!(
"{} does not match its own content digest, which it \
declares as\n {claimed}\nEvery field except the two that \
record WHEN the run happened is covered, so something in \
this document changed after it was written. Re-run the \
measurement rather than trusting the number in it",
file.display()
),
);
}
}
if let Some(path) = records {
let Some(claimed) = result.provenance.records.as_ref() else {
return Output::err_because(
exit::VERIFY_MISMATCH,
"records-unrecorded",
format!(
"{} records no digest for its per-item scores, so {} \
cannot be checked against it. The document was written \
before this was recorded, or by a run that wrote no \
records file",
file.display(),
path.display()
),
);
};
match stegobench_core::result::records_digest(path) {
Err(e) => {
return Output::err(
exit::FAILURE,
format!("cannot read {}: {e}", path.display()),
)
}
Ok(found) if found.digest != claimed.digest => {
return Output::err_because(
exit::VERIFY_MISMATCH,
"records-mismatch",
format!(
"{} is not the scores {} was computed from.\n the \
document names {} over {} record(s)\n that file is \
{} over {}\nThe metrics in the document \
were computed from a different file, so re-deriving \
them from this one gives different numbers",
path.display(),
file.display(),
claimed.digest,
claimed.count,
found.digest,
found.count
),
);
}
Ok(_) => {}
}
}
if result.corpus.digest.is_empty() {
return Output::err(
exit::VERIFY_MISMATCH,
format!(
"{} names no corpus digest, so there is nothing to check it \
against. A corpus whose records state no digests produces a \
result that cannot be verified this way",
file.display()
),
);
}
let found = match score::corpus_digest(corpus) {
Ok(Some(d)) => d,
Ok(None) => {
return Output::err(
exit::VERIFY_MISMATCH,
format!(
"the corpus at {} cannot be named: at least one record \
states no digest for its own image",
corpus.display()
),
)
}
Err(e) => return Output::err_because(exit::FAILURE, e.reason(), e.to_string()),
};
let claimed = &result.corpus.digest;
let mut json = serde_json::json!({
"ok": &found == claimed,
"claimed": claimed,
"found": found,
"corpus": corpus.display().to_string(),
"checked": if shallow { "records" } else { "bytes" },
});
if &found == claimed {
if !shallow {
let (bad, read) = match score::rehash_corpus(corpus, |line| {
eprintln!("verify: {line}");
}) {
Ok(v) => v,
Err(e) => return Output::err_because(exit::FAILURE, e.reason(), e.to_string()),
};
if !bad.is_empty() {
json["ok"] = serde_json::Value::Bool(false);
json["mismatched"] = serde_json::json!(bad
.iter()
.map(|m| serde_json::json!({
"id": m.id,
"claimed": m.claimed,
"found": match &m.found {
Ok(f) => serde_json::json!(f),
Err(e) => serde_json::json!({ "error": e }),
},
}))
.collect::<Vec<_>>());
let mut lines = String::new();
for m in &bad {
let got = match &m.found {
Ok(f) => f.clone(),
Err(e) => format!("could not be read: {e}"),
};
lines.push_str(&format!(
"\n {} claims {}\n and is {}",
m.id, m.claimed, got
));
}
let mut out = Output::err(
exit::VERIFY_MISMATCH,
format!(
"the records at {} match the document, and the images \
do not match the records.{lines}\nThe corpus digest \
is computed from what the records state, so it still \
agrees; the bytes that were scored are not the bytes \
here now. That number cannot be attributed to this \
corpus.",
corpus.display()
),
);
out.json = json;
return out;
}
if let Some(total) = result.corpus.of_items {
let scored = result.metrics.n_clean + result.metrics.n_stego;
let mut out = Output::err_because(
exit::VERIFY_MISMATCH,
"partial-run",
format!(
"{} scored {scored} of the {total} item(s) this corpus \
holds, so it is not a measurement of it. The digests \
agree because a digest names the corpus and not the \
part of it that was read. Score the whole corpus \
before quoting a number for it.",
file.display()
),
);
out.json = json;
return out;
}
let mut out = Output::ok(
json,
format!(
"{} names the corpus at {}. Both name {claimed}, and all \
{read} image(s) hash to what their records state.\nThat \
is a check on the corpus, not on the scores: nothing here \
read them, and `declarations.self_reported` says who \
produced them",
file.display(),
corpus.display()
),
);
out.code = exit::OK;
return out;
}
if let Some(total) = result.corpus.of_items {
let scored = result.metrics.n_clean + result.metrics.n_stego;
let mut out = Output::err_because(
exit::VERIFY_MISMATCH,
"partial-run",
format!(
"{} scored {scored} of the {total} item(s) this corpus \
holds, so it is not a measurement of it. Score the whole \
corpus before quoting a number for it.",
file.display()
),
);
out.json = json;
return out;
}
let mut out = Output::ok(
json,
format!(
"{} names the same records as the corpus at {}. Both name \
{claimed}.\nThe images were NOT re-read, because --shallow \
was given, so this says the two describe the same list of \
records rather than the same bytes. Nothing here read the \
scores either",
file.display(),
corpus.display()
),
);
out.code = exit::OK;
out
} else {
let mut out = Output::err(
exit::VERIFY_MISMATCH,
format!(
"{} and the corpus at {} are not about each other.\n \
the document claims {claimed}\n the corpus is {found}\n\
That number was not measured on these images.",
file.display(),
corpus.display()
),
);
out.json = json;
out
}
}
fn cmd_metrics(file: Option<&Path>, at: &[String]) -> Output {
let file = file.filter(|p| p.as_os_str() != "-");
match metrics::run(file, at) {
Ok(report) => Output::ok(report.to_json(), report.human()),
Err(e) => {
let mut out = Output::err(e.exit_code(), e.to_string());
out.json = serde_json::json!({
"ok": false,
"reason": e.reason(),
"error": e.to_string(),
});
out
}
}
}
fn cmd_completions(shell: clap_complete::Shell) -> Output {
let mut cmd = Cli::command();
let name = cmd.get_name().to_string();
let mut buf = Vec::new();
clap_complete::generate(shell, &mut cmd, name, &mut buf);
let script = match String::from_utf8(buf) {
Ok(s) => s,
Err(e) => {
return Output::err(
exit::FAILURE,
format!(
"the {shell} completion generator produced {} bytes that \
are not valid UTF-8. This is a bug in stegobench; please \
report it with this message: {e}",
e.as_bytes().len()
),
)
}
};
if script.trim().is_empty() {
return Output::err(
exit::FAILURE,
format!(
"the {shell} completion generator produced an empty script, \
which would complete nothing. Refused rather than written"
),
);
}
Output::ok(
serde_json::json!({ "shell": shell.to_string(), "script": script }),
script,
)
}
fn cmd_help(topic: Option<&str>) -> Output {
match topic {
None => {
let human = format!(
"Known topics: {}\n\nExample:\n stegobench help pairing",
help_topics::TOPICS.join(", ")
);
Output::ok(serde_json::json!({ "topics": help_topics::TOPICS }), human)
}
Some(t) => match help_topics::text(t) {
Some(text) => Output::ok(
serde_json::json!({ "topic": t, "text": text }),
text.to_string(),
),
None => Output::err(
exit::USAGE,
format!(
"no help topic {t:?}. Known: {}",
help_topics::TOPICS.join(", ")
),
),
},
}
}
fn cmd_plan(resolved: &Resolved, command: &[String]) -> Output {
if command.is_empty() {
return Output::err(
exit::USAGE,
"plan takes the command you would run, for example:\n \
stegobench plan score --corpus ./pentimento-nano --detector zsteg"
.to_string(),
);
}
let argv = std::iter::once("stegobench".to_string()).chain(command.iter().cloned());
let parsed = match Cli::try_parse_from(argv) {
Ok(c) => c,
Err(e) => {
return Output::err(
exit::USAGE,
format!("that is not a command this can plan:\n{e}"),
)
}
};
let Some(Command::Score {
corpus,
detector,
corpus_id,
limit,
timeout,
out,
records,
split,
trained_on,
..
}) = &parsed.command
else {
return Output::err(
exit::USAGE,
"only `score` can be planned today.".to_string(),
);
};
let reg = &resolved.registry;
let entries = match resolve_detectors(reg, detector) {
Ok(e) => e,
Err(o) => return o,
};
if let Some(refused) = corpus_argument_problem(Some(reg), corpus) {
return refused;
}
for (flag, path) in [("--out", out.as_deref()), ("--records", records.as_deref())] {
if let Some(why) = output_inside_corpus(corpus, flag, path) {
return Output::err_because(exit::USAGE, "output-inside-corpus", why);
}
}
if let Some(why) = unlabelled_corpus(corpus) {
return Output::err_because(exit::PREFLIGHT_REFUSED, "corpus-unlabelled", why);
}
let blocked: Vec<Option<String>> = entries
.iter()
.map(|e| stegobench_cli::unavailable_reason(e, resolved.adapter_roots()))
.collect();
if blocked.iter().all(Option::is_some) {
return Output::err_because(
exit::PREFLIGHT_REFUSED,
"nothing-available",
nothing_available(&entries, &blocked),
);
}
let mut notes: Vec<String> = Vec::new();
let prepared = match score::prepare(
corpus,
None,
*limit,
trained_on.as_deref(),
split.as_deref(),
|line: &str| notes.push(line.to_string()),
) {
Ok(p) => p,
Err(e) => return score_refusal(&e, *limit),
};
let items = prepared.items();
let (configuration, why) = plan_configuration(reg, corpus_id.as_deref(), limit.is_some());
let mut per_detector = Vec::new();
let mut lines = Vec::new();
let mut total_seconds = 0.0f64;
let mut unestimated = 0usize;
let mut unavailable = 0usize;
let mut runnable = 0usize;
for (entry, blocked) in entries.iter().zip(&blocked) {
let per_image = entry.cost.seconds_per_image;
let seconds = match blocked {
Some(_) => None,
None => per_image.map(|s| s * items as f64),
};
match blocked {
Some(_) => unavailable += 1,
None => {
runnable += 1;
match seconds {
Some(s) => total_seconds += s,
None => unestimated += 1,
}
}
}
let duration = match (blocked, seconds) {
(Some(why), _) => format!("NOT AVAILABLE: {why}"),
(None, Some(s)) => format!("at least {}", stegobench_cli::human_duration(s)),
(None, None) => "unknown: it declares no seconds_per_image".to_string(),
};
lines.push(format!("{:<16} {duration}", entry.name));
per_detector.push(serde_json::json!({
"detector": entry.name,
"available": blocked.is_none(),
"unavailable_reason": blocked,
"seconds_per_image": per_image,
"estimated_seconds": seconds,
}));
}
let records_mb = (items as f64 * 60.0 * runnable as f64) / 1_048_576.0;
let worst_case_seconds = items * timeout * runnable as u64;
let mut value = serde_json::Map::new();
value.insert("items".into(), serde_json::json!(items));
value.insert(
"detectors".into(),
serde_json::json!(entries.iter().map(|e| &e.name).collect::<Vec<_>>()),
);
value.insert("per_detector".into(), serde_json::json!(per_detector));
value.insert("runnable_detectors".into(), serde_json::json!(runnable));
value.insert(
"unavailable_detectors".into(),
serde_json::json!(unavailable),
);
value.insert("estimated_seconds".into(), serde_json::json!(total_seconds));
value.insert(
"unestimated_detectors".into(),
serde_json::json!(unestimated),
);
value.insert("records_mb".into(), serde_json::json!(records_mb));
value.insert("configuration".into(), serde_json::json!(configuration));
value.insert(
"worst_case_seconds".into(),
serde_json::json!(worst_case_seconds),
);
value.insert("rate_source".into(), serde_json::json!("declared"));
value.insert("estimate_is_a_lower_bound".into(), serde_json::json!(true));
value.insert("corpus_notes".into(), serde_json::json!(notes));
let mut human = format!("{items} item(s) to score with each of {runnable} detector(s):\n");
human.push_str(&lines.join("\n"));
let estimated = runnable - unestimated;
if estimated == 0 {
human.push_str(
"\n\nNo total: none of these declares a seconds_per_image. The \
worst case below is the only bound there is.",
);
} else {
human.push_str(&format!(
"\n\nTotal at least {}, over the {estimated} that declare \
a rate",
stegobench_cli::human_duration(total_seconds),
));
if unestimated > 0 {
human.push_str(&format!(
"\n {unestimated} declare none, so the real \
total is larger"
));
}
}
human.push_str(&format!(
"\nRecords about {records_mb:.1} MB\nWorst case \
{}\nConfiguration {configuration}: {why}",
stegobench_cli::human_duration(worst_case_seconds as f64)
));
if unavailable > 0 {
human.push_str(&format!(
"\n\n{unavailable} of {} asked for cannot run here and are left \
out of every figure above. `stegobench doctor` says what each \
one needs.",
entries.len()
));
}
if estimated > 0 {
human.push_str(
"\n\nRates are declared by each registry entry, not measured, and \
exclude per-image start-up. Read the total as a floor. \
`stegobench plan --help` says why.",
);
}
if !notes.is_empty() {
human.push_str("\n\nAbout this corpus:\n ");
human.push_str(¬es.join("\n "));
}
Output::ok(serde_json::Value::Object(value), human)
}
fn plan_configuration(
reg: &Registry,
corpus_id: Option<&str>,
limited: bool,
) -> (&'static str, String) {
let Some(id) = corpus_id else {
return (
"custom",
"no --corpus-id, so nothing to check this directory against".into(),
);
};
let Some(entry) = reg.corpora.get(id) else {
return (
"custom",
format!("no corpus with id {id:?} is registered, so `score` would refuse this"),
);
};
if entry
.integrity
.as_ref()
.and_then(|i| i.records_sha256.as_deref())
.is_none()
{
return (
"custom",
format!("{id} declares no records digest, so there is nothing to check against"),
);
}
if limited {
return (
"custom",
"--limit scores part of the corpus, and a prefix of a tier is not the tier".into(),
);
}
(
"named",
format!(
"{id} declares a records digest. `score` checks this directory \
against it and refuses if it disagrees"
),
)
}
enum Outcome {
Measured {
result: Box<Result1>,
tally: stegobench_plugin::runner::Tally,
written: Option<PathBuf>,
},
Skipped { why: String },
Failed {
why: String,
code: i32,
reason: &'static str,
},
}
fn resolve_detectors<'a>(
reg: &'a Registry,
asked: &[String],
) -> Result<Vec<&'a stegobench_core::registry::Entry>, Output> {
if asked.iter().any(|d| d == "all") {
if asked.len() > 1 {
return Err(Output::err_because(
exit::USAGE,
"all-with-others",
"`--detector all` already means every registered detector. Use \
`all` on its own, or list the ones you want."
.to_string(),
));
}
let mut all = reg.of_kind(Kind::Detector);
all.sort_by(|a, b| a.name.cmp(&b.name));
if all.is_empty() {
return Err(Output::err_because(
exit::USAGE,
"no-detectors-registered",
"no detectors are registered, so `--detector all` has nothing \
to score with. `stegobench list detectors` says where it \
looked."
.to_string(),
));
}
return Ok(all);
}
let mut seen = std::collections::BTreeSet::new();
let mut out = Vec::new();
for name in asked {
let Some(entry) = reg.entries.get(name) else {
return Err(Output::err_because(
exit::USAGE,
"unknown-detector",
format!(
"no tool named {name:?} is registered. \
`stegobench list detectors` shows what is."
),
));
};
if entry.kind == Kind::Embedder {
return Err(Output::err_because(
exit::USAGE,
"not-a-detector",
format!(
"{name} is an embedder: it hides payloads, it cannot tell \
two images apart. `stegobench list detectors` shows what \
can."
),
));
}
if seen.insert(name.clone()) {
out.push(entry);
}
}
Ok(out)
}
const SHAPE_CHECK_ENTRIES: usize = 4096;
fn corpus_path_problem(corpus: &Path) -> Option<String> {
let shown = corpus.display();
match std::fs::metadata(corpus) {
Ok(meta) if meta.is_dir() => match std::fs::read_dir(corpus) {
Ok(_) => None,
Err(e) => Some(format!(
"{shown} cannot be listed: {}.\n\
Check the permissions on it, or point at a copy you can read.",
plain(&e)
)),
},
Ok(_) => Some(format!(
"{shown} is a file, and a corpus is a directory.\n\
Point at an unpacked corpus rather than at a shard or a file \
beside one. `stegobench help scope` says what a corpus has to \
hold."
)),
Err(e) => Some(format!(
"there is no corpus at {shown}: {}.\n\
Check the path. `stegobench list corpora` names the ones this \
registry knows, and `stegobench help scope` says what a corpus \
has to hold.",
plain(&e)
)),
}
}
fn corpus_argument_problem(reg: Option<&Registry>, corpus: &Path) -> Option<Output> {
if matches!(std::fs::metadata(corpus), Err(ref e) if e.kind() == std::io::ErrorKind::NotFound) {
return Some(unresolved_corpus(reg, corpus));
}
corpus_path_problem(corpus)
.map(|why| Output::err_because(exit::PREFLIGHT_REFUSED, "corpus-unusable", why))
}
fn unresolved_corpus(reg: Option<&Registry>, corpus: &Path) -> Output {
let typed = corpus.to_string_lossy().into_owned();
if reg.is_some_and(|r| r.corpora.contains_key(typed.as_str())) {
return Output::err_because(
exit::PREFLIGHT_REFUSED,
"corpus-missing",
format!(
"there is no corpus at {typed}, and {typed} is a registered \
corpus id rather than a path: --corpus takes a directory of \
unpacked samples on this machine.\nGet the bytes first:\n \
stegobench fetch {typed} \
--tier <tier>\nthen unpack them, point --corpus at the \
directory, and name the id under --corpus-id {typed} so the \
run is checked against the digest the registry declares.\n\
`stegobench describe {typed}` says how it is obtained when it \
declares no download route."
),
);
}
let as_id = match reg {
Some(r) => {
let close = near_corpus_ids(r, &typed);
if close.is_empty() {
"none is registered under that name".to_string()
} else {
format!(
"none is registered under that name; close: {}",
close.join(", ")
)
}
}
None => "the registry could not be read, so this was not checked".to_string(),
};
Output::err_because(
exit::PREFLIGHT_REFUSED,
"corpus-missing",
format!(
"there is no corpus at {typed}, and it is not a registered corpus \
id either. Both were tried:\n as a directory nothing is there\n \
as a corpus id {as_id}\n`stegobench list corpora` names every \
registered id. --corpus wants a directory of samples, an image \
with a record beside it; `stegobench help scope` says what one \
has to hold."
),
)
}
fn per_arm_lines(arms: &[stegobench_core::result::ArmMetrics]) -> Vec<String> {
if arms.is_empty() {
return Vec::new();
}
let width = arms.iter().map(|a| a.arm.len()).max().unwrap_or(0);
let mut out = vec![format!(
"{:<16} by arm, each against the same {} clean image(s):",
"",
arms.first().map(|a| a.n_clean).unwrap_or(0)
)];
for a in arms {
out.push(format!(
"{:<18}{:<width$} AUC {:.4}{} {} stego",
"",
a.arm,
a.auc,
match a.auc_ci95 {
Some([lo, hi]) => format!(" [{lo:.4}, {hi:.4}]"),
None => String::new(),
},
a.n_stego,
width = width
));
}
out
}
fn unregistered_corpus_id(resolved: &Resolved, id: &str, next: &str) -> Output {
let reg = &resolved.registry;
let mut human = format!("no corpus with id {id:?} is registered.");
if Path::new(id).is_dir() {
human.push_str(&format!(
" It is a directory on this machine, and an id names a dataset \
rather than a path.\nThose bytes are already here, so score them \
directly:\n stegobench score --corpus {id} --detector <name>"
));
} else {
let close = near_corpus_ids(reg, id);
if !close.is_empty() {
human.push_str(&format!(" Did you mean: {}?", close.join(", ")));
}
}
human.push('\n');
human.push_str(next);
human.push('\n');
human.push_str(&resolved.source.line());
Output::err_because(exit::PREFLIGHT_REFUSED, "unregistered-corpus-id", human)
}
fn near_corpus_ids(reg: &Registry, typed: &str) -> Vec<String> {
near_names(reg.corpora.keys().map(String::as_str), typed)
}
fn near_registered_names(reg: &Registry, typed: &str) -> Vec<String> {
near_names(
reg.entries
.keys()
.chain(reg.corpora.keys())
.map(String::as_str),
typed,
)
}
fn near_names<'a>(names: impl Iterator<Item = &'a str>, typed: &str) -> Vec<String> {
let typed = typed.to_lowercase();
let mut scored: Vec<(usize, String)> = names
.filter_map(|name| {
let lower = name.to_lowercase();
let d = edit_distance(&typed, &lower);
let close = d <= 3 || lower.contains(&typed) || typed.contains(&lower);
close.then(|| (d, name.to_string()))
})
.collect();
scored.sort_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(&b.1)));
scored.into_iter().take(3).map(|(_, name)| name).collect()
}
fn resolved_for_comparison(path: &Path) -> Option<PathBuf> {
let absolute = if path.is_absolute() {
path.to_path_buf()
} else {
std::env::current_dir().ok()?.join(path)
};
let mut tail: Vec<std::ffi::OsString> = Vec::new();
let mut head = absolute.as_path();
loop {
if let Ok(base) = head.canonicalize() {
let mut resolved = base;
for component in tail.iter().rev() {
resolved.push(component);
}
return Some(resolved);
}
let name = head.file_name()?;
tail.push(name.to_os_string());
head = head.parent()?;
}
}
fn output_inside_corpus(corpus: &Path, flag: &str, out: Option<&Path>) -> Option<String> {
let out = out?;
let corpus_real = resolved_for_comparison(corpus)?;
let out_real = resolved_for_comparison(out)?;
if !out_real.starts_with(&corpus_real) {
return None;
}
let what = if out_real == corpus_real {
format!("the corpus directory {} itself", corpus.display())
} else {
format!("inside the corpus at {}", corpus.display())
};
let mut why = format!(
"{flag} {} is {what}.\nWriting there adds a file to the corpus, so the \
next run reads a different set of bytes, computes a different corpus \
digest, and reports that the corpus has changed when nothing about \
the images did. The number measured now could not be reproduced \
afterwards.\nPut it anywhere outside the corpus.",
out.display()
);
if let Some(parent) = corpus_real.parent() {
let name = out_real
.file_name()
.unwrap_or_else(|| std::ffi::OsStr::new("results"));
why.push_str(&format!(
" Beside it works: {flag} {}",
parent.join(name).display()
));
}
Some(why)
}
fn unlabelled_corpus(corpus: &Path) -> Option<String> {
let mut images = 0usize;
let mut seen = 0usize;
let mut stack = vec![(corpus.to_path_buf(), 0usize)];
while let Some((dir, depth)) = stack.pop() {
let read = std::fs::read_dir(&dir).ok()?;
for item in read.flatten() {
seen += 1;
if seen > SHAPE_CHECK_ENTRIES {
return None;
}
let path = item.path();
let Ok(meta) = std::fs::metadata(&path) else {
return None;
};
if meta.is_dir() {
if depth < 2 {
stack.push((path, depth + 1));
}
continue;
}
match path
.extension()
.map(|e| e.to_string_lossy().to_lowercase())
.as_deref()
{
Some("json" | "jsonl") => return None,
Some(ext) if stegobench_cli::LOOKS_LIKE_AN_IMAGE.contains(&ext) => images += 1,
_ => {}
}
}
}
if images == 0 {
return None;
}
Some(format!(
"{} holds {images} image(s) and no records saying which of them hides \
anything, so there is nothing to be right or wrong about.\n\n\
`score` measures a detector, which needs the answers in advance. \
Asking the detectors what they make of these files is `examine`, and \
what comes back is an answer rather than a measurement.\n\
`stegobench examine {} --detector <name>`\n\
`stegobench help scope` the difference between the two\n\
`stegobench list detectors` what is registered here\n\
`stegobench help pairing` what a corpus has to carry first",
corpus.display(),
corpus.display()
))
}
struct ScoreRequest<'a> {
corpus: &'a Path,
detectors: &'a [String],
corpus_id: Option<&'a str>,
trained_on: Option<&'a str>,
split: Option<&'a str>,
records: Option<&'a Path>,
out: Option<&'a Path>,
timeout: u64,
jobs: usize,
keep_raw: bool,
seed: Option<u64>,
fixtures: Option<&'a Path>,
limit: Option<u64>,
}
fn cmd_score(resolved: &Resolved, req: ScoreRequest<'_>) -> Output {
let ScoreRequest {
corpus,
detectors,
corpus_id,
trained_on,
split,
records,
out,
timeout,
jobs,
keep_raw,
seed,
fixtures,
limit,
} = req;
let reg = &resolved.registry;
let entries = match resolve_detectors(reg, detectors) {
Ok(e) => e,
Err(o) => return o,
};
let many = entries.len() > 1;
if let Some(refused) = corpus_argument_problem(Some(reg), corpus) {
return refused;
}
for (flag, path) in [("--out", out), ("--records", records)] {
if let Some(why) = output_inside_corpus(corpus, flag, path) {
return Output::err_because(exit::USAGE, "output-inside-corpus", why);
}
}
if let Some(why) = unlabelled_corpus(corpus) {
return Output::err_because(exit::PREFLIGHT_REFUSED, "corpus-unlabelled", why);
}
let registered = match corpus_id {
None => None,
Some(id) => match reg.corpora.get(id) {
Some(entry) => Some(entry),
None => {
return unregistered_corpus_id(
resolved,
id,
"`stegobench list corpora` shows what is. Leaving \
--corpus-id out scores the directory anyway, and marks \
the result `custom`.",
)
}
},
};
if many {
for (flag, path) in [("--out", out), ("--records", records)] {
if path.is_some_and(|p| p.is_file()) {
return Output::err_because(
exit::USAGE,
"output-not-a-directory",
format!(
"{} is a file. Scoring {} detectors writes one {} \
each, so {flag} has to name a directory.",
path.expect("checked").display(),
entries.len(),
if flag == "--out" {
"result document"
} else {
"records file"
}
),
);
}
}
}
let say = |line: &str| eprintln!(" {line}");
let blocked: Vec<Option<String>> = entries
.iter()
.map(|e| stegobench_cli::unavailable_reason(e, resolved.adapter_roots()))
.collect();
let mut runnable = Vec::new();
let mut outcomes: Vec<(String, Outcome)> = Vec::new();
for (entry, why) in entries.iter().zip(&blocked) {
match why {
Some(why) => outcomes.push((entry.name.clone(), Outcome::Skipped { why: why.clone() })),
None => runnable.push(*entry),
}
}
if runnable.is_empty() {
return Output::err_because(
exit::PREFLIGHT_REFUSED,
"nothing-available",
format!(
"nothing was measured: {}",
nothing_available(&entries, &blocked)
),
);
}
for (name, outcome) in &outcomes {
if let Outcome::Skipped { why } = outcome {
say(&format!("SKIPPING {name}: {why}"));
}
}
let out_dir = match resolve_out(corpus, out, many) {
Ok(d) => d,
Err(o) => return o,
};
say(&format!(
"establishing what the corpus is, once, for {} detector(s)",
runnable.len()
));
let prepared = match score::prepare(corpus, registered, limit, trained_on, split, say) {
Ok(p) => p,
Err(e) => return score_refusal(&e, limit),
};
say(&format!(
"corpus established in {:.1}s: {} item(s) to score with each detector",
prepared.preflight_seconds(),
prepared.items()
));
for (i, entry) in runnable.iter().enumerate() {
let records_path = records_for(corpus, records, &out_dir, &entry.name, many, prepared.side);
say(&format!(
"scoring with {} ({} of {} that can run here)",
entry.name,
i + 1,
runnable.len()
));
let outcome = run_one(
entry,
&prepared,
&records_path,
timeout,
jobs,
keep_raw,
seed,
fixtures,
out,
out_dir.as_deref(),
resolved.adapter_roots(),
say,
);
outcomes.push((entry.name.clone(), outcome));
}
summarise(
&outcomes,
&out_dir,
many,
offer_for_run(&resolved.registry, corpus, corpus_id).as_ref(),
)
}
fn offer_for_run(reg: &Registry, corpus: &Path, corpus_id: Option<&str>) -> Option<fetch::Offer> {
let named_it = |name: &str| name == fetch::OFFERED_CORPUS;
if corpus_id.is_some_and(named_it) {
return None;
}
if corpus
.file_name()
.and_then(|n| n.to_str())
.is_some_and(named_it)
{
return None;
}
fetch::offer(reg)
}
fn nothing_available(
entries: &[&stegobench_core::registry::Entry],
blocked: &[Option<String>],
) -> String {
let why = entries
.iter()
.zip(blocked)
.filter_map(|(e, b)| b.as_ref().map(|w| format!("{} {w}", e.name)))
.collect::<Vec<_>>()
.join("\n ");
let subject = if entries.len() == 1 {
"the one detector you asked for is not available here".to_string()
} else {
format!(
"not one of the {} detectors you asked for is available here",
entries.len()
)
};
format!(
"{subject}.\n \
{why}\n`stegobench doctor` checks every registered tool at once and \
says what each one needs."
)
}
fn resolve_out(corpus: &Path, out: Option<&Path>, many: bool) -> Result<Option<PathBuf>, Output> {
if let Some(dir) = out.filter(|p| p.is_dir()) {
return Ok(Some(dir.to_path_buf()));
}
let dir = match out {
Some(p) if !many => {
if let Some(parent) = p.parent().filter(|d| !d.as_os_str().is_empty()) {
if !parent.is_dir() {
return Err(Output::err_because(
exit::PREFLIGHT_REFUSED,
"out-parent-missing",
format!(
"there is no directory at {} to write {} into. \
Create it, or name a path whose parent exists. \
Checked now rather than after the scoring, which \
is where it used to be found",
parent.display(),
p.display()
),
));
}
}
return Ok(None);
}
Some(p) => p.to_path_buf(),
None => {
let mut name = corpus.file_name().unwrap_or_default().to_os_string();
name.push(".results");
corpus.with_file_name(name)
}
};
if let Err(e) = std::fs::create_dir_all(&dir) {
return Err(Output::err_because(
exit::FAILURE,
"results-unwritable",
format!("could not create {} for the results: {e}", dir.display()),
));
}
Ok(Some(dir))
}
fn records_for(
corpus: &Path,
records: Option<&Path>,
out_dir: &Option<PathBuf>,
detector: &str,
many: bool,
split: Option<score::Side>,
) -> PathBuf {
let file = match split {
Some(side) => format!("{detector}.{}.records.jsonl", side.as_str()),
None => format!("{detector}.records.jsonl"),
};
match (records, many, out_dir) {
(Some(p), false, _) => p.to_path_buf(),
(Some(p), true, _) => p.join(file),
(None, _, Some(dir)) => dir.join(file),
(None, true, None) => corpus.with_file_name(file),
(None, false, None) => {
let mut name = corpus.file_name().unwrap_or_default().to_os_string();
name.push(format!(".{file}"));
corpus.with_file_name(name)
}
}
}
#[allow(clippy::too_many_arguments)]
fn run_one<P>(
entry: &stegobench_core::registry::Entry,
prepared: &score::Prepared,
records: &Path,
timeout: u64,
jobs: usize,
keep_raw: bool,
seed: Option<u64>,
fixtures: Option<&Path>,
out: Option<&Path>,
out_dir: Option<&Path>,
adapter_roots: &[PathBuf],
say: P,
) -> Outcome
where
P: Fn(&str),
{
if let Some(parent) = records.parent() {
if !parent.as_os_str().is_empty() && !parent.exists() {
if let Err(e) = std::fs::create_dir_all(parent) {
return Outcome::Failed {
why: format!("could not create {} for the records: {e}", parent.display()),
code: exit::FAILURE,
reason: "records-unwritable",
};
}
}
}
let (result, tally) = match score::score_one(
entry,
prepared,
records,
score::How {
timeout: std::time::Duration::from_secs(timeout),
jobs,
keep_raw,
seed,
fixtures,
adapter_roots,
},
|line: &str| say(line),
) {
Ok(pair) => pair,
Err(e) => {
return Outcome::Failed {
why: e.to_string(),
code: e.exit_code(),
reason: e.reason(),
}
}
};
if let Err(problems) = result.validate() {
return Outcome::Failed {
why: format!(
"the run finished but produced a result that does not \
validate, which is a bug in this harness rather than in the \
detector:\n {}",
problems.join("\n ")
),
code: exit::SCHEMA_INVALID,
reason: "result-invalid",
};
}
let body = match serde_json::to_string_pretty(&result) {
Ok(b) => b,
Err(e) => {
return Outcome::Failed {
why: format!("could not write the result: {e}"),
code: exit::FAILURE,
reason: "result-unwritable",
}
}
};
let destination = match (out_dir, out) {
(Some(dir), _) => Some(dir.join(format!("{}.json", entry.name))),
(None, Some(path)) => Some(path.to_path_buf()),
_ => None,
};
if let Some(path) = &destination {
if let Err(e) = std::fs::write(path, format!("{body}\n")) {
return Outcome::Failed {
why: format!("could not write the result to {}: {e}", path.display()),
code: exit::FAILURE,
reason: "result-unwritable",
};
}
}
Outcome::Measured {
result: Box::new(result),
tally,
written: destination,
}
}
fn summarise(
outcomes: &[(String, Outcome)],
out_dir: &Option<PathBuf>,
many: bool,
offer: Option<&fetch::Offer>,
) -> Output {
let measured = outcomes
.iter()
.filter(|(_, o)| matches!(o, Outcome::Measured { .. }))
.count();
let skipped = outcomes
.iter()
.filter(|(_, o)| matches!(o, Outcome::Skipped { .. }))
.count();
let failed = outcomes
.iter()
.filter(|(_, o)| matches!(o, Outcome::Failed { .. }))
.count();
let mut lines = Vec::new();
for (name, outcome) in outcomes {
match outcome {
Outcome::Measured {
result,
tally,
written,
} => {
lines.push(format!(
"{name:<16} AUC {:.4}{} {} clean / {} stego / {} unanswered{}{}",
result.metrics.auc,
match result.metrics.auc_ci95 {
Some([lo, hi]) => format!(" [{lo:.4}, {hi:.4}]"),
None => String::new(),
},
result.metrics.n_clean,
result.metrics.n_stego,
result.metrics.n_error,
if tally.resumed > 0 {
format!(" {} resumed", tally.resumed)
} else {
String::new()
},
match written {
Some(p) => format!(" {}", p.display()),
None => String::new(),
}
));
lines.extend(per_arm_lines(&result.metrics.per_arm));
}
Outcome::Skipped { why } => {
lines.push(format!("{name:<16} NOT MEASURED, skipped: {why}"))
}
Outcome::Failed { why, .. } => {
lines.push(format!("{name:<16} NOT MEASURED, failed: {why}"))
}
}
}
if measured > 0 {
lines.push("AUC and the bracketed pair: `stegobench help results`.".into());
}
lines.push(String::new());
lines.push(format!(
"{measured} of {} detector(s) measured.",
outcomes.len()
));
if skipped > 0 || failed > 0 {
lines.push(format!(
"{} produced NO number ({skipped} skipped, {failed} failed). A \
report over these documents covers only the {measured} that ran.",
skipped + failed
));
}
if let Some(dir) = out_dir {
let written: std::collections::BTreeSet<&Path> = outcomes
.iter()
.filter_map(|(_, o)| match o {
Outcome::Measured { written, .. } => written.as_deref(),
_ => None,
})
.collect();
let mut stale: Vec<String> = Vec::new();
match std::fs::read_dir(dir) {
Ok(entries) => {
for entry in entries.flatten() {
let path = entry.path();
if path.extension().is_some_and(|e| e == "json") && !written.contains(&*path) {
stale.push(path.display().to_string());
}
}
stale.sort();
}
Err(e) => lines.push(format!(
"could not list {} to check for older documents: {e}. A report \
over it may cover more than this run did.",
dir.display()
)),
}
if !stale.is_empty() {
lines.push(format!(
"WARNING: {} document(s) here are from an earlier run, not \
this one: {}. `stegobench report {}` will include them \
beside today's numbers. Move them aside if this run is meant \
to be the whole table.",
stale.len(),
stale.join(", "),
dir.display()
));
}
}
if !many && out_dir.is_none() {
if let Some(named) = outcomes.iter().find_map(|(_, o)| match o {
Outcome::Measured { written, .. } => written.as_deref(),
_ => None,
}) {
lines.push(format!(
"{} is the result document itself, because one detector was \
asked for. More than one would have made it a directory with \
a document per detector inside.",
named.display()
));
}
}
let mut next: Vec<String> = Vec::new();
if many {
if let Some(dir) = out_dir {
next.push(format!(
"stegobench report {} --format markdown",
dir.display()
));
}
}
let largest = outcomes
.iter()
.filter_map(|(_, o)| match o {
Outcome::Measured { result, .. } => {
Some(result.metrics.n_clean + result.metrics.n_stego)
}
_ => None,
})
.max()
.unwrap_or(0);
if let Some(offer) = offer {
if largest > 0 && largest < fetch::SMALL_RUN_IMAGES {
lines.push(format!(
"{largest} images is a demonstration, not a measurement."
));
next.push(offer.command().to_string());
}
}
for (label, command) in ["Next: ", "Then: "].iter().zip(&next) {
lines.push(format!("{label}{command}"));
}
let code = if failed > 0 {
exit::PLUGIN_FAILED
} else if skipped > 0 {
exit::PREFLIGHT_REFUSED
} else {
exit::OK
};
let json = serde_json::json!({
"ok": code == exit::OK,
"requested": outcomes.len(),
"measured": measured,
"skipped": skipped,
"failed": failed,
"results_dir": out_dir.as_ref().map(|d| d.display().to_string()),
"detectors": outcomes.iter().map(|(name, o)| match o {
Outcome::Measured { result, tally, written } => serde_json::json!({
"name": name,
"status": "measured",
"written": written.as_ref().map(|p| p.display().to_string()),
"scored": tally.scored,
"resumed": tally.resumed,
"result": serde_json::to_value(result.as_ref())
.unwrap_or(serde_json::Value::Null),
}),
Outcome::Skipped { why } => serde_json::json!({
"name": name, "status": "skipped", "reason": why,
}),
Outcome::Failed { why, code, .. } => serde_json::json!({
"name": name, "status": "failed", "reason": why, "exit_code": code,
}),
}).collect::<Vec<_>>(),
});
if !many {
match outcomes.first() {
Some((_, Outcome::Measured { result, .. })) => {
let mut output = Output::ok(
serde_json::to_value(result.as_ref()).unwrap_or(serde_json::Value::Null),
lines.join("\n"),
);
output.code = code;
return output;
}
Some((_, Outcome::Failed { why, code, reason })) => {
return Output::err_because(*code, reason, why.clone())
}
_ => {}
}
}
let mut output = Output::ok(json, lines.join("\n"));
output.code = code;
output
}
fn cmd_report(paths: &[PathBuf], format: ReportFormat, out: Option<&Path>) -> Output {
let report = match report::build(paths) {
Ok(r) => r,
Err(e) => return Output::err(e.exit_code(), e.to_string()),
};
let body = report::render(&report, format);
let json = report::to_json(&report);
let human = match out {
None => body,
Some(path) => {
if let Err(e) = report::write_atomically(path, &body) {
return Output::err(e.exit_code(), e.to_string());
}
format!(
"{} result(s) in {} table(s) written to {}. {}",
report.read,
report.groups.len(),
path.display(),
if report.skipped.is_empty() {
"Every file found was readable.".to_string()
} else {
format!(
"{} file(s) could not be read and are named at the top \
of it, so the tables are incomplete.",
report.skipped.len()
)
}
)
}
};
let mut output = Output::ok(json, human);
output.code = report.exit_code();
output.payload_on_stdout = out.is_none();
output
}
fn fetch_dest(named: Option<&Path>, from_env: Option<&str>) -> PathBuf {
named
.map(Path::to_path_buf)
.or_else(|| from_env.filter(|v| !v.is_empty()).map(PathBuf::from))
.unwrap_or_else(fetch::default_dest)
}
include!(concat!(env!("OUT_DIR"), "/embedded_starter.rs"));
fn write_embedded_starter(dest: &Path) -> Result<usize, String> {
if let Ok(mut entries) = std::fs::read_dir(dest) {
if entries.next().is_some() {
return Err(format!(
"{} already has something in it. Name an empty directory \
with --dest, or move that one aside: writing over half of a \
corpus leaves records describing images from two different \
copies, and nothing downstream can tell.",
dest.display()
));
}
}
for (rel, bytes) in EMBEDDED_STARTER {
let path = dest.join(rel);
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent)
.map_err(|e| format!("could not make {}: {e}", parent.display()))?;
}
std::fs::write(&path, bytes)
.map_err(|e| format!("could not write {}: {e}", path.display()))?;
}
Ok(EMBEDDED_STARTER.len())
}
fn cmd_fetch(
resolved: &Resolved,
id: &str,
tier: &str,
dest: Option<&Path>,
max_bytes: Option<u64>,
budget_minutes: u64,
) -> Output {
let Some(entry) = resolved.registry.corpora.get(id) else {
return unregistered_corpus_id(
resolved,
id,
"`stegobench list corpora` shows what is registered.",
);
};
if entry.id == STARTER_ID && !EMBEDDED_STARTER.is_empty() {
let dest = dest
.map(Path::to_path_buf)
.unwrap_or_else(|| PathBuf::from(STARTER_ID));
return match write_embedded_starter(&dest) {
Err(e) => Output::err(exit::FAILURE, e),
Ok(n) => Output::ok(
serde_json::json!({
"corpus": entry.id,
"dest": dest.display().to_string(),
"files": n,
"source": "compiled into this binary",
}),
format!(
"wrote {n} file(s) to {}, from the copy compiled into \
this binary rather than from a download.\n\nScore it \
with:\n stegobench score --corpus {} --detector \
<name>\n\nEighteen images is a demonstration that the \
machinery works, not a measurement of anything.",
dest.display(),
dest.display()
),
),
};
}
if let Some(refused) = fetch::refusal(entry) {
return Output::err(refused.exit_code(), refused.to_string());
}
if entry.route(tier).is_none() {
let e = fetch::Error::Failed(stegobench_core::fetch::FetchError::UnknownTier {
id: entry.id.clone(),
tier: tier.to_string(),
known: entry.download.iter().map(|r| r.tier.clone()).collect(),
});
return Output::err(e.exit_code(), e.to_string());
}
let limits = fetch::limits(max_bytes, budget_minutes);
let dest = fetch_dest(
dest,
registry::non_empty("STEGOBENCH_CORPUS_DIR").as_deref(),
);
if let Err(e) = std::fs::create_dir_all(&dest) {
return Output::err(
exit::FAILURE,
format!(
"cannot use {} to keep the download: {e}. Name a writable \
directory with --dest.",
dest.display()
),
);
}
let transport = match fetch::Curl::find(limits) {
Ok(t) => t,
Err(e) => return Output::err(e.exit_code(), e.to_string()),
};
let store = stegobench_core::fetch::FileStore::new(&dest);
let mut stderr = std::io::stderr();
let interactive = stderr.is_terminal();
let mut renderer = fetch::Renderer::new(&mut stderr, interactive);
let outcome = fetch::run(entry, tier, &transport, &store, limits, &mut renderer);
let progress_error = renderer.write_error().map(|e| e.to_string());
drop(renderer);
match outcome {
Ok(fetched) => {
let (json, human) = fetch::describe_success(entry, tier, &fetched, progress_error);
Output::ok(json, human)
}
Err(e) => Output::err(e.exit_code(), e.to_string()),
}
}
fn cmd_orientation(offer: Option<fetch::Offer>) -> Output {
let mut human = "stegobench measures how good a steganography detector is, by \
running it over images whose answers are already known.\n\
It can also run those detectors over images of your own, which is an \
answer rather than a measurement (`stegobench help scope`).\n\n \
stegobench list detectors what this installation can run\n \
stegobench examine <image> what the detectors say about a file\n \
stegobench doctor what is installed, and what it needs\n \
stegobench help the reasoning, one topic at a time\n \
stegobench --help every command and flag"
.to_string();
let mut next = vec![
"stegobench list detectors".to_string(),
"stegobench examine <image> --detector <name>".to_string(),
"stegobench doctor".to_string(),
"stegobench help scope".to_string(),
];
if let Some(offer) = &offer {
human.push_str(&format!("\n\n {}", offer.line()));
next.push(offer.run.clone());
}
let mut out = Output::ok(
serde_json::json!({
"ok": false,
"error": "no command given",
"next": next,
}),
human,
);
out.code = exit::USAGE;
out.payload_on_stdout = true;
out
}
const NO_EXTRACTION: &[&str] = &["decode", "extract", "reveal", "unhide"];
fn no_extraction(typed: &str) -> bool {
let typed = typed.to_lowercase();
NO_EXTRACTION.contains(&typed.as_str())
}
const MEANT_EXAMINE: &[&str] = &[
"analyse", "analyze", "check", "detect", "examine", "find", "inspect", "run", "scan", "search",
"test",
];
fn meant_examine(typed: &str) -> bool {
let typed = typed.to_lowercase();
MEANT_EXAMINE.contains(&typed.as_str())
}
const RIGHT_IDEA: &[&str] = &[
"assess",
"bench",
"benchmark",
"benchmarks",
"compare",
"eval",
"evaluate",
"grade",
"measure",
"rank",
];
fn right_idea(typed: &str) -> bool {
let typed = typed.to_lowercase();
RIGHT_IDEA.contains(&typed.as_str())
}
fn cmd_right_idea(word: &str) -> Output {
Output::err(
exit::USAGE,
format!(
"there is no `stegobench {word}`. Measuring a detector against a \
labelled corpus is `score`.\n\n\
`stegobench plan score --corpus <dir> --detector <name>` what \
it would cost\n\
`stegobench score --corpus <dir> --detector <name>` the \
measurement\n\
`stegobench list detectors` what \
is registered here"
),
)
}
fn cmd_no_extraction(word: &str) -> Output {
Output::err(
exit::USAGE,
format!(
"there is no `stegobench {word}`. Stegobench asks detectors \
questions about images; it does not pull a hidden payload back \
out of one.\n\n\
`stegobench examine <image> --detector <name>` what the \
detectors say about it\n\
`stegobench list detectors` what is \
registered here\n\
`stegobench help scope` what an answer \
about one image is worth"
),
)
}
fn cmd_meant_examine(word: &str) -> Output {
Output::err(
exit::USAGE,
format!(
"there is no `stegobench {word}`. Asking detectors about images \
of your own is `examine`.\n\n\
`stegobench examine <image>... --detector <name>` ask one, or \
several at once\n\
`stegobench list detectors` what is \
registered here\n\
`stegobench help scope` why that \
answer is not a measurement"
),
)
}
fn cmd_examine(
resolved: &Resolved,
images: &[PathBuf],
detectors: &[String],
timeout: u64,
jobs: usize,
raw: Option<&Path>,
) -> Output {
let say = |line: &str| eprintln!(" {line}");
let found = match examine::run(
&resolved.registry,
examine::Request {
detectors,
images,
timeout: std::time::Duration::from_secs(timeout),
jobs,
adapter_roots: resolved.adapter_roots(),
raw,
},
say,
) {
Ok(found) => found,
Err(e) => {
let mut human = e.to_string();
if matches!(e, examine::ExamineError::NoDetectorNamed) {
if let Some(way) = stegobench_cli::smallest_way_in(&resolved.registry) {
human.push_str(&format!("\n\n{way}"));
}
}
return Output::err_because(e.exit_code(), e.reason(), human);
}
};
let mut human = found.table();
for note in found.footnotes() {
human.push('\n');
human.push_str(¬e);
human.push('\n');
}
if !found.answered_anything() {
if let Some(way) = stegobench_cli::smallest_way_in(&resolved.registry) {
human.push('\n');
human.push_str(&way);
human.push('\n');
}
}
let mut out = Output::ok(found.to_json(), human);
out.code = found.exit_code();
out.payload_on_stdout = true;
out
}
fn run(cli: &Cli) -> Output {
let Some(command) = &cli.command else {
let offer = registry::resolve(cli.registry.as_deref())
.ok()
.and_then(|r| fetch::offer(&r.registry));
return cmd_orientation(offer);
};
match command {
Command::Schema { name } => cmd_schema(name),
Command::Validate { file } => cmd_validate(file),
Command::Verify {
file,
corpus,
records,
shallow,
} => {
let reg = registry::resolve(cli.registry.as_deref()).ok();
match corpus_argument_problem(reg.as_ref().map(|r| &r.registry), corpus) {
Some(refused) => refused,
None => cmd_verify(file, corpus, records.as_deref(), *shallow),
}
}
Command::List { kind } => with_registry(cli, |r| cmd_list(r, kind)),
Command::Describe { name, toml } => with_registry(cli, |r| cmd_describe(r, name, *toml)),
Command::Embed {
embedder,
cover,
payload,
out,
passphrase,
no_verify,
} => with_registry(cli, |r| {
cmd_embed(
r,
embedder,
cover,
payload,
out,
passphrase.as_deref(),
*no_verify,
)
}),
Command::Plan { command } => with_registry(cli, |r| cmd_plan(r, command)),
Command::Doctor {
fixtures,
no_selftest,
strict,
registry_reach,
} => with_registry(cli, |r| {
if *no_selftest {
return cmd_doctor(r, None, *strict, *registry_reach);
}
match fixtures::resolve(fixtures.as_deref()) {
Ok(found) => cmd_doctor(r, Some(&found), *strict, *registry_reach),
Err(e) => Output::err(exit::PREFLIGHT_REFUSED, e.to_string()),
}
}),
Command::Examine {
images,
detectors,
timeout,
jobs,
raw,
} => with_registry(cli, |r| {
cmd_examine(r, images, detectors, *timeout, *jobs, raw.as_deref())
}),
Command::Score {
corpus,
detector,
corpus_id,
trained_on,
split,
records,
out,
timeout,
jobs,
keep_raw,
seed,
limit,
} => with_registry(cli, |r| {
let found = fixtures::resolve(None).ok();
cmd_score(
r,
ScoreRequest {
corpus,
detectors: detector.as_slice(),
corpus_id: corpus_id.as_deref(),
trained_on: trained_on.as_deref(),
split: split.as_deref(),
records: records.as_deref(),
out: out.as_deref(),
timeout: *timeout,
jobs: *jobs,
keep_raw: *keep_raw,
seed: *seed,
fixtures: found.as_ref().map(|f| f.dir()),
limit: *limit,
},
)
}),
Command::Fetch {
corpus,
tier,
dest,
max_bytes,
budget_minutes,
} => with_registry(cli, |r| {
cmd_fetch(
r,
corpus,
tier,
dest.as_deref(),
*max_bytes,
*budget_minutes,
)
}),
Command::Metrics { file, at } => cmd_metrics(file.as_deref(), at),
Command::Report { paths, format, out } => cmd_report(paths, *format, out.as_deref()),
Command::Completions { shell } => cmd_completions(*shell),
Command::Help { topic } => cmd_help(topic.as_deref()),
}
}
fn parse_or_explain() -> Cli {
match Cli::try_parse() {
Ok(cli) => cli,
Err(e) if e.kind() == clap::error::ErrorKind::InvalidSubcommand => {
let typed = e
.get(clap::error::ContextKind::InvalidSubcommand)
.map(|v| v.to_string());
let Some(typed) = typed else {
e.exit();
};
if no_extraction(&typed) {
eprintln!("{}", cmd_no_extraction(&typed).human);
std::process::exit(exit::USAGE);
}
if meant_examine(&typed) {
eprintln!("{}", cmd_meant_examine(&typed).human);
std::process::exit(exit::USAGE);
}
if right_idea(&typed) {
eprintln!("{}", cmd_right_idea(&typed).human);
std::process::exit(exit::USAGE);
}
let nearest = nearest_command(&typed);
eprintln!("there is no `stegobench {typed}`.");
if let Some(nearest) = nearest {
eprintln!("The closest command is `{nearest}`.");
}
eprintln!(
"`stegobench` on its own says what the tool is for, and \
`stegobench --help` lists every command."
);
std::process::exit(exit::USAGE);
}
Err(ref e) if plan_without_its_verb(e).is_some() => {
let rest = plan_without_its_verb(e).expect("matched");
eprintln!(
"`plan` takes the command you would run, whole, so it has to \
start with that command's own name."
);
eprintln!("\nYou typed:\n stegobench plan {rest}");
eprintln!("You probably meant:\n stegobench plan score {rest}");
eprintln!("\n`score` is the only command that can be planned today.");
std::process::exit(exit::USAGE);
}
Err(e) if looks_like_a_folder_of_photos(&e) => {
eprintln!(
"a path was given to `score` without a flag, and this does \
not take a folder of images that way."
);
eprintln!(
"\nIf you have a labelled corpus, name it with --corpus, and \
a detector with --detector."
);
eprintln!(
"If you meant your own pictures, this is not the tool: it \
measures how good a detector is, using images whose answers \
are already known, and it cannot tell you whether something \
is hidden in yours."
);
eprintln!("\n`stegobench help scope` is the whole of why.");
std::process::exit(exit::USAGE);
}
Err(ref e) if misleading_suggestion(e).is_some() => {
let typed = misleading_suggestion(e).expect("matched");
let command = std::env::args().nth(1).unwrap_or_default();
eprintln!("there is no `{typed}` here.");
if command.is_empty() || command.starts_with('-') {
eprintln!("`stegobench --help` lists every flag.");
} else {
eprintln!("`stegobench {command} --help` lists every flag it takes.");
}
std::process::exit(exit::USAGE);
}
Err(e) => e.exit(),
}
}
fn misleading_suggestion(e: &clap::Error) -> Option<String> {
if e.kind() != clap::error::ErrorKind::UnknownArgument {
return None;
}
let typed = e.get(clap::error::ContextKind::InvalidArg)?.to_string();
let suggested = e.get(clap::error::ContextKind::SuggestedArg)?.to_string();
let bare = |s: &str| s.trim_matches(|c: char| !c.is_alphanumeric()).to_string();
let typed_bare = bare(&typed);
let any_close = suggested
.split(|c: char| c == ',' || c.is_whitespace())
.map(bare)
.filter(|s| !s.is_empty())
.any(|s| near_enough(&typed_bare, &s));
if any_close {
return None;
}
Some(typed)
}
fn near_enough(typed: &str, candidate: &str) -> bool {
let d = edit_distance(typed, candidate);
d <= 2 && d * 2 <= candidate.len().max(typed.len())
}
fn plan_without_its_verb(e: &clap::Error) -> Option<String> {
let argv: Vec<String> = std::env::args().skip(1).collect();
plan_verb_hint(e.kind(), &argv)
}
fn plan_verb_hint(kind: clap::error::ErrorKind, argv: &[String]) -> Option<String> {
if kind != clap::error::ErrorKind::UnknownArgument {
return None;
}
if argv.first().map(String::as_str) != Some("plan") {
return None;
}
let rest = &argv[1..];
if !rest.first().is_some_and(|a| a.starts_with('-')) {
return None;
}
Some(
rest.iter()
.map(|w| shell_word(w))
.collect::<Vec<_>>()
.join(" "),
)
}
fn looks_like_a_folder_of_photos(e: &clap::Error) -> bool {
let argv: Vec<String> = std::env::args().skip(1).collect();
let Some(first) = argv.first() else {
return false;
};
if first != "score" && first != "plan" {
return false;
}
let kind = e.kind();
if kind != clap::error::ErrorKind::MissingRequiredArgument
&& kind != clap::error::ErrorKind::UnknownArgument
{
return false;
}
let mut expecting_value = false;
for arg in argv.iter().skip(1) {
if expecting_value {
expecting_value = false;
continue;
}
if arg.starts_with('-') {
expecting_value = !arg.contains('=');
continue;
}
return true;
}
false
}
fn nearest_command(typed: &str) -> Option<String> {
let mut cmd = Cli::command();
cmd.build();
let typed = typed.to_lowercase();
cmd.get_subcommands()
.filter(|s| !s.is_hide_set())
.map(|s| s.get_name().to_string())
.map(|name| {
let d = edit_distance(&typed, &name);
(d, name)
})
.filter(|(_, name)| near_enough(&typed, name))
.min_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(&b.1)))
.map(|(_, name)| name)
}
fn edit_distance(a: &str, b: &str) -> usize {
let a: Vec<char> = a.chars().collect();
let b: Vec<char> = b.chars().collect();
let mut prev: Vec<usize> = (0..=b.len()).collect();
let mut cur = vec![0usize; b.len() + 1];
for (i, ca) in a.iter().enumerate() {
cur[0] = i + 1;
for (j, cb) in b.iter().enumerate() {
let cost = usize::from(ca != cb);
cur[j + 1] = (prev[j] + cost).min(prev[j + 1] + 1).min(cur[j] + 1);
}
std::mem::swap(&mut prev, &mut cur);
}
prev[b.len()]
}
fn human_is_content(command: &Command) -> bool {
match command {
Command::List { .. }
| Command::Describe { .. }
| Command::Help { .. }
| Command::Plan { .. }
| Command::Metrics { .. }
| Command::Completions { .. } => true,
Command::Schema { .. }
| Command::Validate { .. }
| Command::Verify { .. }
| Command::Embed { .. }
| Command::Doctor { .. }
| Command::Score { .. }
| Command::Fetch { .. }
| Command::Report { .. }
| Command::Examine { .. } => false,
}
}
fn main() {
let cli = parse_or_explain();
let out = run(&cli);
if cli.json {
let mut stdout = std::io::stdout().lock();
let _ = serde_json::to_writer_pretty(&mut stdout, &out.json);
let _ = writeln!(stdout);
} else if out.payload_on_stdout {
let mut stdout = std::io::stdout().lock();
let _ = writeln!(stdout, "{}", out.human);
} else if out.code == exit::OK {
if matches!(cli.command, Some(Command::Schema { .. })) {
let mut stdout = std::io::stdout().lock();
let _ = serde_json::to_writer_pretty(&mut stdout, &out.json);
let _ = writeln!(stdout);
} else if cli.command.as_ref().is_some_and(human_is_content) {
let mut stdout = std::io::stdout().lock();
let _ = writeln!(stdout, "{}", out.human);
} else {
eprintln!("{}", out.human);
}
} else {
eprintln!("{}", out.human);
}
std::process::exit(out.code);
}
#[cfg(test)]
mod tests {
use super::*;
use std::path::PathBuf;
#[test]
fn a_run_that_names_no_destination_still_keeps_its_document() {
let tmp = tempfile::tempdir().expect("a temporary directory");
let corpus = tmp.path().join("starter");
let expected = tmp.path().join("starter.results");
for many in [false, true] {
assert_eq!(
resolve_out(&corpus, None, many).expect("a destination"),
Some(expected.clone()),
"a run of {} detector(s) with no --out lost its document",
if many { "several" } else { "one" }
);
}
let file = tmp.path().join("one.json");
assert_eq!(
resolve_out(&corpus, Some(&file), false).expect("a file"),
None
);
}
#[test]
fn a_half_corpus_run_keeps_its_own_records_file() {
let corpus = PathBuf::from("/data/pentimento-core");
let whole = records_for(&corpus, None, &None, "stegcore", false, None);
let test = records_for(
&corpus,
None,
&None,
"stegcore",
false,
Some(score::Side::Test),
);
let train = records_for(
&corpus,
None,
&None,
"stegcore",
false,
Some(score::Side::Train),
);
assert_ne!(whole, test);
assert_ne!(test, train);
assert!(
test.to_string_lossy()
.ends_with("stegcore.test.records.jsonl"),
"{}",
test.display()
);
}
#[test]
fn a_records_path_the_caller_named_is_used_whatever_the_split() {
let corpus = PathBuf::from("/data/corpus");
let asked = PathBuf::from("/tmp/mine.jsonl");
let got = records_for(
&corpus,
Some(&asked),
&None,
"stegcore",
false,
Some(score::Side::Test),
);
assert_eq!(got, asked);
}
const CONTRACT_EXIT_CODES: &[i32] = &[
exit::OK,
exit::FAILURE,
exit::USAGE,
exit::PREFLIGHT_REFUSED,
exit::PLUGIN_FAILED,
exit::VERIFY_MISMATCH,
exit::SCHEMA_INVALID,
exit::LICENCE_REFUSED,
exit::ENVIRONMENT_UNFIT,
exit::INTERRUPTED,
];
#[test]
fn the_command_tree_is_well_formed() {
Cli::command().debug_assert();
}
#[test]
fn the_globals_reach_every_subcommand_so_the_man_pages_can_name_them() {
let mut root = Cli::command();
root.build();
let mut seen = 0;
for sub in root.get_subcommands() {
let longs: Vec<&str> = sub.get_arguments().filter_map(|a| a.get_long()).collect();
for global in ["json", "registry"] {
assert!(
longs.contains(&global),
"`{}` does not carry --{global}, so its man page cannot \
name a flag the binary accepts",
sub.get_name()
);
}
seen += 1;
}
assert!(seen >= 9, "checked only {seen} subcommand(s)");
}
#[test]
fn json_is_accepted_by_every_subcommand() {
for args in [
vec!["stegobench", "--json", "schema", "result-v1"],
vec!["stegobench", "--json", "validate", "x.json"],
vec!["stegobench", "--json", "list", "detectors"],
vec!["stegobench", "--json", "describe", "steghide"],
vec!["stegobench", "--json", "plan", "score"],
vec!["stegobench", "--json", "doctor"],
vec!["stegobench", "--json", "doctor", "--no-selftest"],
vec![
"stegobench",
"--json",
"score",
"--corpus",
"x",
"--detector",
"y",
],
vec!["stegobench", "--json", "completions", "bash"],
vec!["stegobench", "--json", "help", "pairing"],
vec!["stegobench", "--json", "help"],
] {
assert!(Cli::try_parse_from(&args).is_ok(), "rejected: {args:?}");
}
}
#[test]
fn a_corpus_path_that_is_not_a_readable_directory_is_refused_in_plain_words() {
let missing = corpus_path_problem(Path::new("/nope/not-here-either"))
.expect("a path that is not there is a problem");
assert!(
missing.contains("no corpus at"),
"says what is wrong: {missing}"
);
assert!(
!missing.contains("os error"),
"an operating system error number reached the user: {missing}"
);
let tmp = tempfile::tempdir().expect("tmp");
let file = tmp.path().join("one.png");
std::fs::write(&file, b"not really a png").expect("written");
let is_file = corpus_path_problem(&file).expect("a file is a problem");
assert!(
is_file.contains("is a file"),
"says what is wrong: {is_file}"
);
assert!(
corpus_path_problem(tmp.path()).is_none(),
"a readable directory is not a problem, and if it were this test \
would be passing for the wrong reason"
);
}
#[test]
fn a_corpus_value_is_tried_as_an_id_before_it_is_refused_as_a_path() {
let reg = Registry::load(&shipped_registry()).expect("the real registry loads");
let as_id = corpus_argument_problem(Some(®), Path::new("pentimento-core"))
.expect("a registered id is not a directory");
assert_eq!(as_id.code, exit::PREFLIGHT_REFUSED);
assert!(
as_id.human.contains("registered corpus id")
&& as_id.human.contains("stegobench fetch pentimento-core"),
"a registered id must be recognised as one: {}",
as_id.human
);
let neither = corpus_argument_problem(Some(®), Path::new("pentimento"))
.expect("a value that is nothing is a problem");
assert_eq!(neither.code, exit::PREFLIGHT_REFUSED);
assert!(
neither.human.contains("as a directory") && neither.human.contains("as a corpus id"),
"the refusal must say both meanings were tried: {}",
neither.human
);
assert!(neither.human.contains("no corpus at"), "{}", neither.human);
assert!(
neither.human.contains("pentimento-core"),
"a near miss must be named: {}",
neither.human
);
let blind = corpus_argument_problem(None, Path::new("pentimento-core"))
.expect("still not a directory");
assert_eq!(blind.code, exit::PREFLIGHT_REFUSED);
assert!(
blind.human.contains("registry could not be read"),
"{}",
blind.human
);
let tmp = tempfile::tempdir().expect("tmp");
assert!(corpus_argument_problem(Some(®), tmp.path()).is_none());
let file = tmp.path().join("shard.tar");
std::fs::write(&file, b"bytes").expect("written");
let wrong_shape =
corpus_argument_problem(Some(®), &file).expect("a file is not a corpus");
assert_eq!(wrong_shape.code, exit::PREFLIGHT_REFUSED);
assert!(
wrong_shape.human.contains("is a file"),
"{}",
wrong_shape.human
);
}
#[test]
fn an_unregistered_corpus_id_names_the_close_ones_and_spots_a_directory() {
let dir = shipped_registry();
let resolved = resolved_at(&dir);
let typo = unregistered_corpus_id(
&resolved,
"bossbas",
"`stegobench list corpora` shows what is.",
);
assert_eq!(typo.code, exit::PREFLIGHT_REFUSED);
assert!(
typo.human.contains("Did you mean") && typo.human.contains("bossbase"),
"{}",
typo.human
);
assert!(
typo.human.lines().last() == Some(resolved.source.line().as_str()),
"the refusal does not end by naming the registry:\n{}",
typo.human
);
let tmp = tempfile::tempdir().expect("tmp");
let as_path = unregistered_corpus_id(
&resolved,
&tmp.path().display().to_string(),
"`stegobench list corpora` shows what is.",
);
assert_eq!(as_path.code, exit::PREFLIGHT_REFUSED);
assert!(
as_path.human.contains("an id names a dataset")
&& as_path.human.contains("score --corpus"),
"a directory handed to an id flag must be recognised: {}",
as_path.human
);
}
#[test]
fn an_output_path_inside_the_corpus_is_refused_and_one_outside_it_is_not() {
let tmp = tempfile::tempdir().expect("tmp");
let corpus = tmp.path().join("mycorpus");
std::fs::create_dir_all(&corpus).expect("made");
let inside = output_inside_corpus(&corpus, "--out", Some(&corpus.join("result.json")))
.expect("a file in the corpus is refused");
assert!(
inside.contains("inside the corpus") && inside.contains("digest"),
"the refusal must say what would have happened: {inside}"
);
assert!(
inside.contains("Beside it works"),
"the refusal must say where to put it instead: {inside}"
);
let itself =
output_inside_corpus(&corpus, "--out", Some(&corpus)).expect("the corpus itself");
assert!(itself.contains("itself"), "{itself}");
let round_trip = corpus.join("..").join("mycorpus").join("result.json");
assert!(
output_inside_corpus(&corpus, "--out", Some(&round_trip)).is_some(),
"a path that leaves the corpus and returns was not recognised"
);
assert!(
output_inside_corpus(
&corpus,
"--records",
Some(&corpus.join("not-made-yet").join("deeper").join("r.jsonl"))
)
.is_some(),
"an unwritten directory inside the corpus was not recognised"
);
assert!(
output_inside_corpus(&corpus, "--out", Some(&tmp.path().join("result.json"))).is_none()
);
assert!(output_inside_corpus(
&corpus,
"--out",
Some(&tmp.path().join("nowhere").join("yet").join("result.json"))
)
.is_none());
assert!(output_inside_corpus(
&corpus,
"--out",
Some(&tmp.path().join("mycorpus-2/r.json"))
)
.is_none());
assert!(output_inside_corpus(&corpus, "--out", None).is_none());
}
#[cfg(unix)]
#[test]
fn a_symbolic_link_does_not_hide_an_output_path_inside_the_corpus() {
let tmp = tempfile::tempdir().expect("tmp");
let real = tmp.path().join("real-corpus");
std::fs::create_dir_all(&real).expect("made");
let link = tmp.path().join("linked");
std::os::unix::fs::symlink(&real, &link).expect("linked");
assert!(
output_inside_corpus(&link, "--out", Some(&real.join("result.json"))).is_some(),
"the link hid an output path inside the corpus"
);
assert!(
output_inside_corpus(&real, "--out", Some(&link.join("result.json"))).is_some(),
"the link hid an output path inside the corpus"
);
}
#[test]
fn score_and_plan_both_refuse_before_they_touch_the_corpus() {
let dir = tempfile::tempdir().expect("tmp");
let reg = dir.path().join("registry");
std::fs::create_dir_all(®).expect("made");
std::fs::write(
reg.join("ghost.toml"),
"name = \"ghost\"\nkind = \"detector\"\nlicence = \"MIT\"\n\
[binary]\ncommand = [\"definitely-not-installed-xyzzy\"]\n\
version_args = [\"--version\"]\n\
[emits]\noutput = \"score\"\nhigher_means_stego = true\n\
[invoke]\nargv = [\"{file}\"]\nparser = \"number\"\n\
[selftest]\nmust_detect = \"a.png\"\nmust_clear = \"b.png\"\n",
)
.expect("written");
let corpus = dir.path().join("mycorpus");
std::fs::create_dir_all(&corpus).expect("made");
let inside = corpus.join("result.json");
let scored = cmd_score(
&resolved_at(®),
ScoreRequest {
corpus: &corpus,
detectors: &["ghost".to_string()],
corpus_id: None,
trained_on: None,
split: None,
records: None,
out: Some(&inside),
timeout: 5,
jobs: 1,
keep_raw: false,
seed: None,
fixtures: None,
limit: None,
},
);
assert_eq!(scored.code, exit::USAGE, "{}", scored.human);
assert!(
scored.human.contains("inside the corpus"),
"{}",
scored.human
);
assert!(
!inside.exists(),
"the refusal wrote into the corpus it was refusing to write into"
);
let planned = cmd_plan(
&resolved_at(®),
&[
"score".to_string(),
"--corpus".to_string(),
corpus.display().to_string(),
"--detector".to_string(),
"ghost".to_string(),
"--records".to_string(),
corpus.join("r.jsonl").display().to_string(),
],
);
assert_eq!(planned.code, exit::USAGE, "{}", planned.human);
assert!(
planned.human.contains("inside the corpus"),
"{}",
planned.human
);
}
#[test]
fn score_answers_a_registered_id_in_the_path_flag_with_the_way_out() {
let refused = cmd_score(
&resolved_at(shipped_registry()),
ScoreRequest {
corpus: Path::new("pentimento-core"),
detectors: &["zsteg".to_string()],
corpus_id: None,
trained_on: None,
split: None,
records: None,
out: None,
timeout: 5,
jobs: 1,
keep_raw: false,
seed: None,
fixtures: None,
limit: None,
},
);
assert_eq!(refused.code, exit::PREFLIGHT_REFUSED, "{}", refused.human);
assert!(
refused.human.contains("stegobench fetch pentimento-core"),
"{}",
refused.human
);
}
#[test]
fn the_corpus_directory_comes_from_the_flag_then_the_environment_then_the_default() {
let named = Path::new("/tmp/named-by-the-flag");
assert_eq!(
fetch_dest(Some(named), Some("/tmp/named-by-the-variable")),
named,
"the environment overrode an explicit --dest"
);
assert_eq!(
fetch_dest(None, Some("/tmp/named-by-the-variable")),
Path::new("/tmp/named-by-the-variable"),
"the variable was not read"
);
assert_eq!(fetch_dest(None, Some("")), fetch::default_dest());
assert_eq!(fetch_dest(None, None), fetch::default_dest());
}
#[test]
fn no_argument_reads_its_environment_variable_through_clap() {
fn walk(cmd: &clap::Command, found: &mut Vec<String>) {
for arg in cmd.get_arguments() {
if let Some(var) = arg.get_env() {
found.push(format!(
"{} --{} reads {}",
cmd.get_name(),
arg.get_id(),
var.to_string_lossy()
));
}
}
for sub in cmd.get_subcommands() {
walk(sub, found);
}
}
let root = Cli::command();
let mut found = Vec::new();
walk(&root, &mut found);
assert!(
found.is_empty(),
"an empty value in one of these variables would refuse every \
command: {}",
found.join(", ")
);
}
#[test]
fn no_help_text_carries_this_project_s_internal_vocabulary() {
const INTERNAL: &[&str] = &[
"a journey",
"journeys",
"persona",
"rung",
"the panel",
"subagent",
"DEFERRED",
"the baseline",
"the operator decided",
];
let root = Cli::command();
let mut checked = 0;
let mut found: Vec<String> = Vec::new();
for sub in root.get_subcommands() {
let name = sub.get_name().to_string();
let rendered = sub.clone().render_long_help().to_string();
checked += 1;
for word in INTERNAL {
if rendered.to_lowercase().contains(&word.to_lowercase()) {
let line = rendered
.lines()
.find(|l| l.to_lowercase().contains(&word.to_lowercase()))
.unwrap_or("")
.trim()
.to_string();
found.push(format!("{name} --help says {word:?}: {line}"));
}
}
}
assert!(
checked >= 10,
"rendered help for {checked} subcommand(s), so this looked at \
almost nothing"
);
assert!(
found.is_empty(),
"internal vocabulary reached the help text. A `///` comment on a \
flag is printed to users; move the rationale to a `//` comment:\n{}",
found.join("\n")
);
}
#[test]
fn every_subcommand_in_the_tree_accepts_json_and_the_run_output_is_parseable() {
let root = Cli::command();
let walked = root.get_subcommands().count();
assert!(
walked >= 10,
"walked {walked} subcommand(s); the command tree carries schema, \
validate, verify, list, describe, doctor, plan, score, \
completions and help, so this walk looked at almost nothing"
);
for sub in root.get_subcommands() {
let name = sub.get_name().to_string();
let mut argv = vec!["stegobench".to_string(), "--json".to_string(), name.clone()];
for arg in sub.get_arguments() {
if !arg.is_required_set() {
continue;
}
let placeholder = if name == "completions" {
"bash".to_string()
} else {
"x".to_string()
};
if let Some(long) = arg.get_long() {
argv.push(format!("--{long}"));
}
argv.push(placeholder);
}
if name == "metrics" {
argv.push("no-such-scores-file.json".to_string());
}
let parsed = Cli::try_parse_from(&argv);
assert!(parsed.is_ok(), "{name:?} rejected --json: {argv:?}");
let cli = parsed.unwrap();
let out = run(&cli);
let rendered =
serde_json::to_string(&out.json).expect("a Value always renders to text");
let reparsed: serde_json::Value = serde_json::from_str(&rendered)
.unwrap_or_else(|e| panic!("{name:?} produced JSON that will not re-parse: {e}"));
assert!(
reparsed.is_object(),
"{name:?} emitted {reparsed} on --json, but every subcommand's \
machine output is a JSON object"
);
assert!(
CONTRACT_EXIT_CODES.contains(&out.code),
"{name:?} exited {} with --json, which is not in the exit code \
contract the help text and the man page publish",
out.code
);
assert!(
!out.human.trim().is_empty(),
"{name:?} produced no human text, so a failure would print \
nothing to stderr"
);
}
}
fn shipped_registry() -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../plugins/registry")
}
fn resolved_at(dir: impl AsRef<Path>) -> Resolved {
Resolved::from_dir(dir.as_ref()).expect("the registry under test loads")
}
#[test]
fn every_shipped_corpus_validates_and_there_is_at_least_one_to_validate() {
let reg = Registry::load(&shipped_registry()).expect("the real registry loads");
assert!(
!reg.corpora.is_empty(),
"no corpora were loaded from {}, so this test verified nothing. A \
count of zero here is a failure, not a pass",
shipped_registry().display()
);
for (id, corpus) in ®.corpora {
assert_eq!(corpus.validate(), Ok(()), "{id} is registered but invalid");
assert_eq!(id, &corpus.id, "keyed under a name that is not its id");
}
}
#[test]
fn no_surface_calls_an_entry_usable_that_nothing_here_can_drive() {
let reg = Registry::load(&shipped_registry()).expect("the real registry loads");
assert!(
!reg.entries.is_empty(),
"no tools were loaded from {}, so this test verified nothing",
shipped_registry().display()
);
let mut undrivable = 0;
for entry in reg.entries.values() {
let present = stegobench_plugin::availability::Availability {
name: entry.name.clone(),
presence: stegobench_plugin::availability::Presence::Present {
pin: "supplied by the test".into(),
},
verified: None,
missing_secrets: Vec::new(),
};
let needs = needs::of_tool(entry, &present, &[]);
let word = readiness_word(&needs, "tool", &entry.name);
let row = entry.summary();
if entry.can_be_driven() {
assert_eq!(
needs.readiness,
needs::Readiness::Ready,
"{} is drivable and present and was not reported ready",
entry.name
);
assert!(
!row.contains("can't drive it"),
"{} is drivable and its listing row says otherwise: {row}",
entry.name
);
continue;
}
undrivable += 1;
assert_eq!(
needs.readiness,
needs::Readiness::CannotBeDriven,
"{} declares nothing to drive and was reported otherwise",
entry.name
);
assert_ne!(
word, "stegobench can run it",
"describe says it can run {}, which no command here can",
entry.name
);
assert!(
row.contains("can't drive it"),
"{} sits in the listing unmarked: {row}",
entry.name
);
let why = stegobench_cli::unavailable_reason(entry, &[])
.unwrap_or_else(|| panic!("{} would be run by score and plan", entry.name));
assert!(
why.contains("block"),
"{} is skipped for a reason that does not name the missing \
block: {why}",
entry.name
);
}
assert!(
undrivable > 0,
"no registered entry is undrivable, so the branch that this test \
exists for was never reached. If that is now true of the \
registry, delete this assertion and say so."
);
}
#[test]
fn every_result_this_repository_publishes_validates() {
let dir = Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../..")
.join("results/v1");
let mut checked = 0;
for entry in std::fs::read_dir(&dir).expect("results/v1 exists") {
let path = entry.expect("readable").path();
if path.extension().is_none_or(|e| e != "json") {
continue;
}
let out = cmd_validate(&path);
assert_eq!(
out.code,
exit::OK,
"{} does not validate:\n{}",
path.display(),
out.human
);
checked += 1;
}
assert!(
checked >= 20,
"checked {checked} document(s) in {}; this repository publishes \
two dozen, so the walk found almost nothing",
dir.display()
);
}
#[test]
fn list_corpora_and_describe_reach_a_corpus_the_way_they_reach_a_tool() {
let dir = shipped_registry();
let listed = cmd_list(&resolved_at(&dir), "corpora");
assert_eq!(listed.code, exit::OK);
assert!(
listed.human.contains("reveal") && listed.human.contains("republish:"),
"got: {}",
listed.human
);
assert!(
listed.json["count"].as_u64().is_some_and(|n| n > 0),
"the machine output must carry the count, not only the rows"
);
let described = cmd_describe(&resolved_at(&dir), "reveal", false);
assert_eq!(described.code, exit::OK);
assert_eq!(described.json["licence"]["redistribution"], "permitted");
assert!(
described.human.contains("verified_on"),
"the summary drops the verification date: {}",
described.human
);
assert!(
described.human.contains("Republish"),
"the summary drops the redistribution answer: {}",
described.human
);
assert_eq!(
cmd_describe(&resolved_at(&dir), "steghide", false).code,
exit::OK
);
let missing = cmd_describe(&resolved_at(&dir), "not-registered", false);
assert_eq!(missing.code, exit::USAGE);
assert!(missing.human.contains("reveal") && missing.human.contains("steghide"));
}
#[test]
fn describe_summarises_by_default_and_prints_the_entry_only_under_toml() {
let dir = shipped_registry();
for name in ["steghide", "reveal"] {
let summary = cmd_describe(&resolved_at(&dir), name, false);
assert_eq!(summary.code, exit::OK);
assert!(
!summary.human.contains("[licence]") && !summary.human.contains("[properties]"),
"{name}'s summary still carries the raw TOML: {}",
summary.human
);
assert!(
summary.human.contains("--toml"),
"{name}'s summary does not say where the whole entry went: {}",
summary.human
);
let raw = cmd_describe(&resolved_at(&dir), name, true);
assert_eq!(raw.code, exit::OK);
assert!(
raw.payload_on_stdout,
"{name} --toml was diverted to stderr, so it cannot be piped"
);
let parsed: toml::Value = toml::from_str(&raw.human).unwrap_or_else(|e| {
panic!("{name} --toml is not parseable TOML: {e}\n{}", raw.human)
});
assert!(
parsed.get("name").is_some() || parsed.get("id").is_some(),
"{name} --toml is missing the entry itself: {}",
raw.human
);
assert!(
!raw.human.contains("Needs from you") && !raw.human.contains("Nothing needed"),
"{name} --toml carries the summary too: {}",
raw.human
);
}
let zsteg = cmd_describe(&resolved_at(&dir), "zsteg", false);
assert_eq!(zsteg.code, exit::OK);
for expected in ["Runs as", "Costs", "Run it", "-a /work/{file}"] {
assert!(
zsteg.human.contains(expected),
"the summary drops {expected:?}: {}",
zsteg.human
);
}
}
#[test]
fn describe_says_it_takes_a_name_when_it_is_handed_a_path() {
let dir = shipped_registry();
let tmp = tempfile::tempdir().expect("tmp");
let refused = cmd_describe(&resolved_at(&dir), &tmp.path().display().to_string(), false);
assert_eq!(refused.code, exit::USAGE);
assert!(
refused.human.contains("is a path on this machine")
&& refused.human.contains("score --corpus"),
"{}",
refused.human
);
let typo = cmd_describe(&resolved_at(&dir), "steghid", false);
assert_eq!(typo.code, exit::USAGE);
assert!(
typo.human.contains("Did you mean") && typo.human.contains("steghide"),
"{}",
typo.human
);
}
#[test]
fn a_listing_lines_its_columns_up_however_long_the_longest_name_is() {
let dir = shipped_registry();
for kind in ["corpora", "detectors", "embedders"] {
let out = cmd_list(&resolved_at(&dir), kind);
assert_eq!(out.code, exit::OK, "{kind}");
let rows: Vec<&str> = out
.human
.lines()
.take_while(|l| !l.trim().is_empty())
.collect();
assert!(rows.len() > 1, "{kind}: nothing to align");
let longest = rows
.iter()
.map(|r| r.split_whitespace().next().unwrap_or("").len())
.max()
.expect("rows");
for row in &rows {
let name = row.split_whitespace().next().expect("a name");
let second = row[name.len()..]
.find(|c: char| c != ' ')
.map(|i| i + name.len());
assert_eq!(
second,
Some(longest + 1),
"{kind}: {row:?} does not start its second column where the others do"
);
}
assert!(
rows.iter().any(|r| r.len() > longest + 1),
"{kind}: every row is a bare name, so this proved nothing"
);
}
}
#[test]
fn list_all_carries_the_corpora_as_well_as_the_tools() {
let out = cmd_list(&resolved_at(shipped_registry()), "all");
assert_eq!(out.code, exit::OK);
assert!(out.json["tools"].as_array().is_some_and(|a| !a.is_empty()));
assert!(out.json["corpora"]
.as_array()
.is_some_and(|a| !a.is_empty()));
assert!(out.human.contains("republishable"), "got: {}", out.human);
}
#[test]
fn an_empty_tools_listing_says_where_it_looked() {
let dir = tempfile::tempdir().unwrap();
let out = cmd_list(&resolved_at(dir.path()), "detectors");
assert_eq!(out.code, exit::OK);
assert!(
out.human.contains("no tools registered")
&& out.human.contains(&dir.path().display().to_string()),
"got: {}",
out.human
);
}
#[test]
fn a_kind_with_no_tools_is_distinguished_from_an_empty_registry() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(
dir.path().join("only-a-detector.toml"),
format!(
"name = \"solo\"\nkind = \"detector\"\nlicence = \"MIT\"\n\
[image]\nreference = \"ghcr.io/x/y@sha256:{}\"\n\
[selftest]\nmust_detect = \"a.png\"\nmust_clear = \"b.png\"\n",
"5".repeat(64)
),
)
.unwrap();
let out = cmd_list(&resolved_at(dir.path()), "embedders");
assert_eq!(out.code, exit::OK);
assert!(out.json["tools"].as_array().is_some_and(|a| a.is_empty()));
assert!(
out.human.contains("none of the 1 tool(s)") && out.human.contains("is a embedder"),
"got: {}",
out.human
);
assert!(
!out.human.contains("no tools registered"),
"a registry with a tool in it was reported as empty: {}",
out.human
);
}
#[test]
fn an_empty_corpora_listing_says_so_rather_than_printing_nothing() {
let dir = tempfile::tempdir().unwrap();
let out = cmd_list(&resolved_at(dir.path()), "corpora");
assert_eq!(out.code, exit::OK);
assert_eq!(out.json["count"], 0);
assert!(
out.human.contains("no corpora registered"),
"an empty registry printed: {:?}",
out.human
);
}
#[test]
fn the_registry_flag_carries_no_relative_default_value() {
let mut cmd = Cli::command();
cmd.build();
let arg = cmd
.get_arguments()
.find(|a| a.get_long() == Some("registry"))
.expect("--registry exists");
assert!(
arg.get_default_values().is_empty(),
"--registry has a default value again, so an installed binary run \
outside a checkout resolves it against the current directory and \
every command that reads a registry fails"
);
}
#[test]
fn a_bare_invocation_points_somewhere_rather_than_printing_everything() {
let out = cmd_orientation(None);
assert_eq!(out.code, exit::USAGE);
assert!(
out.payload_on_stdout,
"the orientation text is the answer to what was asked, so it \
belongs on stdout"
);
assert!(
out.human.contains("stegobench list detectors"),
"no obvious next command: {}",
out.human
);
assert!(
out.human.contains("help scope"),
"the bare invocation does not say what the tool is not for: {}",
out.human
);
assert!(
out.human.lines().count() <= 16,
"the orientation is {} lines, which is the whole help again",
out.human.lines().count()
);
}
#[test]
fn the_guessed_words_run_examine_rather_than_reporting_a_typo() {
for word in ["check", "scan", "inspect", "detect", "analyse", "analyze"] {
let parsed = Cli::try_parse_from(["stegobench", word, "./photo.png", "-d", "zsteg"])
.unwrap_or_else(|e| panic!("`{word}` is rejected rather than routed: {e}"));
assert!(
matches!(parsed.command, Some(Command::Examine { .. })),
"`{word}` parses as something other than examine"
);
}
}
#[test]
fn an_alias_refuses_a_missing_file_without_starting_a_detector() {
let parsed = Cli::try_parse_from([
"stegobench",
"check",
"./no-such-image-exists-here.png",
"-d",
"zsteg",
])
.expect("the alias parses");
let out = run(&parsed);
assert_eq!(out.code, exit::PREFLIGHT_REFUSED, "{}", out.human);
assert_eq!(out.json["reason"], "not-a-file", "{}", out.json);
}
#[test]
fn every_extraction_word_is_told_that_extraction_is_absent() {
assert!(
NO_EXTRACTION.windows(2).all(|w| w[0] < w[1]),
"the list is meant to stay sorted so a reader can find a word in it"
);
for word in NO_EXTRACTION {
assert!(no_extraction(word), "`{word}` is on the list and unmatched");
assert!(
no_extraction(&word.to_uppercase()),
"`{word}` typed in capitals is the same misunderstanding"
);
assert!(
!meant_examine(word),
"`{word}` is on both lists, so its answer depends on arm order"
);
let out = cmd_no_extraction(word);
assert_eq!(out.code, exit::USAGE, "`{word}` should be a usage error");
for expected in [
"does not pull a hidden payload",
"examine",
"list detectors",
] {
assert!(
out.human.contains(expected),
"`{word}` does not say {expected:?}: {}",
out.human
);
}
match Cli::try_parse_from(["stegobench", word, "./photo.png"]) {
Ok(_) => panic!("`{word}` parsed as a command, and it is not one"),
Err(e) => assert_eq!(
e.kind(),
clap::error::ErrorKind::InvalidSubcommand,
"`{word}` fails in a way `parse_or_explain` does not catch"
),
}
}
}
#[test]
fn every_examiner_word_reaches_examine() {
assert!(
MEANT_EXAMINE.windows(2).all(|w| w[0] < w[1]),
"the list is meant to stay sorted so a reader can find a word in it"
);
for word in MEANT_EXAMINE {
assert!(meant_examine(word), "`{word}` is on the list and unmatched");
assert!(
meant_examine(&word.to_uppercase()),
"`{word}` typed in capitals means the same thing"
);
match Cli::try_parse_from(["stegobench", word, "./photo.png", "-d", "zsteg"]) {
Ok(parsed) => assert!(
matches!(parsed.command, Some(Command::Examine { .. })),
"`{word}` parses as something other than examine"
),
Err(e) => {
assert_eq!(
e.kind(),
clap::error::ErrorKind::InvalidSubcommand,
"`{word}` fails in a way `parse_or_explain` does not catch"
);
let out = cmd_meant_examine(word);
assert_eq!(out.code, exit::USAGE);
assert!(
out.human.contains("is `examine`"),
"`{word}` does not name the command: {}",
out.human
);
}
}
}
}
#[test]
fn a_real_command_is_never_read_as_a_guess() {
for word in [
"list",
"score",
"help",
"doctor",
"verify",
"validate",
"report",
"frobnicate",
"",
] {
assert!(!no_extraction(word), "`{word}` was diverted");
assert!(!meant_examine(word), "`{word}` was diverted");
}
}
#[test]
fn a_shell_word_is_quoted_only_when_the_shell_would_take_it_apart() {
assert_eq!(shell_word("zsteg"), "zsteg");
assert_eq!(shell_word("/work/{file}"), "/work/{file}");
assert_eq!(
shell_word("PYTHONPATH=/opt/aletheia"),
"PYTHONPATH=/opt/aletheia"
);
assert_eq!(shell_word("/my pictures/a.png"), "'/my pictures/a.png'");
assert_eq!(shell_word("it's.png"), r"'it'\''s.png'");
assert_eq!(shell_word(""), "''");
}
fn parsed_entry(text: &str) -> stegobench_core::registry::Entry {
toml::from_str(text).expect("the entry under test parses")
}
#[test]
fn a_containerised_tool_pastes_as_a_pinned_sandboxed_docker_run() {
let e = parsed_entry(
"name = \"zsteg\"\nkind = \"detector\"\nlicence = \"MIT\"\n\
[image]\nreference = \"stegobench/zsteg@sha256:abc\"\n\
[emits]\noutput = \"verdict\"\n\
[invoke]\nargv = [\"zsteg\", \"-a\", \"{file}\"]\nparser = \"zsteg\"\n",
);
let line = runnable_command(&e, &[]).expect("a container has a command");
assert!(line.contains("stegobench/zsteg@sha256:abc"), "{line}");
for flag in [
"--rm",
"--network=none",
"--cap-drop=ALL",
"--security-opt no-new-privileges",
"--read-only",
"--memory=2g",
] {
assert!(line.contains(flag), "{flag} is missing from {line}");
}
assert!(
line.contains("-v \"$PWD/{file}\":/work/{file}:ro"),
"{line}"
);
assert!(line.ends_with("zsteg -a /work/{file}"), "{line}");
assert!(
!line.contains("--user"),
"a read-only run needs no uid: {line}"
);
}
#[test]
fn a_tool_that_writes_beside_its_input_pastes_with_the_directory_mounted() {
let e = parsed_entry(
"name = \"stegexpose\"\nkind = \"detector\"\nlicence = \"GPL-3.0\"\n\
[image]\nreference = \"stegobench/stegexpose@sha256:def\"\n\
[emits]\noutput = \"score\"\nhigher_means_stego = true\n\
[invoke]\nargv = [\"/work\", \"default\", \"0.2\"]\n\
writable_workdir = true\nparser = \"stegexpose\"\n",
);
let line = runnable_command(&e, &[]).expect("a container has a command");
assert!(line.contains("-v \"$PWD\":/work"), "{line}");
assert!(line.contains("--user $(id -u):$(id -g)"), "{line}");
assert!(
!line.contains("{file}"),
"nothing to substitute here: {line}"
);
}
#[test]
fn a_binary_tool_pastes_with_the_program_name_it_needs() {
let e = parsed_entry(
"name = \"stegcore\"\nkind = \"detector\"\nlicence = \"AGPL-3.0-or-later\"\n\
[binary]\ncommand = [\"stegcore\", \"analyse\", \"--json\"]\n\
[emits]\noutput = \"score\"\nhigher_means_stego = true\n\
[invoke]\nargv = [\"{file}\"]\nparser = \"stegcore\"\n",
);
assert_eq!(
runnable_command(&e, &[]).as_deref(),
Some("stegcore analyse --json {file}")
);
}
#[test]
fn an_entry_with_nothing_runnable_pastes_no_command_at_all() {
let missing_adapter = parsed_entry(
"name = \"aletheia-rs\"\nkind = \"detector\"\nlicence = \"MIT\"\n\
[image]\nreference = \"stegobench/aletheia@sha256:abc\"\n\
[emits]\noutput = \"score\"\nhigher_means_stego = true\n\
[invoke]\nadapter = \"nowhere/at/all.py\"\nentrypoint = \"python3\"\n\
argv = [\"{adapter}\", \"{file}\"]\nparser = \"number\"\n",
);
assert_eq!(runnable_command(&missing_adapter, &[]), None);
let nothing_declared = parsed_entry(
"name = \"aletheia-rich\"\nkind = \"detector\"\nlicence = \"MIT\"\n\
[image]\nreference = \"stegobench/aletheia-rich@sha256:abc\"\n\
[emits]\noutput = \"score\"\nhigher_means_stego = true\n",
);
assert_eq!(runnable_command(¬hing_declared, &[]), None);
}
#[test]
fn an_adapter_is_mounted_and_then_referred_to_by_its_inside_path() {
let tmp = tempfile::tempdir().unwrap();
let adapters = tmp.path().join("plugins/adapters");
std::fs::create_dir_all(&adapters).unwrap();
std::fs::write(adapters.join("one.py"), "print(0)\n").unwrap();
let e = parsed_entry(
"name = \"aletheia-rs\"\nkind = \"detector\"\nlicence = \"MIT\"\n\
[image]\nreference = \"stegobench/aletheia@sha256:abc\"\n\
[emits]\noutput = \"score\"\nhigher_means_stego = true\n\
[invoke]\nadapter = \"plugins/adapters/one.py\"\nentrypoint = \"python3\"\n\
env = [\"PYTHONPATH=/opt/aletheia\"]\n\
argv = [\"{adapter}\", \"{file}\", \"rs\"]\nparser = \"number\"\n",
);
let line = runnable_command(&e, &[tmp.path().to_path_buf()]).expect("resolvable");
assert!(line.contains(":/adapter/one.py:ro"), "{line}");
assert!(line.contains("-e PYTHONPATH=/opt/aletheia"), "{line}");
assert!(line.contains("--entrypoint python3"), "{line}");
assert!(line.ends_with("/adapter/one.py /work/{file} rs"), "{line}");
assert!(
!line.contains("{adapter}"),
"the placeholder survived: {line}"
);
}
#[test]
fn a_command_is_never_printed_against_an_adapter_that_will_not_survive() {
let built_in = registry::Resolved::built_in().expect("the built-in registry loads");
let out = cmd_describe(&built_in, "aletheia-rs", false);
assert_eq!(out.code, exit::OK, "{}", out.human);
let temp = std::env::temp_dir().display().to_string();
assert!(
!out.human.contains(&temp),
"a scratch path was offered to be pasted: {}",
out.human
);
assert!(
out.human.contains("no command to paste"),
"the reader is left with nothing and no reason: {}",
out.human
);
assert!(
out.human.contains("--registry plugins/registry"),
"nothing says how to get the command: {}",
out.human
);
let on_disk = cmd_describe(&resolved_at(shipped_registry()), "aletheia-rs", false);
assert!(
on_disk.human.contains("docker run --rm"),
"a registry with its adapters beside it still cannot paste: {}",
on_disk.human
);
}
#[test]
fn a_service_pastes_the_adapter_that_runs_on_this_machine() {
let tmp = tempfile::tempdir().unwrap();
let adapters = tmp.path().join("plugins/adapters");
std::fs::create_dir_all(&adapters).unwrap();
std::fs::write(adapters.join("svc.py"), "print(0)\n").unwrap();
let e = parsed_entry(
"name = \"stegashield\"\nkind = \"detector\"\nlicence = \"proprietary\"\n\
[image]\nreference = \"5iprojects/stegashield@sha256:abc\"\n\
[emits]\noutput = \"score\"\nhigher_means_stego = true\n\
[invoke]\nhost = true\nadapter = \"plugins/adapters/svc.py\"\n\
entrypoint = \"python3\"\nargv = [\"{adapter}\", \"{file}\"]\nparser = \"number\"\n",
);
let line = runnable_command(&e, &[tmp.path().to_path_buf()]).expect("resolvable");
assert!(line.starts_with("python3 "), "{line}");
assert!(
!line.contains("docker"),
"a service is not started here: {line}"
);
assert!(line.ends_with("{file}"), "{line}");
}
#[test]
fn the_status_word_says_who_it_is_about() {
let ready = needs::Needs {
readiness: needs::Readiness::Ready,
steps: Vec::new(),
};
assert_eq!(
readiness_word(&ready, "tool", "zsteg"),
"stegobench can run it"
);
let unknown = needs::Needs {
readiness: needs::Readiness::Unknown,
steps: Vec::new(),
};
assert_eq!(
readiness_word(&unknown, "corpus", STARTER_ID),
"included in this binary"
);
assert_eq!(
readiness_word(&unknown, "corpus", "bossbase"),
"bring your own copy"
);
assert_eq!(readiness_word(&unknown, "tool", "stegashield"), "unknown");
let needs_you = needs::Needs {
readiness: needs::Readiness::NeedsYou,
steps: Vec::new(),
};
assert_eq!(readiness_word(&needs_you, "tool", "stegcore"), "NEEDS YOU");
}
fn numbers_in(line: &str) -> Vec<u64> {
line.split(|c: char| !c.is_ascii_digit())
.filter(|s| !s.is_empty())
.filter_map(|s| s.parse().ok())
.collect()
}
#[test]
fn every_count_doctor_prints_adds_up_to_the_total_beside_it() {
let out = cmd_doctor(&resolved_at(shipped_registry()), None, false, false);
let total = out.json["checked"].as_u64().expect("a total");
assert!(total > 1, "a registry of one proves nothing here");
let mut summed = 0;
for (head, from_second) in [
("tool(s):", true),
("On this machine:", false),
("Stegobench's own self-test:", false),
] {
let line = out
.human
.lines()
.find(|l| l.contains(head))
.unwrap_or_else(|| panic!("no line for {head:?}: {}", out.human));
let found = numbers_in(line);
let (stated, parts) = if from_second {
(found[0], &found[1..])
} else {
(total, &found[..])
};
assert_eq!(stated, total, "{head:?} states a different total: {line}");
assert_eq!(
parts.iter().sum::<u64>(),
total,
"{head:?} does not add up to {total}: {line}"
);
summed += 1;
}
assert_eq!(summed, 3, "a line went unchecked");
}
#[test]
fn doctor_and_the_listing_both_say_which_kind_each_tool_is() {
let dir = shipped_registry();
let doctor = cmd_doctor(&resolved_at(&dir), None, false, false);
let listing = cmd_list(&resolved_at(&dir), "all");
let detectors = doctor.json["detectors"].as_u64().expect("a count");
let embedders = doctor.json["embedders"].as_u64().expect("a count");
assert!(detectors > 0 && embedders > 0, "one kind is missing here");
for screen in [&doctor.human, &listing.human] {
for heading in [
format!("DETECTORS ({detectors})"),
format!("EMBEDDERS ({embedders})"),
] {
assert!(screen.contains(&heading), "no {heading:?} in:\n{screen}");
}
}
for (kind, count) in [("detectors", detectors), ("embedders", embedders)] {
let one = cmd_list(&resolved_at(&dir), kind);
assert_eq!(
one.json["tools"].as_array().map(Vec::len),
Some(count as usize),
"`list {kind}` and the grouped screens disagree"
);
}
}
#[test]
fn a_bare_listing_says_where_the_tools_end_and_the_corpora_begin() {
let dir = shipped_registry();
let out = cmd_list(&resolved_at(&dir), "all");
let corpora = out.json["corpora"].as_array().expect("corpora").len();
assert!(
out.human.contains(&format!("CORPORA ({corpora})")),
"{}",
out.human
);
let heading = out.human.find("CORPORA (").expect("the corpora heading");
let tools = out.human.find("DETECTORS (").expect("the tools heading");
assert!(tools < heading, "the corpora are listed before the tools");
}
#[test]
fn a_fact_of_several_lines_keeps_the_column() {
let block = describe_block(
&needs::Needs {
readiness: needs::Readiness::Ready,
steps: Vec::new(),
},
"thing",
"tool",
&[
("Short", "one".to_string()),
("Much longer", "first\nsecond".to_string()),
],
);
let lines: Vec<&str> = block.lines().collect();
let continuation = lines
.iter()
.find(|l| l.trim() == "second")
.expect("the second line survived");
let head = lines
.iter()
.find(|l| l.starts_with("Much longer"))
.expect("the labelled line");
assert_eq!(
continuation.find("second"),
head.find("first"),
"the continuation does not line up: {continuation:?} against {head:?}"
);
}
#[test]
fn the_nearest_command_is_named_for_a_near_miss_and_withheld_for_a_stranger() {
assert_eq!(nearest_command("lst").as_deref(), Some("list"));
assert_eq!(nearest_command("doctr").as_deref(), Some("doctor"));
assert_eq!(nearest_command("REPORT").as_deref(), Some("report"));
assert_eq!(nearest_command("frobnicate"), None);
for word in ["chek", "scn", "sca"] {
let got = nearest_command(word);
assert!(
!matches!(got.as_deref(), Some("check" | "scan")),
"{word} was pointed at a hidden signpost: {got:?}"
);
}
assert_eq!(
nearest_command("chek"),
None,
"`chek` was pointed at some third command, which is what clap did \
when it answered it with `schema`"
);
}
#[test]
fn the_edit_distance_is_the_ordinary_one() {
assert_eq!(edit_distance("", ""), 0);
assert_eq!(edit_distance("", "list"), 4);
assert_eq!(edit_distance("list", ""), 4);
assert_eq!(edit_distance("list", "list"), 0);
assert_eq!(edit_distance("lst", "list"), 1);
assert_eq!(edit_distance("kitten", "sitting"), 3);
}
#[test]
fn the_long_help_names_the_commands_before_the_exit_codes_and_keeps_both() {
let mut cmd = Cli::command();
let mut buf: Vec<u8> = Vec::new();
cmd.write_long_help(&mut buf).expect("the help renders");
let help = String::from_utf8(buf).expect("help is text");
let commands = help.find("Commands:").expect("the command list is there");
let codes = help
.find("EXIT CODES")
.expect("the exit codes are still there");
assert!(
commands < codes,
"the exit codes come before the list of commands again"
);
for code in CONTRACT_EXIT_CODES {
assert!(
help.contains(&format!("{code}")),
"exit code {code} is no longer documented anywhere in the help"
);
}
assert!(
help.contains("stdout") && help.contains("stderr"),
"the stream contract was dropped rather than moved"
);
let lower = help.to_lowercase();
assert!(
lower.contains("images of your own"),
"the long help no longer says the tool can be pointed at your own images"
);
assert!(
lower.contains("not a measurement"),
"the long help no longer says an examination is not a measurement"
);
}
#[test]
fn scope_is_a_help_topic_and_says_what_this_is_not_for() {
let out = cmd_help(Some("scope"));
assert_eq!(out.code, exit::OK);
assert!(out.human.contains("IS SOMETHING HIDDEN IN THESE PICTURES"));
assert!(out.human.contains("stegobench list detectors"));
let listed = cmd_help(None);
assert!(
listed.human.contains("scope"),
"the topic list does not offer scope: {}",
listed.human
);
}
#[test]
fn a_folder_of_photographs_is_recognised_and_a_corpus_is_not() {
let dir = tempfile::tempdir().unwrap();
assert_eq!(
unlabelled_corpus(dir.path()),
None,
"an empty directory is not a photo album"
);
std::fs::write(dir.path().join("DSC_0001.JPG"), b"x").unwrap();
std::fs::write(dir.path().join("DSC_0002.png"), b"x").unwrap();
let why = unlabelled_corpus(dir.path()).expect("two images and no records");
assert!(why.contains("help scope"), "got: {why}");
assert!(why.contains("2 image(s)"), "got: {why}");
std::fs::write(dir.path().join("DSC_0001.json"), b"{}").unwrap();
assert_eq!(
unlabelled_corpus(dir.path()),
None,
"one record is enough to stop guessing and let the corpus loader speak"
);
assert_eq!(
unlabelled_corpus(&dir.path().join("not-there")),
None,
"a missing directory is the corpus loader's error to report, not this one's"
);
}
#[test]
fn score_refuses_a_folder_of_unlabelled_images_before_it_talks_about_detectors() {
let dir = tempfile::tempdir().unwrap();
let corpus = dir.path().join("holiday-photos");
std::fs::create_dir_all(&corpus).unwrap();
std::fs::write(corpus.join("a.png"), b"x").unwrap();
let out = cmd_score(
&resolved_at(shipped_registry()),
ScoreRequest {
corpus: &corpus,
detectors: &["all".to_string()],
corpus_id: None,
trained_on: None,
records: None,
out: None,
timeout: 5,
jobs: 1,
keep_raw: false,
seed: None,
fixtures: None,
limit: None,
split: None,
},
);
assert_eq!(out.code, exit::PREFLIGHT_REFUSED);
assert!(out.human.contains("help scope"), "got: {}", out.human);
assert!(
!out.human.contains("docker pull"),
"the refusal sent somebody to install a container: {}",
out.human
);
}
#[test]
fn plan_refuses_a_folder_of_unlabelled_images_the_same_way_score_does() {
let dir = tempfile::tempdir().unwrap();
let corpus = dir.path().join("holiday-photos");
std::fs::create_dir_all(&corpus).unwrap();
std::fs::write(corpus.join("a.png"), b"x").unwrap();
let out = cmd_plan(
&resolved_at(shipped_registry()),
&[
"score".to_string(),
"--corpus".to_string(),
corpus.display().to_string(),
"--detector".to_string(),
"zsteg".to_string(),
],
);
assert_eq!(out.code, exit::PREFLIGHT_REFUSED);
assert!(out.human.contains("help scope"), "got: {}", out.human);
}
#[test]
fn list_and_doctor_both_name_the_registry_that_answered() {
let resolved = resolved_at(shipped_registry());
let listed = cmd_list(&resolved, "detectors");
assert!(
listed.human.contains("registry "),
"list does not name the registry: {}",
listed.human
);
assert_eq!(listed.json["registry"]["source"], "explicit");
assert_eq!(
listed.json["registry"]["path"],
shipped_registry().display().to_string()
);
let doctor = cmd_doctor(&resolved, None, false, false);
assert!(
doctor.human.starts_with("registry "),
"doctor does not open by naming the registry: {}",
doctor.human.lines().next().unwrap_or_default()
);
assert_eq!(doctor.json["registry"]["source"], "explicit");
assert!(doctor.json["registry"]["tools"].as_u64().unwrap() > 0);
}
#[test]
fn an_unknown_list_kind_names_corpora_among_the_known_ones() {
let out = cmd_list(&resolved_at(shipped_registry()), "corpuses");
assert_eq!(out.code, exit::USAGE);
assert!(out.human.contains("corpora"), "got: {}", out.human);
}
#[test]
fn every_shell_gets_a_script_with_the_command_names_in_it() {
for shell in [
clap_complete::Shell::Bash,
clap_complete::Shell::Zsh,
clap_complete::Shell::Fish,
clap_complete::Shell::PowerShell,
clap_complete::Shell::Elvish,
] {
let out = cmd_completions(shell);
assert_eq!(out.code, exit::OK, "{shell} completions failed");
assert!(
out.human.contains("doctor") && out.human.contains("validate"),
"{shell} script does not mention the subcommands it completes"
);
}
}
#[test]
fn schema_generates_and_names_the_format() {
let out = cmd_schema("result-v1");
assert_eq!(out.code, exit::OK);
let text = serde_json::to_string(&out.json).unwrap();
assert!(
text.contains("Result1"),
"schema should describe the result type"
);
assert!(
text.contains("n_error"),
"the required honesty field must be in the schema"
);
}
#[test]
fn an_unknown_schema_name_is_a_usage_error_not_a_crash() {
assert_eq!(cmd_schema("result-v9").code, exit::USAGE);
}
#[test]
fn validating_a_missing_file_fails_without_panicking() {
let out = cmd_validate(&PathBuf::from("/definitely/not/here.json"));
assert_eq!(out.code, exit::FAILURE);
}
#[test]
fn a_non_result_document_exits_schema_invalid() {
let dir = tempfile::tempdir().unwrap();
let p = dir.path().join("x.json");
std::fs::write(&p, r#"{"hello":"world"}"#).unwrap();
assert_eq!(cmd_validate(&p).code, exit::SCHEMA_INVALID);
}
fn corpus_named(root: &Path, digests: &[&str]) {
std::fs::create_dir_all(root).unwrap();
for (i, d) in digests.iter().enumerate() {
let role = if i == 0 { "clean" } else { "stego" };
std::fs::write(root.join(format!("i{i}.png")), b"\x89PNG\r\n\x1a\n").unwrap();
std::fs::write(
root.join(format!("i{i}.json")),
format!(r#"{{"role":"{role}","sha256":"{d}"}}"#),
)
.unwrap();
}
}
#[test]
fn the_starter_corpus_is_carried_in_the_binary_and_can_be_written_out() {
assert!(
!EMBEDDED_STARTER.is_empty(),
"no starter corpus was compiled in, so a first run needs a checkout"
);
let dir = tempfile::tempdir().unwrap();
let dest = dir.path().join("out");
let n = write_embedded_starter(&dest).expect("written");
assert_eq!(n, EMBEDDED_STARTER.len());
let digest = score::corpus_digest(&dest)
.expect("readable")
.expect("every record states a digest");
assert!(digest.starts_with("sha256:"), "got {digest}");
}
#[test]
fn writing_the_starter_corpus_over_something_is_refused() {
let dir = tempfile::tempdir().unwrap();
let dest = dir.path().join("out");
std::fs::create_dir_all(&dest).unwrap();
std::fs::write(dest.join("something.txt"), b"mine").unwrap();
let err = write_embedded_starter(&dest).expect_err("should have refused");
assert!(err.contains("already has something in it"), "got {err}");
}
fn corpus_honest(root: &Path, bodies: &[&[u8]]) {
use sha2::{Digest, Sha256};
std::fs::create_dir_all(root).unwrap();
for (i, body) in bodies.iter().enumerate() {
let role = if i == 0 { "clean" } else { "stego" };
std::fs::write(root.join(format!("i{i}.png")), body).unwrap();
let d = format!("{:x}", Sha256::digest(body));
std::fs::write(
root.join(format!("i{i}.json")),
format!(r#"{{"role":"{role}","sha256":"{d}"}}"#),
)
.unwrap();
}
}
#[test]
fn a_document_edited_after_it_was_written_is_refused() {
let dir = tempfile::tempdir().unwrap();
let corpus = dir.path().join("corpus");
corpus_honest(&corpus, &[b"cover bytes", b"stego bytes"]);
let digest = score::corpus_digest(&corpus).unwrap().unwrap();
let doc = dir.path().join("r.json");
result_claiming(&doc, &digest);
let mut sealed: Result1 =
serde_json::from_str(&std::fs::read_to_string(&doc).unwrap()).unwrap();
sealed.seal();
sealed.metrics.auc = 0.99;
std::fs::write(&doc, serde_json::to_string(&sealed).unwrap()).unwrap();
let out = cmd_verify(&doc, &corpus, None, false);
assert_eq!(out.code, exit::VERIFY_MISMATCH, "{}", out.human);
assert!(
out.human.contains("does not match its own content digest"),
"{}",
out.human
);
}
#[test]
fn a_document_whose_digest_matches_passes_the_check_it_used_to_skip() {
let dir = tempfile::tempdir().unwrap();
let corpus = dir.path().join("corpus");
corpus_honest(&corpus, &[b"cover bytes", b"stego bytes"]);
let digest = score::corpus_digest(&corpus).unwrap().unwrap();
let doc = dir.path().join("r.json");
result_claiming(&doc, &digest);
let mut sealed: Result1 =
serde_json::from_str(&std::fs::read_to_string(&doc).unwrap()).unwrap();
sealed.seal();
std::fs::write(&doc, serde_json::to_string(&sealed).unwrap()).unwrap();
let out = cmd_verify(&doc, &corpus, None, false);
assert_eq!(out.code, exit::OK, "{}", out.human);
}
#[test]
fn an_edited_records_file_is_not_the_one_the_document_names() {
let dir = tempfile::tempdir().unwrap();
let corpus = dir.path().join("corpus");
corpus_honest(&corpus, &[b"cover bytes", b"stego bytes"]);
let digest = score::corpus_digest(&corpus).unwrap().unwrap();
let doc = dir.path().join("r.json");
result_claiming(&doc, &digest);
let records = dir.path().join("r.records.jsonl");
std::fs::write(
&records,
"{\"id\":\"covers/000000\",\"score\":0.01}\n\
{\"id\":\"stego/000000\",\"score\":0.02}\n",
)
.unwrap();
let honest = stegobench_core::result::records_digest(&records).unwrap();
assert_eq!(honest.count, 2);
let mut sealed: Result1 =
serde_json::from_str(&std::fs::read_to_string(&doc).unwrap()).unwrap();
sealed.provenance.records = Some(honest);
sealed.seal();
std::fs::write(&doc, serde_json::to_string(&sealed).unwrap()).unwrap();
let out = cmd_verify(&doc, &corpus, Some(&records), false);
assert_eq!(out.code, exit::OK, "the honest pair: {}", out.human);
std::fs::write(
&records,
"{\"id\":\"covers/000000\",\"score\":0.01}\n\
{\"id\":\"stego/000000\",\"score\":0.99}\n",
)
.unwrap();
let out = cmd_verify(&doc, &corpus, Some(&records), false);
assert_eq!(out.code, exit::VERIFY_MISMATCH, "{}", out.human);
assert!(
out.human.contains("not the scores"),
"the refusal should name what disagreed: {}",
out.human
);
}
#[test]
fn a_records_file_offered_to_a_document_that_names_none_is_refused() {
let dir = tempfile::tempdir().unwrap();
let corpus = dir.path().join("corpus");
corpus_honest(&corpus, &[b"cover bytes", b"stego bytes"]);
let digest = score::corpus_digest(&corpus).unwrap().unwrap();
let doc = dir.path().join("r.json");
result_claiming(&doc, &digest);
let records = dir.path().join("r.records.jsonl");
std::fs::write(&records, "{\"id\":\"covers/000000\",\"score\":0.01}\n").unwrap();
let out = cmd_verify(&doc, &corpus, Some(&records), false);
assert_eq!(out.code, exit::VERIFY_MISMATCH, "{}", out.human);
assert!(out.human.contains("records no digest"), "{}", out.human);
}
#[test]
fn a_trailing_newline_is_not_counted_as_a_record() {
let dir = tempfile::tempdir().unwrap();
let records = dir.path().join("r.jsonl");
std::fs::write(&records, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n").unwrap();
assert_eq!(
stegobench_core::result::records_digest(&records)
.unwrap()
.count,
2
);
std::fs::write(&records, "{\"id\":\"a\"}\n{\"id\":\"b\"}").unwrap();
assert_eq!(
stegobench_core::result::records_digest(&records)
.unwrap()
.count,
2
);
}
#[test]
fn a_document_carrying_no_content_digest_is_not_accused_of_a_mismatch() {
let dir = tempfile::tempdir().unwrap();
let corpus = dir.path().join("corpus");
corpus_honest(&corpus, &[b"cover bytes", b"stego bytes"]);
let digest = score::corpus_digest(&corpus).unwrap().unwrap();
let doc = dir.path().join("r.json");
result_claiming(&doc, &digest);
assert!(!std::fs::read_to_string(&doc)
.unwrap()
.contains("content_digest"));
let out = cmd_verify(&doc, &corpus, None, false);
assert_eq!(out.code, exit::OK, "{}", out.human);
}
#[test]
fn a_changed_timestamp_is_not_silently_blessed() {
let dir = tempfile::tempdir().unwrap();
let corpus = dir.path().join("corpus");
corpus_honest(&corpus, &[b"cover bytes", b"stego bytes"]);
let digest = score::corpus_digest(&corpus).unwrap().unwrap();
let doc = dir.path().join("r.json");
result_claiming(&doc, &digest);
let mut sealed: Result1 =
serde_json::from_str(&std::fs::read_to_string(&doc).unwrap()).unwrap();
sealed.seal();
sealed.provenance.started_utc = "2021-01-01T00:00:00Z".into();
std::fs::write(&doc, serde_json::to_string(&sealed).unwrap()).unwrap();
let out = cmd_validate(&doc);
assert_eq!(out.code, exit::OK, "{}", out.human);
assert!(out.human.contains("started_utc"), "{}", out.human);
assert!(out.human.contains("elapsed_seconds"), "{}", out.human);
assert_eq!(out.json["seal"]["sealed"], serde_json::json!(true));
assert_eq!(
out.json["seal"]["excluded"],
serde_json::json!(["provenance.started_utc", "provenance.elapsed_seconds"])
);
assert!(!out.human.contains(" "), "{}", out.human);
}
#[test]
fn an_unsealed_result_says_so_rather_than_claiming_a_timing_gap() {
let dir = tempfile::tempdir().unwrap();
let corpus = dir.path().join("corpus");
corpus_honest(&corpus, &[b"cover bytes", b"stego bytes"]);
let digest = score::corpus_digest(&corpus).unwrap().unwrap();
let doc = dir.path().join("r.json");
result_claiming(&doc, &digest);
let out = cmd_validate(&doc);
assert_eq!(out.code, exit::OK, "{}", out.human);
assert!(out.human.contains("no content digest"), "{}", out.human);
assert_eq!(out.json["seal"]["sealed"], serde_json::json!(false));
}
#[test]
fn the_per_arm_block_is_absent_for_a_corpus_of_one_arm() {
assert!(per_arm_lines(&[]).is_empty());
}
#[test]
fn the_per_arm_block_names_every_arm_and_omits_an_interval_it_does_not_have() {
use stegobench_core::result::ArmMetrics;
let arms = vec![
ArmMetrics {
arm: "lsb-0100".into(),
auc: 0.5,
auc_ci95: Some([0.1135, 0.8865]),
n_clean: 6,
n_stego: 6,
},
ArmMetrics {
arm: "a-much-longer-arm".into(),
auc: 0.6944,
auc_ci95: None,
n_clean: 6,
n_stego: 6,
},
];
let text = per_arm_lines(&arms).join("\n");
assert!(text.contains("lsb-0100"), "{text}");
assert!(text.contains("a-much-longer-arm"), "{text}");
assert!(text.contains("0.5000"), "{text}");
assert!(text.contains("[0.1135, 0.8865]"), "{text}");
assert!(
!text.contains("0.6944 ["),
"an interval was fabricated for the arm that had none: {text}"
);
assert!(
text.contains("same 6 clean image(s)"),
"the shared clean set should be stated once: {text}"
);
}
#[test]
fn verify_reads_the_images_and_says_so_when_they_match() {
let dir = tempfile::tempdir().unwrap();
let corpus = dir.path().join("corpus");
corpus_honest(&corpus, &[b"cover bytes", b"stego bytes"]);
let digest = score::corpus_digest(&corpus)
.expect("readable")
.expect("named");
let doc = dir.path().join("r.json");
result_claiming(&doc, &digest);
let out = cmd_verify(&doc, &corpus, None, false);
assert_eq!(out.code, exit::OK, "{}", out.human);
assert_eq!(out.json["checked"], serde_json::json!("bytes"));
assert!(
out.human.contains("hash to what their records state"),
"the answer does not say the bytes were read: {}",
out.human
);
}
#[test]
fn verify_catches_an_image_swapped_under_a_record_that_still_agrees() {
let dir = tempfile::tempdir().unwrap();
let corpus = dir.path().join("corpus");
corpus_honest(&corpus, &[b"cover bytes", b"stego bytes"]);
let digest = score::corpus_digest(&corpus)
.expect("readable")
.expect("named");
let doc = dir.path().join("r.json");
result_claiming(&doc, &digest);
std::fs::write(corpus.join("i1.png"), b"a much easier image").unwrap();
let after = score::corpus_digest(&corpus)
.expect("readable")
.expect("named");
assert_eq!(after, digest, "the swap changed the record level digest");
let out = cmd_verify(&doc, &corpus, None, false);
assert_eq!(out.code, exit::VERIFY_MISMATCH, "{}", out.human);
assert_eq!(out.json["ok"], serde_json::json!(false));
assert!(
out.human.contains("the images do not match the records"),
"the refusal does not name the fault: {}",
out.human
);
let shallow = cmd_verify(&doc, &corpus, None, true);
assert_eq!(shallow.code, exit::OK, "{}", shallow.human);
assert!(
shallow.human.contains("NOT re-read"),
"the shallow answer overclaims: {}",
shallow.human
);
}
fn result_claiming(path: &Path, digest: &str) {
let doc = serde_json::json!({
"schema": stegobench_core::result::RESULT_SCHEMA_ID,
"subject": {"name": "x", "version": "sha256:a", "kind": "detector"},
"corpus": {"name": "c", "source": "supplied", "digest": digest, "pairs": 2},
"arm": {"embedder": "wow", "domain": "spatial", "format": "png"},
"metrics": {"auc": 0.9, "tpr_at_fpr": {}, "n_clean": 1, "n_stego": 1, "n_error": 0},
"provenance": {
"plugins": [{"name": "x", "image": "sha256:a", "determinism": "nondeterministic", "pinned_by": "executable-hash", "isolation": "host"}],
"harness_version": "0.1.0",
"started_utc": "2026-09-25T00:00:00Z",
"elapsed_seconds": 1.0,
"network_reachable": true
},
"declarations": {
"split_discipline": "not-applicable",
"pairing": "unverified",
"configuration": "custom",
"self_reported": false
}
});
std::fs::write(path, serde_json::to_string(&doc).unwrap()).unwrap();
}
#[test]
fn verify_agrees_when_the_document_and_the_corpus_are_the_same_corpus() {
let dir = tempfile::tempdir().unwrap();
let corpus = dir.path().join("corpus");
corpus_named(&corpus, &["aa", "bb"]);
let digest = score::corpus_digest(&corpus)
.expect("readable")
.expect("named");
let doc = dir.path().join("r.json");
result_claiming(&doc, &digest);
let out = cmd_verify(&doc, &corpus, None, true);
assert_eq!(out.code, exit::OK, "{}", out.human);
assert_eq!(out.json["ok"], serde_json::json!(true));
}
#[test]
fn verify_refuses_a_corpus_the_document_was_not_measured_on() {
let dir = tempfile::tempdir().unwrap();
let measured = dir.path().join("measured");
let other = dir.path().join("other");
corpus_named(&measured, &["aa", "bb"]);
corpus_named(&other, &["aa", "bc"]);
let digest = score::corpus_digest(&measured)
.expect("readable")
.expect("named");
let doc = dir.path().join("r.json");
result_claiming(&doc, &digest);
let out = cmd_verify(&doc, &other, None, true);
assert_eq!(out.code, exit::VERIFY_MISMATCH);
assert!(out.human.contains("not about each other"), "{}", out.human);
assert_eq!(out.json["ok"], serde_json::json!(false));
}
#[test]
fn verify_says_so_when_there_is_nothing_to_check() {
let dir = tempfile::tempdir().unwrap();
let corpus = dir.path().join("corpus");
corpus_named(&corpus, &["aa", "bb"]);
let doc = dir.path().join("r.json");
result_claiming(&doc, "");
let out = cmd_verify(&doc, &corpus, None, true);
assert_eq!(out.code, exit::VERIFY_MISMATCH);
assert!(
out.human.contains("names no corpus digest"),
"{}",
out.human
);
}
#[test]
fn verify_refuses_a_corpus_that_cannot_be_named() {
let dir = tempfile::tempdir().unwrap();
let corpus = dir.path().join("corpus");
std::fs::create_dir_all(&corpus).unwrap();
std::fs::write(corpus.join("a.png"), b"\x89PNG\r\n\x1a\n").unwrap();
std::fs::write(corpus.join("a.json"), r#"{"role":"clean"}"#).unwrap();
let doc = dir.path().join("r.json");
result_claiming(&doc, "sha256:whatever");
let out = cmd_verify(&doc, &corpus, None, true);
assert_eq!(out.code, exit::VERIFY_MISMATCH);
assert!(out.human.contains("cannot be named"), "{}", out.human);
}
#[test]
fn verify_sends_a_document_that_is_not_a_result_to_validate() {
let dir = tempfile::tempdir().unwrap();
let doc = dir.path().join("x.json");
std::fs::write(&doc, r#"{"hello":"world"}"#).unwrap();
let out = cmd_verify(&doc, dir.path(), None, true);
assert_eq!(out.code, exit::SCHEMA_INVALID);
assert!(out.human.contains("stegobench validate"), "{}", out.human);
}
#[test]
fn a_report_with_a_dropped_file_still_puts_the_table_on_stdout() {
let dir = tempfile::tempdir().unwrap();
std::fs::copy(
Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../../results/v1/round3-q95-structural-0000-aletheia-rs.json"),
dir.path().join("good.json"),
)
.unwrap();
std::fs::write(dir.path().join("bad.json"), "{nope").unwrap();
let out = cmd_report(&[dir.path().to_path_buf()], ReportFormat::Text, None);
assert_eq!(out.code, exit::SCHEMA_INVALID);
assert!(out.payload_on_stdout, "the table was diverted to stderr");
assert!(out.human.contains("THIS REPORT IS INCOMPLETE"));
assert!(out.human.contains("bad.json"));
assert!(out.human.contains("round3-q95"));
assert_eq!(out.json["complete"], serde_json::json!(false));
}
#[test]
fn writing_a_report_to_a_file_confirms_on_stderr_and_says_if_it_is_short() {
let dir = tempfile::tempdir().unwrap();
std::fs::copy(
Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../../results/v1/round3-q95-structural-0000-aletheia-rs.json"),
dir.path().join("good.json"),
)
.unwrap();
let target = dir.path().join("out").join("report.md");
std::fs::create_dir_all(target.parent().unwrap()).unwrap();
let out = cmd_report(
&[dir.path().join("good.json")],
ReportFormat::Markdown,
Some(&target),
);
assert_eq!(out.code, exit::OK);
assert!(!out.payload_on_stdout);
assert!(out.human.contains("Every file found was readable"));
let written = std::fs::read_to_string(&target).unwrap();
assert!(written.starts_with("# Steganalysis results"));
assert!(written.contains("round3-q95"));
std::fs::write(dir.path().join("bad.json"), "{").unwrap();
let out = cmd_report(
&[dir.path().to_path_buf()],
ReportFormat::Markdown,
Some(&target),
);
assert_eq!(out.code, exit::SCHEMA_INVALID);
assert!(out.human.contains("tables are incomplete"), "{}", out.human);
}
#[test]
fn a_report_over_nothing_is_a_usage_error_rather_than_an_empty_table() {
let dir = tempfile::tempdir().unwrap();
let out = cmd_report(&[dir.path().to_path_buf()], ReportFormat::Text, None);
assert_eq!(out.code, exit::USAGE);
assert!(out.human.contains("no table to print"), "{}", out.human);
}
#[test]
fn a_report_to_an_unwritable_destination_fails_rather_than_reporting_success() {
let dir = tempfile::tempdir().unwrap();
std::fs::copy(
Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../../results/v1/round3-q95-structural-0000-aletheia-rs.json"),
dir.path().join("good.json"),
)
.unwrap();
let out = cmd_report(
&[dir.path().to_path_buf()],
ReportFormat::Csv,
Some(&dir.path().join("no-such-dir").join("r.csv")),
);
assert_eq!(out.code, exit::FAILURE);
assert!(out.human.contains("could not write the report"));
}
#[cfg(unix)]
fn registry_with_one_present_and_one_missing(dir: &Path) -> PathBuf {
use std::os::unix::fs::PermissionsExt;
let reg = dir.join("registry");
std::fs::create_dir_all(®).unwrap();
let script = dir.join("size.sh");
std::fs::write(
&script,
"#!/bin/sh\n[ \"$1\" = \"--version\" ] && { echo v1; exit 0; }\n\
wc -c < \"$1\" | tr -d ' ' | awk '{print $1/1000}'\n",
)
.unwrap();
let mut perms = std::fs::metadata(&script).unwrap().permissions();
perms.set_mode(0o755);
std::fs::set_permissions(&script, perms).unwrap();
std::fs::write(
reg.join("sizer.toml"),
format!(
"name = \"sizer\"\nkind = \"detector\"\nlicence = \"MIT\"\n\
[binary]\ncommand = [{:?}]\nversion_args = [\"--version\"]\n\
[emits]\noutput = \"score\"\nhigher_means_stego = true\n\
[invoke]\nargv = [\"{{file}}\"]\nparser = \"number\"\n\
[selftest]\nmust_detect = \"a.png\"\nmust_clear = \"b.png\"\n",
script.display().to_string()
),
)
.unwrap();
std::fs::write(
reg.join("ghost.toml"),
"name = \"ghost\"\nkind = \"detector\"\nlicence = \"MIT\"\n\
[binary]\ncommand = [\"definitely-not-installed-xyzzy\"]\n\
version_args = [\"--version\"]\n\
[emits]\noutput = \"score\"\nhigher_means_stego = true\n\
[invoke]\nargv = [\"{file}\"]\nparser = \"number\"\n\
[selftest]\nmust_detect = \"a.png\"\nmust_clear = \"b.png\"\n",
)
.unwrap();
reg
}
#[cfg(unix)]
fn test_png(padding: usize) -> Vec<u8> {
let mut v = b"\x89PNG\r\n\x1a\n".to_vec();
v.extend_from_slice(&13u32.to_be_bytes());
v.extend_from_slice(b"IHDR");
v.extend_from_slice(&32u32.to_be_bytes());
v.extend_from_slice(&32u32.to_be_bytes());
v.extend_from_slice(&[8, 2, 0, 0, 0]);
v.extend_from_slice(&[0, 0, 0, 0]);
v.extend_from_slice(&vec![0u8; padding]);
v
}
#[cfg(unix)]
fn scratch_corpus(root: &Path) {
std::fs::create_dir_all(root).unwrap();
for i in 0..3 {
std::fs::write(root.join(format!("c{i}.png")), test_png(0)).unwrap();
std::fs::write(
root.join(format!("c{i}.json")),
r#"{"role":"clean","sha256":"0"}"#,
)
.unwrap();
std::fs::write(root.join(format!("s{i}.png")), test_png(64)).unwrap();
std::fs::write(
root.join(format!("s{i}.json")),
format!(r#"{{"role":"stego","source_png":"c{i}.png","sha256":"0"}}"#),
)
.unwrap();
}
}
#[cfg(unix)]
#[test]
fn one_detector_writing_into_a_directory_puts_its_document_inside_it() {
let tmp = tempfile::tempdir().unwrap();
let reg = registry_with_one_present_and_one_missing(tmp.path());
let corpus = tmp.path().join("corpus");
scratch_corpus(&corpus);
let out_dir = tmp.path().join("results");
std::fs::create_dir_all(&out_dir).unwrap();
let out = cmd_score(
&resolved_at(®),
ScoreRequest {
corpus: &corpus,
detectors: &["sizer".to_string()],
corpus_id: None,
trained_on: None,
records: None,
out: Some(&out_dir),
timeout: 5,
jobs: 1,
keep_raw: false,
seed: None,
fixtures: None,
limit: None,
split: None,
},
);
assert_eq!(out.code, exit::OK, "{}", out.human);
let written = out_dir.join("sizer.json");
assert!(
written.exists(),
"one detector did not write into the directory it was given: {}",
out.human
);
assert_eq!(cmd_validate(&written).code, exit::OK);
assert!(
out_dir.join("sizer.records.jsonl").exists(),
"the records did not follow the document into the directory"
);
}
#[cfg(unix)]
#[test]
fn the_score_summary_says_where_its_two_unexplained_figures_are_defined() {
let tmp = tempfile::tempdir().unwrap();
let reg = registry_with_one_present_and_one_missing(tmp.path());
let corpus = tmp.path().join("corpus");
scratch_corpus(&corpus);
let out_dir = tmp.path().join("results");
std::fs::create_dir_all(&out_dir).unwrap();
let out = cmd_score(
&resolved_at(®),
ScoreRequest {
corpus: &corpus,
detectors: &["sizer".to_string()],
corpus_id: None,
trained_on: None,
records: None,
out: Some(&out_dir),
timeout: 5,
jobs: 1,
keep_raw: false,
seed: None,
fixtures: None,
limit: None,
split: None,
},
);
assert!(
out.human.contains("AUC "),
"nothing to explain: {}",
out.human
);
assert!(
out.human.contains("`stegobench help results`"),
"the summary explains neither figure and points nowhere: {}",
out.human
);
let figure = out.human.find("AUC ").expect("an AUC");
let pointer = out
.human
.find("`stegobench help results`")
.expect("a pointer");
assert!(pointer > figure, "the pointer comes before the figure");
assert!(
out.human[figure..pointer].lines().count() <= 6,
"the pointer is not beside the figures: {}",
out.human
);
}
#[cfg(unix)]
#[test]
fn several_detectors_aimed_at_a_file_are_refused_before_anything_is_scored() {
let tmp = tempfile::tempdir().unwrap();
let reg = registry_with_one_present_and_one_missing(tmp.path());
let corpus = tmp.path().join("corpus");
scratch_corpus(&corpus);
let target = tmp.path().join("one-file.json");
std::fs::write(&target, "{}").unwrap();
let out = cmd_score(
&resolved_at(®),
ScoreRequest {
corpus: &corpus,
detectors: &["ghost".to_string(), "sizer".to_string()],
corpus_id: None,
trained_on: None,
records: None,
out: Some(&target),
timeout: 5,
jobs: 1,
keep_raw: false,
seed: None,
fixtures: None,
limit: None,
split: None,
},
);
assert_eq!(out.code, exit::USAGE, "{}", out.human);
let expected = "is a file. Scoring 2 detectors writes one result \
document each, so --out has to name a directory.";
assert!(out.human.contains(expected), "{}", out.human);
assert_eq!(
std::fs::read_to_string(&target).unwrap(),
"{}",
"the run wrote over the file it should have refused"
);
}
#[cfg(unix)]
#[test]
fn one_missing_detector_does_not_lose_the_others_and_never_exits_zero() {
let tmp = tempfile::tempdir().unwrap();
let reg = registry_with_one_present_and_one_missing(tmp.path());
let corpus = tmp.path().join("corpus");
scratch_corpus(&corpus);
let out_dir = tmp.path().join("results");
let out = cmd_score(
&resolved_at(®),
ScoreRequest {
corpus: &corpus,
detectors: &["ghost".to_string(), "sizer".to_string()],
corpus_id: None,
trained_on: None,
records: None,
out: Some(&out_dir),
timeout: 5,
jobs: 1,
keep_raw: false,
seed: None,
fixtures: None,
limit: None,
split: None,
},
);
assert_eq!(
out.code,
exit::PREFLIGHT_REFUSED,
"a run that measured one of two detectors exited {}: {}",
out.code,
out.human
);
assert_ne!(out.code, exit::OK, "{}", out.human);
let written = out_dir.join("sizer.json");
assert!(written.exists(), "{}", out.human);
assert_eq!(cmd_validate(&written).code, exit::OK);
assert!(
out.human.contains("1 of 2 detector(s) measured"),
"{}",
out.human
);
assert!(out.human.contains("NOT MEASURED"), "{}", out.human);
assert_eq!(out.json["measured"], serde_json::json!(1));
assert_eq!(out.json["skipped"], serde_json::json!(1));
assert_eq!(out.json["requested"], serde_json::json!(2));
assert_eq!(out.json["ok"], serde_json::json!(false));
}
#[cfg(unix)]
#[test]
fn a_document_left_from_an_earlier_run_of_a_now_skipped_detector_is_named() {
let tmp = tempfile::tempdir().unwrap();
let reg = registry_with_one_present_and_one_missing(tmp.path());
let corpus = tmp.path().join("corpus");
scratch_corpus(&corpus);
let out_dir = tmp.path().join("results");
std::fs::create_dir_all(&out_dir).unwrap();
std::fs::write(out_dir.join("ghost.json"), "{}").unwrap();
let out = cmd_score(
&resolved_at(®),
ScoreRequest {
corpus: &corpus,
detectors: &["ghost".to_string(), "sizer".to_string()],
corpus_id: None,
trained_on: None,
records: None,
out: Some(&out_dir),
timeout: 5,
jobs: 1,
keep_raw: false,
seed: None,
fixtures: None,
limit: None,
split: None,
},
);
assert!(
out.human.contains("are from an earlier run"),
"a stale document went unmentioned: {}",
out.human
);
assert!(out.human.contains("ghost.json"), "{}", out.human);
assert!(
out_dir.join("ghost.json").exists(),
"the stale document was deleted rather than named"
);
}
#[cfg(unix)]
#[test]
fn a_document_from_a_detector_this_run_never_asked_for_is_named_too() {
let tmp = tempfile::tempdir().unwrap();
let reg = registry_with_one_present_and_one_missing(tmp.path());
let corpus = tmp.path().join("corpus");
scratch_corpus(&corpus);
let out_dir = tmp.path().join("results");
std::fs::create_dir_all(&out_dir).unwrap();
std::fs::write(out_dir.join("somebody-elses-tool.json"), "{}").unwrap();
let out = cmd_score(
&resolved_at(®),
ScoreRequest {
corpus: &corpus,
detectors: &["sizer".to_string(), "ghost".to_string()],
corpus_id: None,
trained_on: None,
records: None,
out: Some(&out_dir),
timeout: 5,
jobs: 1,
keep_raw: false,
seed: None,
fixtures: None,
limit: None,
split: None,
},
);
assert!(
out.human.contains("somebody-elses-tool.json"),
"a document nobody asked about today went unmentioned: {}",
out.human
);
}
#[cfg(unix)]
#[test]
fn a_run_that_wrote_every_document_in_its_directory_warns_about_nothing() {
let tmp = tempfile::tempdir().unwrap();
let reg = registry_with_one_present_and_one_missing(tmp.path());
std::fs::copy(reg.join("sizer.toml"), reg.join("sizer2.toml")).unwrap();
let text = std::fs::read_to_string(reg.join("sizer2.toml"))
.unwrap()
.replace("name = \"sizer\"", "name = \"sizer2\"");
std::fs::write(reg.join("sizer2.toml"), text).unwrap();
let corpus = tmp.path().join("corpus");
scratch_corpus(&corpus);
let out_dir = tmp.path().join("results");
let out = cmd_score(
&resolved_at(®),
ScoreRequest {
corpus: &corpus,
detectors: &["sizer".to_string(), "sizer2".to_string()],
corpus_id: None,
trained_on: None,
records: None,
out: Some(&out_dir),
timeout: 5,
jobs: 1,
keep_raw: false,
seed: None,
fixtures: None,
limit: None,
split: None,
},
);
assert_eq!(out.code, exit::OK, "{}", out.human);
assert!(
!out.human.contains("NOT measured by this run"),
"warned about documents it had just written itself: {}",
out.human
);
}
#[cfg(unix)]
#[test]
fn two_detectors_over_one_corpus_do_not_resume_from_each_others_answers() {
let tmp = tempfile::tempdir().unwrap();
let reg = registry_with_one_present_and_one_missing(tmp.path());
std::fs::copy(reg.join("sizer.toml"), reg.join("sizer2.toml")).unwrap();
let text = std::fs::read_to_string(reg.join("sizer2.toml"))
.unwrap()
.replace("name = \"sizer\"", "name = \"sizer2\"");
std::fs::write(reg.join("sizer2.toml"), text).unwrap();
let corpus = tmp.path().join("corpus");
scratch_corpus(&corpus);
let out_dir = tmp.path().join("results");
let out = cmd_score(
&resolved_at(®),
ScoreRequest {
corpus: &corpus,
detectors: &["sizer".to_string(), "sizer2".to_string()],
corpus_id: None,
trained_on: None,
records: None,
out: Some(&out_dir),
timeout: 5,
jobs: 1,
keep_raw: false,
seed: None,
fixtures: None,
limit: None,
split: None,
},
);
assert_eq!(out.code, exit::OK, "{}", out.human);
for d in out.json["detectors"].as_array().expect("rows") {
assert_eq!(d["scored"], serde_json::json!(6), "{d}");
assert_eq!(
d["resumed"],
serde_json::json!(0),
"{} resumed from another detector's records: {d}",
d["name"]
);
}
assert!(out_dir.join("sizer.records.jsonl").exists());
assert!(out_dir.join("sizer2.records.jsonl").exists());
}
#[cfg(unix)]
#[test]
fn a_second_run_of_the_same_command_resumes_each_detector_separately() {
let tmp = tempfile::tempdir().unwrap();
let reg = registry_with_one_present_and_one_missing(tmp.path());
let corpus = tmp.path().join("corpus");
scratch_corpus(&corpus);
let out_dir = tmp.path().join("results");
let args = ["sizer".to_string()];
let first = cmd_score(
&resolved_at(®),
ScoreRequest {
corpus: &corpus,
detectors: &args,
corpus_id: None,
trained_on: None,
records: None,
out: Some(&out_dir),
timeout: 5,
jobs: 1,
keep_raw: false,
seed: None,
fixtures: None,
limit: None,
split: None,
},
);
assert_eq!(first.code, exit::OK, "{}", first.human);
let second = cmd_score(
&resolved_at(®),
ScoreRequest {
corpus: &corpus,
detectors: &args,
corpus_id: None,
trained_on: None,
records: None,
out: Some(&out_dir),
timeout: 5,
jobs: 1,
keep_raw: false,
seed: None,
fixtures: None,
limit: None,
split: None,
},
);
assert_eq!(second.code, exit::OK, "{}", second.human);
assert!(
second.human.contains("6 resumed"),
"the second run re-scored instead of resuming: {}",
second.human
);
}
#[cfg(unix)]
#[test]
fn several_detectors_with_no_out_write_beside_the_corpus_and_say_where() {
let tmp = tempfile::tempdir().unwrap();
let reg = registry_with_one_present_and_one_missing(tmp.path());
let corpus = tmp.path().join("corpus");
scratch_corpus(&corpus);
let out = cmd_score(
&resolved_at(®),
ScoreRequest {
corpus: &corpus,
detectors: &["sizer".to_string(), "ghost".to_string()],
corpus_id: None,
trained_on: None,
records: None,
out: None,
timeout: 5,
jobs: 1,
keep_raw: false,
seed: None,
fixtures: None,
limit: None,
split: None,
},
);
let expected = tmp.path().join("corpus.results");
assert!(expected.join("sizer.json").exists(), "{}", out.human);
assert!(
out.human.contains(&expected.display().to_string()),
"the command did not say where it wrote: {}",
out.human
);
assert!(out.human.contains("stegobench report"), "{}", out.human);
}
#[cfg(unix)]
#[test]
fn a_single_detector_still_puts_the_result_document_on_stdout() {
let tmp = tempfile::tempdir().unwrap();
let reg = registry_with_one_present_and_one_missing(tmp.path());
let corpus = tmp.path().join("corpus");
scratch_corpus(&corpus);
let out = cmd_score(
&resolved_at(®),
ScoreRequest {
corpus: &corpus,
detectors: &["sizer".to_string()],
corpus_id: None,
trained_on: None,
records: None,
out: None,
timeout: 5,
jobs: 1,
keep_raw: false,
seed: None,
fixtures: None,
limit: None,
split: None,
},
);
assert_eq!(out.code, exit::OK, "{}", out.human);
assert_eq!(
out.json["schema"],
serde_json::json!(stegobench_core::result::RESULT_SCHEMA_ID),
"a single run stopped emitting the result document itself"
);
assert!(out.json["metrics"]["auc"].is_number());
}
#[cfg(unix)]
#[test]
fn one_detector_is_told_its_out_path_is_the_document_and_not_a_directory() {
let tmp = tempfile::tempdir().unwrap();
let reg = registry_with_one_present_and_one_missing(tmp.path());
let corpus = tmp.path().join("corpus");
scratch_corpus(&corpus);
let named = tmp.path().join("results-test");
let out = cmd_score(
&resolved_at(®),
ScoreRequest {
corpus: &corpus,
detectors: &["sizer".to_string()],
corpus_id: None,
trained_on: None,
records: None,
out: Some(&named),
timeout: 5,
jobs: 1,
keep_raw: false,
seed: None,
fixtures: None,
limit: None,
split: None,
},
);
assert_eq!(out.code, exit::OK, "{}", out.human);
assert!(named.is_file(), "the document is not at the named path");
assert!(
out.human.contains("is the result document itself"),
"nothing said the path was a file rather than a directory: {}",
out.human
);
}
#[cfg(unix)]
#[test]
fn several_detectors_are_not_told_their_out_path_is_a_document() {
let tmp = tempfile::tempdir().unwrap();
let reg = registry_with_one_present_and_one_missing(tmp.path());
let corpus = tmp.path().join("corpus");
scratch_corpus(&corpus);
let named = tmp.path().join("results-dir");
let out = cmd_score(
&resolved_at(®),
ScoreRequest {
corpus: &corpus,
detectors: &["sizer".to_string(), "ghost".to_string()],
corpus_id: None,
trained_on: None,
records: None,
out: Some(&named),
timeout: 5,
jobs: 1,
keep_raw: false,
seed: None,
fixtures: None,
limit: None,
split: None,
},
);
assert!(
!out.human.contains("is the result document itself"),
"a directory run was told its path was a document: {}",
out.human
);
assert!(named.is_dir(), "{}", out.human);
}
#[cfg(unix)]
#[test]
fn a_plan_over_a_leaking_corpus_refuses_with_the_code_the_run_would() {
let tmp = tempfile::tempdir().unwrap();
let reg = registry_with_one_present_and_one_missing(tmp.path());
let corpus = tmp.path().join("corpus");
std::fs::create_dir_all(&corpus).unwrap();
for i in 0..3 {
std::fs::write(corpus.join(format!("c{i}.png")), test_png(0)).unwrap();
std::fs::write(
corpus.join(format!("c{i}.json")),
format!(r#"{{"role":"clean","sha256":"{i}","split":"train"}}"#),
)
.unwrap();
std::fs::write(corpus.join(format!("s{i}.png")), test_png(64)).unwrap();
std::fs::write(
corpus.join(format!("s{i}.json")),
format!(
r#"{{"role":"stego","source_png":"c{i}.png","sha256":"{i}","split":"test"}}"#
),
)
.unwrap();
}
let out = cmd_plan(
&resolved_at(®),
&[
"score".into(),
"--corpus".into(),
corpus.display().to_string(),
"--detector".into(),
"sizer".into(),
],
);
assert_eq!(
out.code,
exit::PREFLIGHT_REFUSED,
"a plan over a corpus score refuses returned {}: {}",
out.code,
out.human
);
assert!(
out.human
.contains("different side of the train and test split"),
"{}",
out.human
);
}
#[cfg(unix)]
#[test]
fn a_command_where_no_detector_is_available_refuses_rather_than_reporting_zero() {
let tmp = tempfile::tempdir().unwrap();
let reg = tmp.path().join("registry");
std::fs::create_dir_all(®).unwrap();
std::fs::write(
reg.join("ghost.toml"),
"name = \"ghost\"\nkind = \"detector\"\nlicence = \"MIT\"\n\
[binary]\ncommand = [\"definitely-not-installed-xyzzy\"]\n\
version_args = [\"--version\"]\n\
[emits]\noutput = \"score\"\nhigher_means_stego = true\n\
[invoke]\nargv = [\"{file}\"]\nparser = \"number\"\n\
[selftest]\nmust_detect = \"a.png\"\nmust_clear = \"b.png\"\n",
)
.unwrap();
let corpus = tmp.path().join("corpus");
scratch_corpus(&corpus);
let out = cmd_score(
&resolved_at(®),
ScoreRequest {
corpus: &corpus,
detectors: &["all".to_string()],
corpus_id: None,
trained_on: None,
records: None,
out: None,
timeout: 5,
jobs: 1,
keep_raw: false,
seed: None,
fixtures: None,
limit: None,
split: None,
},
);
assert_eq!(out.code, exit::PREFLIGHT_REFUSED, "{}", out.human);
assert!(out.human.contains("nothing was measured"), "{}", out.human);
assert!(!corpus.with_file_name("corpus.results").exists());
}
#[test]
fn all_beside_a_named_detector_is_a_usage_error_rather_than_a_guess() {
let reg = Registry::load(&shipped_registry()).expect("loads");
let err = resolve_detectors(®, &["all".into(), "zsteg".into()]).expect_err("refused");
assert_eq!(err.code, exit::USAGE);
assert!(err
.human
.contains("already means every registered detector"));
}
#[test]
fn all_is_every_detector_and_never_an_embedder() {
let reg = Registry::load(&shipped_registry()).expect("loads");
let picked = resolve_detectors(®, &["all".into()]).expect("resolved");
assert!(picked.len() >= 5, "resolved only {}", picked.len());
assert!(picked.iter().all(|e| e.kind == Kind::Detector));
let names: Vec<_> = picked.iter().map(|e| e.name.clone()).collect();
let mut sorted = names.clone();
sorted.sort();
assert_eq!(names, sorted);
}
#[test]
fn naming_an_embedder_is_refused_before_the_corpus_is_touched() {
let reg = Registry::load(&shipped_registry()).expect("loads");
let embedder = reg
.of_kind(Kind::Embedder)
.first()
.map(|e| e.name.clone())
.expect("the shipped registry has embedders");
let err = resolve_detectors(®, std::slice::from_ref(&embedder)).expect_err("refused");
assert_eq!(err.code, exit::USAGE);
assert!(err.human.contains("it cannot tell two images apart"));
}
#[test]
fn the_same_detector_named_twice_is_one_measurement() {
let reg = Registry::load(&shipped_registry()).expect("loads");
let picked = resolve_detectors(®, &["zsteg".into(), "zsteg".into()]).expect("resolved");
assert_eq!(picked.len(), 1);
}
#[test]
fn doctor_and_describe_both_say_what_a_tool_needs_and_agree() {
let dir = shipped_registry();
let doctor = cmd_doctor(&resolved_at(&dir), None, false, false);
assert!(
doctor.json["needing_action"].is_number(),
"doctor does not report how many tools need something"
);
for tool in doctor.json["tools"].as_array().expect("rows") {
assert!(
tool["needs"]["readiness"].is_string(),
"{} has no readiness: {tool}",
tool["name"]
);
assert!(tool["needs"]["steps"].is_array());
}
let described = cmd_describe(&resolved_at(&dir), "stegashield", false);
assert_eq!(described.code, exit::OK);
assert!(described.json["needs"]["readiness"].is_string());
assert!(
described.human.contains("Needs from you")
|| described.human.contains("Nothing needed"),
"{}",
described.human
);
let corpus = cmd_describe(&resolved_at(&dir), "reveal", false);
assert_eq!(corpus.code, exit::OK);
assert!(corpus.json["needs"]["steps"].is_array());
assert!(corpus.human.contains("Needs from you"), "{}", corpus.human);
}
#[test]
fn the_list_legend_does_not_call_a_service_a_sandbox() {
let out = cmd_list(&resolved_at(shipped_registry()), "detectors");
assert_eq!(out.code, exit::OK);
assert!(
out.human.contains("SERVICE, not sandboxed"),
"the legend still presents every container entry as sandboxed: {}",
out.human
);
assert!(out.human.contains("stegashield"), "{}", out.human);
}
#[test]
fn the_exit_code_contract_is_driven_or_explicitly_not_yet_reachable() {
assert_eq!(cmd_schema("result-v1").code, exit::OK);
assert_eq!(
cmd_validate(&PathBuf::from("/definitely/not/here.json")).code,
exit::FAILURE
);
assert_eq!(cmd_schema("not-a-real-schema").code, exit::USAGE);
assert_eq!(cmd_help(Some("not-a-real-topic")).code, exit::USAGE);
{
let dir = tempfile::tempdir().unwrap();
let reg = dir.path().join("detectors");
std::fs::create_dir_all(®).unwrap();
std::fs::write(
reg.join("ghost.toml"),
"name = \"ghost\"\nkind = \"detector\"\nlicence = \"MIT\"\n\
[binary]\ncommand = [\"definitely-not-installed-xyzzy\"]\n\
version_args = [\"--version\"]\n\
[emits]\noutput = \"score\"\nhigher_means_stego = true\n\
[invoke]\nargv = [\"{file}\"]\nparser = \"number\"\n\
[selftest]\nmust_detect = \"a.png\"\nmust_clear = \"b.png\"\n",
)
.unwrap();
let ask = |corpus: &Path| {
cmd_score(
&resolved_at(dir.path()),
ScoreRequest {
corpus,
detectors: &["ghost".to_string()],
corpus_id: None,
trained_on: None,
records: None,
out: None,
timeout: 5,
jobs: 1,
keep_raw: false,
seed: None,
fixtures: None,
limit: None,
split: None,
},
)
};
let missing = ask(&dir.path().join("no-such-corpus"));
assert_eq!(missing.code, exit::PREFLIGHT_REFUSED, "{}", missing.human);
assert!(
missing.human.contains("no corpus at"),
"a path that is not there should be answered as itself: {}",
missing.human
);
let empty = dir.path().join("empty-corpus");
std::fs::create_dir_all(&empty).unwrap();
let absent = ask(&empty);
assert_eq!(absent.code, exit::PREFLIGHT_REFUSED, "{}", absent.human);
assert!(
absent.human.contains("not on this machine"),
"{}",
absent.human
);
}
{
let dir = tempfile::tempdir().unwrap();
let p = dir.path().join("x.json");
std::fs::write(&p, r#"{"hello":"world"}"#).unwrap();
assert_eq!(cmd_validate(&p).code, exit::SCHEMA_INVALID);
}
}
#[cfg(unix)]
fn registry_with_an_undrivable_detector(dir: &Path) -> PathBuf {
let reg = registry_with_one_present_and_one_missing(dir);
std::fs::write(
reg.join("paperwork.toml"),
"name = \"paperwork\"\nkind = \"detector\"\nlicence = \"MIT\"\n\
[binary]\ncommand = [\"sh\"]\nversion_args = [\"-c\", \"echo v1\"]\n\
[emits]\noutput = \"score\"\nhigher_means_stego = true\n\
[selftest]\nmust_detect = \"a.png\"\nmust_clear = \"b.png\"\n",
)
.unwrap();
reg
}
#[cfg(unix)]
#[test]
fn a_detector_with_no_invoke_block_is_skipped_before_the_corpus_is_walked() {
let tmp = tempfile::tempdir().unwrap();
let reg = registry_with_an_undrivable_detector(tmp.path());
let corpus = tmp.path().join("corpus");
scratch_corpus(&corpus);
let out = cmd_score(
&resolved_at(®),
ScoreRequest {
corpus: &corpus,
detectors: &["paperwork".to_string()],
corpus_id: None,
trained_on: None,
records: None,
out: None,
timeout: 5,
jobs: 1,
keep_raw: false,
seed: None,
fixtures: None,
limit: None,
split: None,
},
);
assert_eq!(out.code, exit::PREFLIGHT_REFUSED, "{}", out.human);
assert!(
out.human.contains("invoke block"),
"the refusal has to name the real reason: {}",
out.human
);
assert!(
!out.human.contains("0 clean"),
"our registry's gap was reported as a fault in their corpus: {}",
out.human
);
assert!(
!corpus.join("paperwork.records.jsonl").exists(),
"an entry that cannot be driven was still run over the corpus"
);
}
#[cfg(unix)]
#[test]
fn an_undrivable_detector_is_a_skip_beside_a_working_one_too() {
let tmp = tempfile::tempdir().unwrap();
let reg = registry_with_an_undrivable_detector(tmp.path());
let corpus = tmp.path().join("corpus");
scratch_corpus(&corpus);
let out_dir = tmp.path().join("results");
let out = cmd_score(
&resolved_at(®),
ScoreRequest {
corpus: &corpus,
detectors: &["paperwork".to_string(), "sizer".to_string()],
corpus_id: None,
trained_on: None,
records: None,
out: Some(&out_dir),
timeout: 5,
jobs: 1,
keep_raw: false,
seed: None,
fixtures: None,
limit: None,
split: None,
},
);
assert_eq!(out.code, exit::PREFLIGHT_REFUSED, "{}", out.human);
assert!(
out.human.contains("skipped: declares no invoke block"),
"{}",
out.human
);
assert_eq!(out.json["skipped"], serde_json::json!(1), "{}", out.json);
assert_eq!(out.json["failed"], serde_json::json!(0), "{}", out.json);
assert!(
out_dir.join("sizer.json").exists(),
"the detector that could run lost its work: {}",
out.human
);
}
#[cfg(unix)]
#[test]
fn a_limit_that_reaches_only_covers_names_the_flag_rather_than_the_corpus() {
let tmp = tempfile::tempdir().unwrap();
let reg = registry_with_one_present_and_one_missing(tmp.path());
let corpus = tmp.path().join("corpus");
scratch_corpus(&corpus);
let out = cmd_score(
&resolved_at(®),
ScoreRequest {
corpus: &corpus,
detectors: &["sizer".to_string()],
corpus_id: None,
trained_on: None,
records: None,
out: None,
timeout: 5,
jobs: 1,
keep_raw: false,
seed: None,
fixtures: None,
limit: Some(2),
split: None,
},
);
assert_eq!(out.code, exit::PREFLIGHT_REFUSED, "{}", out.human);
assert!(out.human.contains("--limit 2"), "{}", out.human);
assert!(
out.human.contains("covers before its stego arms"),
"{}",
out.human
);
assert_eq!(out.json["reason"], serde_json::json!("one-sided"));
}
#[cfg(unix)]
#[test]
fn a_one_sided_corpus_is_not_blamed_on_a_limit_nobody_gave() {
let tmp = tempfile::tempdir().unwrap();
let reg = registry_with_one_present_and_one_missing(tmp.path());
let corpus = tmp.path().join("covers-only");
std::fs::create_dir_all(&corpus).unwrap();
for i in 0..3 {
std::fs::write(corpus.join(format!("c{i}.png")), test_png(0)).unwrap();
std::fs::write(
corpus.join(format!("c{i}.json")),
r#"{"role":"clean","sha256":"0"}"#,
)
.unwrap();
}
let out = cmd_score(
&resolved_at(®),
ScoreRequest {
corpus: &corpus,
detectors: &["sizer".to_string()],
corpus_id: None,
trained_on: None,
records: None,
out: None,
timeout: 5,
jobs: 1,
keep_raw: false,
seed: None,
fixtures: None,
limit: None,
split: None,
},
);
assert_eq!(out.code, exit::PREFLIGHT_REFUSED, "{}", out.human);
assert!(!out.human.contains("--limit"), "{}", out.human);
assert_eq!(out.json["reason"], serde_json::json!("one-sided"));
}
#[cfg(unix)]
#[test]
fn plan_warns_about_a_confounded_corpus_before_anything_is_scored() {
let tmp = tempfile::tempdir().unwrap();
let reg = registry_with_one_present_and_one_missing(tmp.path());
let corpus = tmp.path().join("confounded");
std::fs::create_dir_all(&corpus).unwrap();
let mut wider = b"\x89PNG\r\n\x1a\n".to_vec();
wider.extend_from_slice(&13u32.to_be_bytes());
wider.extend_from_slice(b"IHDR");
wider.extend_from_slice(&64u32.to_be_bytes());
wider.extend_from_slice(&64u32.to_be_bytes());
wider.extend_from_slice(&[8, 2, 0, 0, 0]);
wider.extend_from_slice(&[0, 0, 0, 0]);
for i in 0..3 {
std::fs::write(corpus.join(format!("c{i}.png")), test_png(0)).unwrap();
std::fs::write(
corpus.join(format!("c{i}.json")),
r#"{"role":"clean","sha256":"0"}"#,
)
.unwrap();
std::fs::write(corpus.join(format!("s{i}.png")), &wider).unwrap();
std::fs::write(
corpus.join(format!("s{i}.json")),
format!(r#"{{"role":"stego","source_png":"c{i}.png","sha256":"0"}}"#),
)
.unwrap();
}
let out = cmd_plan(
&resolved_at(®),
&[
"score".into(),
"--corpus".into(),
corpus.display().to_string(),
"--detector".into(),
"sizer".into(),
],
);
assert_eq!(out.code, exit::OK, "{}", out.human);
assert!(
out.human.contains("differ from their cover in more than"),
"plan said nothing about a corpus score would warn about: {}",
out.human
);
assert!(
out.human.contains("different sizes"),
"the warning has to carry an example: {}",
out.human
);
assert!(
!out.json["corpus_notes"].as_array().unwrap().is_empty(),
"{}",
out.json
);
}
#[cfg(unix)]
#[test]
fn plan_says_its_rate_is_declared_rather_than_measured() {
let tmp = tempfile::tempdir().unwrap();
let reg = registry_with_one_present_and_one_missing(tmp.path());
let sizer = reg.join("sizer.toml");
let with_cost = format!(
"{}\n[cost]\nseconds_per_image = 0.5\n",
std::fs::read_to_string(&sizer).unwrap()
);
std::fs::write(&sizer, with_cost).unwrap();
let corpus = tmp.path().join("corpus");
scratch_corpus(&corpus);
let out = cmd_plan(
&resolved_at(®),
&[
"score".into(),
"--corpus".into(),
corpus.display().to_string(),
"--detector".into(),
"sizer".into(),
],
);
assert_eq!(out.code, exit::OK, "{}", out.human);
assert!(
out.human.contains("Total at least"),
"{}",
out.human
);
assert!(out.human.contains("declared"), "{}", out.human);
assert!(
out.human.contains("Read the total as a floor"),
"{}",
out.human
);
assert!(out.human.contains("Worst case"), "{}", out.human);
assert!(
!out.human.contains("a ceiling, not a forecast"),
"the explanation belongs in `plan --help`, not in every run: {}",
out.human
);
assert_eq!(out.json["rate_source"], serde_json::json!("declared"));
assert_eq!(
out.json["estimate_is_a_lower_bound"],
serde_json::json!(true)
);
}
#[test]
fn a_plan_command_starting_with_a_flag_is_answered_with_the_verb_it_needs() {
let argv = |words: &[&str]| words.iter().map(|w| w.to_string()).collect::<Vec<_>>();
let unknown = clap::error::ErrorKind::UnknownArgument;
let hint = plan_verb_hint(unknown, &argv(&["plan", "--corpus", "./mine"]))
.expect("the natural first attempt is not recognised");
assert_eq!(hint, "--corpus ./mine");
let quoted = plan_verb_hint(unknown, &argv(&["plan", "--corpus", "my corpus"]))
.expect("a quoted value is not recognised");
assert_eq!(quoted, "--corpus 'my corpus'");
assert!(plan_verb_hint(unknown, &argv(&["plan", "score", "--corpus", "x"])).is_none());
assert!(plan_verb_hint(unknown, &argv(&["score", "--nope"])).is_none());
assert!(plan_verb_hint(unknown, &argv(&["plan"])).is_none());
assert!(plan_verb_hint(unknown, &[]).is_none());
assert!(plan_verb_hint(
clap::error::ErrorKind::MissingRequiredArgument,
&argv(&["plan", "--corpus", "x"])
)
.is_none());
}
#[cfg(unix)]
#[test]
fn every_score_refusal_carries_a_word_a_script_can_branch_on() {
let tmp = tempfile::tempdir().unwrap();
let reg = registry_with_one_present_and_one_missing(tmp.path());
let corpus = tmp.path().join("corpus");
scratch_corpus(&corpus);
let ask = |corpus: &Path, detectors: &[String]| {
cmd_score(
&resolved_at(®),
ScoreRequest {
corpus,
detectors,
corpus_id: None,
trained_on: None,
records: None,
out: None,
timeout: 5,
jobs: 1,
keep_raw: false,
seed: None,
fixtures: None,
limit: None,
split: None,
},
)
};
let cases = [
(
ask(&tmp.path().join("nope"), &["sizer".to_string()]),
"corpus-missing",
),
(ask(&corpus, &["ghost".to_string()]), "nothing-available"),
(ask(&corpus, &["nobody".to_string()]), "unknown-detector"),
(
ask(&corpus, &["all".to_string(), "sizer".to_string()]),
"all-with-others",
),
];
for (out, want) in cases {
assert_ne!(out.code, exit::OK, "{}", out.human);
assert_eq!(out.json["reason"], serde_json::json!(want), "{}", out.human);
assert!(out.json["error"].is_string(), "{}", out.json);
}
}
#[test]
fn the_content_digest_refusal_wraps_like_every_other_message() {
let dir = tempfile::tempdir().unwrap();
let corpus = dir.path().join("corpus");
corpus_honest(&corpus, &[b"cover bytes", b"stego bytes"]);
let digest = score::corpus_digest(&corpus).unwrap().unwrap();
let doc = dir.path().join("r.json");
result_claiming(&doc, &digest);
let mut sealed: Result1 =
serde_json::from_str(&std::fs::read_to_string(&doc).unwrap()).unwrap();
sealed.seal();
sealed.metrics.auc = 0.99;
std::fs::write(&doc, serde_json::to_string(&sealed).unwrap()).unwrap();
let out = cmd_verify(&doc, &corpus, None, true);
assert_eq!(out.code, exit::VERIFY_MISMATCH, "{}", out.human);
assert!(
!out.human.contains(" "),
"the message still carries runs of whitespace:\n{}",
out.human
);
assert!(
out.human
.contains("Every field except the two that record WHEN the run happened"),
"the sentence did not survive rewrapping:\n{}",
out.human
);
}
#[test]
fn a_listing_is_content_and_a_refusal_is_not() {
for command in [
Command::List { kind: "all".into() },
Command::Describe {
name: "zsteg".into(),
toml: false,
},
Command::Help { topic: None },
Command::Completions {
shell: clap_complete::Shell::Bash,
},
Command::Plan {
command: vec!["score".into()],
},
Command::Metrics {
file: None,
at: vec![],
},
] {
assert!(human_is_content(&command), "not routed to stdout");
}
for command in [
Command::Score {
corpus: PathBuf::from("c"),
detector: vec!["d".into()],
split: None,
corpus_id: None,
trained_on: None,
records: None,
out: None,
timeout: 60,
jobs: 1,
keep_raw: false,
seed: None,
limit: None,
},
Command::Validate {
file: PathBuf::from("f"),
},
Command::Fetch {
corpus: "c".into(),
tier: "nano".into(),
dest: None,
max_bytes: None,
budget_minutes: 1,
},
] {
assert!(!human_is_content(&command), "progress reached stdout");
}
}
#[test]
fn doctors_report_goes_to_stdout_under_either_verdict() {
let out = cmd_doctor(&resolved_at(shipped_registry()), None, false, false);
assert!(out.payload_on_stdout, "{}", out.human);
}
#[test]
fn skipping_the_self_tests_judges_what_is_installed() {
let out = cmd_doctor(&resolved_at(shipped_registry()), None, false, false);
assert_eq!(
out.json["verified"].as_u64(),
Some(0),
"this test only means something when nothing was self-tested"
);
let present = out.json["present"].as_u64().expect("a count");
let expected = if present == 0 {
exit::ENVIRONMENT_UNFIT
} else {
exit::OK
};
assert_eq!(out.code, expected, "{present} installed:\n{}", out.human);
assert!(
!out.human.contains("no tool here is usable"),
"judged on a self-test it did not run:\n{}",
out.human
);
if present > 0 {
assert!(
out.human.contains("The self-tests were skipped"),
"the report claims more than it checked:\n{}",
out.human
);
}
}
#[test]
fn the_toml_output_carries_no_key_the_author_never_wrote() {
let resolved = resolved_at(shipped_registry());
let out = cmd_describe(&resolved, "zsteg", true);
assert_eq!(out.code, exit::OK, "{}", out.human);
assert!(
!out.human
.contains(stegobench_core::registry::MAY_BE_SERVED_KEY),
"the derived answer reached the verbatim entry:\n{}",
out.human
);
let parsed: toml::Value =
toml::from_str(&out.human).expect("the entry parses back as TOML");
assert!(parsed.get("redistribution").is_some(), "{parsed:?}");
let json = cmd_describe(&resolved, "zsteg", false).json;
assert_eq!(
json.get(stegobench_core::registry::MAY_BE_SERVED_KEY),
Some(&serde_json::json!(true)),
"{json}"
);
}
#[test]
fn a_machine_with_nothing_installed_is_told_it_is_fresh_rather_than_broken() {
let out = cmd_doctor(&resolved_at(shipped_registry()), None, false, false);
let present = out.json["present"].as_u64().expect("a count");
if present > 0 {
eprintln!("skipped: {present} tool(s) installed, so not a fresh machine");
return;
}
assert_eq!(out.code, exit::ENVIRONMENT_UNFIT, "{}", out.human);
assert!(
out.human.contains("nothing is installed here yet"),
"does not say which situation this is:\n{}",
out.human
);
assert!(
out.human.contains("rather than a fault on this machine"),
"still reads as a verdict on the machine:\n{}",
out.human
);
assert!(
!out.human.contains("no tool here is usable"),
"a fresh install is still called unusable:\n{}",
out.human
);
assert!(
out.human.contains("you need ONE detector"),
"no way out of the state it just reported:\n{}",
out.human
);
assert!(
out.human.contains("docker pull"),
"names no command to run:\n{}",
out.human
);
}
#[test]
fn listing_the_detectors_names_the_cheapest_place_to_start() {
let resolved = resolved_at(shipped_registry());
let out = cmd_list(&resolved, "detectors");
assert!(
out.human.contains("you need ONE detector"),
"the listing still answers seven names and no guidance:\n{}",
out.human
);
for word in ["best", "recommended", "most accurate", "strongest"] {
assert!(
!out.human.to_lowercase().contains(word),
"the listing ranks detectors by {word:?}:\n{}",
out.human
);
}
}
#[test]
fn the_verdict_line_carries_no_exit_code() {
for strict in [false, true] {
let out = cmd_doctor(&resolved_at(shipped_registry()), None, strict, false);
assert!(
out.human.contains("\nFIT:") || out.human.contains("\nUNFIT:"),
"no verdict at all:\n{}",
out.human
);
assert!(
!out.human.contains("(exit "),
"the verdict quotes its own exit code:\n{}",
out.human
);
}
}
#[test]
fn the_self_test_summary_uses_a_word_the_help_defines() {
let out = cmd_doctor(&resolved_at(shipped_registry()), None, false, false);
let line = out
.human
.lines()
.find(|l| l.contains("Stegobench's own self-test:"))
.expect("a summary line");
assert!(line.contains("responded"), "{line}");
assert!(!line.contains("answering"), "{line}");
let mut doctor = Cli::command();
doctor.build();
let help = doctor
.get_subcommands()
.find(|s| s.get_name() == "doctor")
.expect("doctor is a command")
.get_long_about()
.expect("doctor explains itself")
.to_string();
assert!(help.contains("responded"), "the word is defined nowhere");
assert_eq!(out.json["responded"], serde_json::json!(0));
}
#[test]
fn doctor_names_where_it_read_from_in_full() {
let out = cmd_doctor(&resolved_at(shipped_registry()), None, false, false);
let first = out.human.lines().next().expect("a registry line");
let path = first
.strip_prefix("registry ")
.expect("the registry line comes first");
assert!(
Path::new(path).is_absolute(),
"a relative path is not somewhere a reader can go: {first}"
);
}
#[cfg(unix)]
#[test]
fn a_plan_leaves_out_what_score_would_skip() {
let tmp = tempfile::tempdir().unwrap();
let reg = registry_with_one_present_and_one_missing(tmp.path());
let corpus = tmp.path().join("corpus");
scratch_corpus(&corpus);
let out = cmd_plan(
&resolved_at(®),
&[
"score".into(),
"--corpus".into(),
corpus.display().to_string(),
"--detector".into(),
"all".into(),
],
);
assert_eq!(out.code, exit::OK, "{}", out.human);
assert_eq!(out.json["runnable_detectors"], serde_json::json!(1));
assert_eq!(out.json["unavailable_detectors"], serde_json::json!(1));
assert!(
out.human.contains("to score with each of 1 detector(s)"),
"the header counts what cannot run:\n{}",
out.human
);
assert!(
out.human.contains("ghost") && out.human.contains("NOT AVAILABLE"),
"the blocker is not on the screen:\n{}",
out.human
);
assert!(
out.human.contains("cannot run here and are left out"),
"nothing says the figures exclude it:\n{}",
out.human
);
let ghost = out.json["per_detector"]
.as_array()
.expect("per detector")
.iter()
.find(|d| d["detector"] == "ghost")
.expect("ghost is listed");
assert_eq!(ghost["available"], serde_json::json!(false));
assert!(ghost["estimated_seconds"].is_null(), "{ghost}");
}
#[cfg(unix)]
#[test]
fn a_plan_over_nothing_available_refuses_the_way_score_does() {
let tmp = tempfile::tempdir().unwrap();
let reg = registry_with_one_present_and_one_missing(tmp.path());
let corpus = tmp.path().join("corpus");
scratch_corpus(&corpus);
let out = cmd_plan(
&resolved_at(®),
&[
"score".into(),
"--corpus".into(),
corpus.display().to_string(),
"--detector".into(),
"ghost".into(),
],
);
assert_eq!(out.code, exit::PREFLIGHT_REFUSED, "{}", out.human);
assert_eq!(out.json["reason"], serde_json::json!("nothing-available"));
}
#[test]
fn a_timeout_of_zero_is_refused_rather_than_given_a_meaning() {
let Err(e) = Cli::try_parse_from([
"stegobench",
"score",
"--corpus",
"c",
"--detector",
"d",
"--timeout",
"0",
]) else {
panic!("0 has no honest meaning here")
};
let said = e.to_string();
assert!(said.contains("kill the detector"), "{said}");
assert!(said.contains("no value meaning no timeout"), "{said}");
let ok = Cli::try_parse_from([
"stegobench",
"score",
"--corpus",
"c",
"--detector",
"d",
"--timeout",
"1",
]);
assert!(ok.is_ok(), "one second is a real deadline");
}
#[test]
fn an_unrelated_flag_is_not_offered_as_a_correction() {
let Err(e) = Cli::try_parse_from([
"stegobench",
"score",
"--corpus",
"c",
"--detector",
"d",
"--threads",
"4",
]) else {
panic!("there is no --threads")
};
assert_eq!(
misleading_suggestion(&e).as_deref(),
Some("--threads"),
"clap's suggestion was let through: {e}"
);
let Err(e) =
Cli::try_parse_from(["stegobench", "score", "--corpu", "c", "--detector", "d"])
else {
panic!("--corpu is a typo")
};
assert_eq!(misleading_suggestion(&e), None, "{e}");
}
#[test]
fn the_words_that_mean_score_are_redirected_to_score() {
for word in ["benchmark", "eval", "evaluate", "Benchmark"] {
assert!(right_idea(word), "{word} gets the generic message");
assert!(
!no_extraction(word),
"{word} is not asking for a payload, only using the wrong verb"
);
assert!(
!meant_examine(word),
"{word} means the measurement, not an examination"
);
}
let out = cmd_right_idea("benchmark");
assert_eq!(out.code, exit::USAGE);
assert!(
out.human.contains("stegobench score --corpus"),
"{}",
out.human
);
assert!(
!out.human.contains("does not pull a hidden payload"),
"a reader who typed benchmark has understood the tool:\n{}",
out.human
);
}
#[test]
fn the_starter_corpus_counts_the_same_thing_on_every_screen() {
let reg = Registry::load(&shipped_registry()).expect("the real registry loads");
let starter = reg.corpora.get(STARTER_ID).expect("the starter is listed");
let facts = corpus_facts(starter);
let size = facts
.iter()
.find(|(k, _)| *k == "Size")
.map(|(_, v)| v.clone())
.expect("a size fact");
assert_eq!(size, "6 covers, 18 images in all", "{size}");
assert_eq!(
EMBEDDED_STARTER.len(),
39,
"the count the quickstart quotes for `fetch` has moved"
);
}
#[test]
fn a_run_closes_with_one_next_step() {
let doc: Result1 = serde_json::from_value(serde_json::json!({
"schema": stegobench_core::result::RESULT_SCHEMA_ID,
"subject": {"name": "x", "version": "sha256:a", "kind": "detector"},
"corpus": {"name": "c", "source": "supplied", "digest": "sha256:a", "pairs": 2},
"arm": {"embedder": "wow", "domain": "spatial", "format": "png"},
"metrics": {"auc": 0.9, "tpr_at_fpr": {}, "n_clean": 1, "n_stego": 1, "n_error": 0},
"provenance": {
"plugins": [{"name": "x", "image": "sha256:a", "determinism": "nondeterministic", "pinned_by": "executable-hash", "isolation": "host"}],
"harness_version": "0.1.0",
"started_utc": "2026-09-25T00:00:00Z",
"elapsed_seconds": 1.0,
"network_reachable": true
},
"declarations": {
"split_discipline": "not-applicable",
"pairing": "unverified",
"configuration": "custom",
"self_reported": false
}
}))
.expect("the fixture document parses");
let dir = tempfile::tempdir().unwrap();
let reg = Registry::load(&shipped_registry()).expect("the real registry loads");
let offer = fetch::offer(®).expect("the registry offers a real corpus");
let outcomes = vec![
(
"one".to_string(),
Outcome::Measured {
result: Box::new(doc.clone()),
tally: stegobench_plugin::runner::Tally::default(),
written: Some(dir.path().join("one.json")),
},
),
(
"two".to_string(),
Outcome::Measured {
result: Box::new(doc),
tally: stegobench_plugin::runner::Tally::default(),
written: Some(dir.path().join("two.json")),
},
),
];
let out = summarise(
&outcomes,
&Some(dir.path().to_path_buf()),
true,
Some(&offer),
);
let nexts = out.human.lines().filter(|l| l.starts_with("Next:")).count();
assert_eq!(nexts, 1, "two next steps is none:\n{}", out.human);
assert!(
out.human.contains("Then: "),
"the second suggestion was dropped rather than ordered:\n{}",
out.human
);
assert!(
out.human.contains("is a demonstration, not a measurement."),
"{}",
out.human
);
}
#[test]
fn a_version_probe_that_printed_wreckage_is_reported_as_a_failed_probe() {
let hash = "a".repeat(64);
let digest = format!("sha256:{hash}");
let short = format!("sha256:{}", &hash[..12]);
let wreck = pin_columns(&format!("Traceback (most recent call last): {digest}"));
assert_eq!(wreck, format!("version probe failed {short}"));
let good = pin_columns(&format!("stegcore 0.9.1 {digest}"));
assert_eq!(good, format!("stegcore 0.9.1 {short}"));
assert_eq!(pin_columns(&digest), short);
assert_eq!(
pin_columns(&format!("unknown version {digest}")),
format!("version probe failed {short}")
);
}
#[test]
fn a_doctor_row_keeps_its_gap_however_long_the_longest_name_is() {
let names = ["stegcore", "stegashield-edge"];
let width = names.iter().map(|n| n.len()).max().expect("names") + 1;
let rows: Vec<String> = names
.iter()
.map(|name| {
doctor_row(
&availability::Availability {
name: (*name).to_string(),
presence: Presence::Absent {
reason: "not on PATH".into(),
},
verified: None,
missing_secrets: Vec::new(),
},
true,
width,
)
})
.collect();
let columns: Vec<usize> = rows
.iter()
.map(|r| r.find("MISSING").expect("a state column"))
.collect();
assert_eq!(columns[0], columns[1], "ragged columns: {rows:?}");
for (name, row) in names.iter().zip(&rows) {
assert!(
row[name.len()..].starts_with(" "),
"no gap after the name: {row:?}"
);
}
}
#[test]
fn the_nothing_available_refusal_reads_as_english_for_one_detector() {
let reg = Registry::load(&shipped_registry()).expect("the real registry loads");
let all: Vec<&stegobench_core::registry::Entry> = reg.entries.values().collect();
let one = nothing_available(&all[..1], &[Some("not pulled".to_string())]);
assert!(
one.starts_with("the one detector you asked for is not available here."),
"{one}"
);
let two = nothing_available(
&all[..2],
&[
Some("not pulled".to_string()),
Some("not pulled".to_string()),
],
);
assert!(
two.starts_with("not one of the 2 detectors you asked for is available here."),
"{two}"
);
}
#[test]
fn the_corpus_offer_is_dropped_for_a_run_it_does_not_apply_to() {
let reg = Registry::load(&shipped_registry()).expect("the real registry loads");
let elsewhere = Path::new("/tmp/my-samples");
assert!(
offer_for_run(®, elsewhere, None).is_some(),
"the offer never fires at all"
);
assert!(offer_for_run(®, elsewhere, Some(fetch::OFFERED_CORPUS)).is_none());
assert!(offer_for_run(®, Path::new("/tmp/pentimento-core"), None).is_none());
}
}