use std::path::{Path, PathBuf};
use std::process::Command;
use std::time::Duration;
use stegobench_core::registry::Entry;
use crate::parsers::{self, Reading};
#[derive(Debug, Clone, PartialEq)]
pub enum Verified {
Passed,
Failed(String),
Skipped(String),
Answered(String),
}
fn answered_or_died(reading: Reading, out: &std::process::Output, name: &str) -> Reading {
if out.status.success() {
return reading;
}
match reading {
found @ (Reading::Verdict(true) | Reading::Score(_)) => found,
Reading::Failed(why) => Reading::Failed(why),
Reading::Verdict(false) => {
let code = out
.status
.code()
.map(|c| c.to_string())
.unwrap_or_else(|| "a signal".to_string());
let last = String::from_utf8_lossy(&out.stderr);
let last = last.lines().next().unwrap_or("").trim();
Reading::Failed(format!(
"{name} exited {code} without answering, so this is not a \
clean result: {}",
clip_line(last)
))
}
}
}
fn clip_line(line: &str) -> String {
const MOST: usize = 120;
if line.is_empty() {
return "it printed nothing at all".to_string();
}
if line.chars().count() <= MOST {
return line.to_string();
}
let kept: String = line.chars().take(MOST).collect();
format!("{kept}...")
}
pub const SANDBOX: &[&str] = &[
"--network=none",
"--cap-drop=ALL",
"--security-opt",
"no-new-privileges",
"--read-only",
"--memory=2g",
];
const PRESENTED_STEM: &str = "image";
fn presented_name(path: &Path) -> String {
match path.extension().and_then(|e| e.to_str()) {
Some(ext) if !ext.is_empty() => format!("{PRESENTED_STEM}.{ext}"),
_ => PRESENTED_STEM.to_string(),
}
}
fn presented_copy(file: &Path) -> Result<(tempfile::TempDir, PathBuf), String> {
let absolute = file
.canonicalize()
.map_err(|e| format!("{} could not be resolved: {e}", file.display()))?;
let dir = tempfile::tempdir().map_err(|e| format!("no scratch directory: {e}"))?;
let staged = dir.path().join(presented_name(&absolute));
if std::fs::hard_link(&absolute, &staged).is_err() {
std::fs::copy(&absolute, &staged)
.map_err(|e| format!("could not stage {}: {e}", absolute.display()))?;
}
Ok((dir, staged))
}
fn run_one(
entry: &Entry,
image: &str,
fixture: &Path,
timeout: Duration,
adapter_roots: &[PathBuf],
raw: &mut Raw,
) -> Reading {
let Some(invoke) = &entry.invoke else {
return Reading::Failed("entry declares no invoke block".into());
};
let Ok(absolute) = fixture.canonicalize() else {
return Reading::Failed(format!("fixture {} not found", fixture.display()));
};
if absolute.file_name().is_none() {
return Reading::Failed("fixture has no usable filename".into());
}
let name = presented_name(&absolute);
let name = name.as_str();
let scratch = if invoke.writable_workdir {
match tempfile::tempdir() {
Ok(d) => {
if let Err(e) = std::fs::copy(&absolute, d.path().join(name)) {
return Reading::Failed(format!("could not stage the fixture: {e}"));
}
Some(d)
}
Err(e) => return Reading::Failed(format!("no scratch directory: {e}")),
}
} else {
None
};
let mount = match &scratch {
Some(d) => format!("{}:/work", d.path().display()),
None => format!("{}:/work/{}:ro", absolute.display(), name),
};
let inner = format!("/work/{name}");
let mut args: Vec<String> = vec!["run".into(), "--rm".into()];
args.extend(SANDBOX.iter().map(|s| s.to_string()));
args.push("-v".into());
args.push(mount);
let mut adapter_inner = String::new();
if let Some(rel) = &invoke.adapter {
let abs = match crate::adapter::resolve(rel, adapter_roots) {
Ok(p) => p,
Err(why) => return Reading::Failed(why.to_string()),
};
let Some(base) = abs.file_name().and_then(|n| n.to_str()) else {
return Reading::Failed("adapter has no usable filename".into());
};
adapter_inner = format!("/adapter/{base}");
args.push("-v".into());
args.push(format!("{}:{}:ro", abs.display(), adapter_inner));
}
for kv in &invoke.env {
args.push("-e".into());
args.push(kv.clone());
}
if let Some(ep) = &invoke.entrypoint {
args.push("--entrypoint".into());
args.push(ep.clone());
}
args.push(image.into());
args.extend(invoke.argv.iter().map(|a| {
if a == "{files}" {
return inner.clone();
}
a.replace("{file}", &inner)
.replace("{adapter}", &adapter_inner)
}));
#[cfg(unix)]
if let Some(dir) = &scratch {
use std::os::unix::fs::MetadataExt;
if let Ok(meta) = std::fs::metadata(dir.path()) {
args.insert(2, "--user".into());
args.insert(3, format!("{}:{}", meta.uid(), meta.gid()));
}
}
let mut docker = Command::new("docker");
docker.args(&args);
let out = match crate::exec::captured(docker, "the container", timeout) {
Ok(out) => out,
Err(e) => return Reading::Failed(e),
};
let text = match (&invoke.output_file, &scratch) {
(Some(rel), Some(dir)) => {
let path = dir.path().join(rel.trim_start_matches("/work/"));
match std::fs::read_to_string(&path) {
Ok(t) => t,
Err(e) => {
return Reading::Failed(format!(
"{} wrote no {}: {e}. stderr: {}",
entry.name,
rel,
String::from_utf8_lossy(&out.stderr).trim()
))
}
}
}
_ => String::from_utf8_lossy(&out.stdout).into_owned(),
};
raw.stdout = text.clone();
raw.stderr = String::from_utf8_lossy(&out.stderr).into_owned();
if invoke.batch.is_some_and(|b| b > 1) {
let keys = vec![inner.clone()];
let one = parsers::parse_keyed(&invoke.parser, &text, &raw.stderr.clone(), &keys)
.pop()
.unwrap_or_else(|| Reading::Failed("the batch parser returned nothing".into()));
return answered_or_died(one, &out, &entry.name);
}
answered_or_died(
parsers::parse(&invoke.parser, &text, &raw.stderr.clone()),
&out,
&entry.name,
)
}
fn run_binary(entry: &Entry, fixture: &Path, timeout: Duration, raw: &mut Raw) -> Reading {
let (Some(bin), Some(invoke)) = (&entry.binary, &entry.invoke) else {
return Reading::Failed("entry is not a runnable binary".into());
};
let Some(program) = bin.command.first() else {
return Reading::Failed("binary.command is empty".into());
};
let Some(path) = crate::which(program) else {
return Reading::Failed(format!("{program} is not on PATH"));
};
if !fixture.is_file() {
return Reading::Failed(format!("fixture {} not found", fixture.display()));
}
let (_scratch, staged) = match presented_copy(fixture) {
Ok(pair) => pair,
Err(why) => return Reading::Failed(why),
};
let file = staged.display().to_string();
let mut argv: Vec<String> = bin.command[1..].to_vec();
argv.extend(invoke.argv.iter().map(|a| {
if a == "{files}" {
return file.clone();
}
a.replace("{file}", &file)
}));
let mut program_cmd = Command::new(&path);
program_cmd.args(&argv);
match crate::exec::captured(program_cmd, program, timeout) {
Ok(out) => {
raw.stdout = String::from_utf8_lossy(&out.stdout).into_owned();
raw.stderr = String::from_utf8_lossy(&out.stderr).into_owned();
answered_or_died(
parsers::parse(&invoke.parser, &raw.stdout.clone(), &raw.stderr.clone()),
&out,
program,
)
}
Err(e) => Reading::Failed(e),
}
}
#[derive(Debug, Default, Clone, PartialEq)]
pub struct Raw {
pub stdout: String,
pub stderr: String,
}
impl Raw {
pub fn is_empty(&self) -> bool {
self.stdout.is_empty() && self.stderr.is_empty()
}
}
pub fn read_one_observed(
entry: &Entry,
fixture: &Path,
timeout: Duration,
adapter_roots: &[PathBuf],
raw: &mut Raw,
) -> Reading {
if entry.invoke.as_ref().is_some_and(|i| i.host) {
return run_host_adapter(entry, fixture, timeout, adapter_roots, raw);
}
match (&entry.image, &entry.binary) {
(Some(img), _) => run_one(
entry,
&img.reference.clone(),
fixture,
timeout,
adapter_roots,
raw,
),
(_, Some(_)) => run_binary(entry, fixture, timeout, raw),
_ => Reading::Failed("entry declares neither an image nor a binary".into()),
}
}
pub fn read_many_observed(
entry: &Entry,
files: &[PathBuf],
timeout: Duration,
adapter_roots: &[PathBuf],
raw: &mut Raw,
) -> Vec<Reading> {
let fail = |why: String| vec![Reading::Failed(why); files.len()];
if files.is_empty() {
return Vec::new();
}
let Some(invoke) = &entry.invoke else {
return fail("entry declares no invoke block".into());
};
if invoke.host || entry.image.is_none() {
return files
.iter()
.map(|f| read_one_observed(entry, f, timeout, adapter_roots, &mut Raw::default()))
.collect();
}
let Some(image) = &entry.image else {
return fail("entry declares no image".into());
};
let mut args: Vec<String> = vec!["run".into(), "--rm".into()];
args.extend(SANDBOX.iter().map(|s| s.to_string()));
let mut keys: Vec<String> = Vec::with_capacity(files.len());
for (i, file) in files.iter().enumerate() {
let Ok(absolute) = file.canonicalize() else {
return fail(format!("{} not found", file.display()));
};
let ext = absolute
.extension()
.and_then(|e| e.to_str())
.map(|e| format!(".{e}"))
.unwrap_or_default();
let inner = format!("/work/{i:06}{ext}");
args.push("-v".into());
args.push(format!("{}:{}:ro", absolute.display(), inner));
keys.push(inner);
}
let mut adapter_inner = String::new();
if let Some(rel) = &invoke.adapter {
let abs = match crate::adapter::resolve(rel, adapter_roots) {
Ok(p) => p,
Err(why) => return fail(why.to_string()),
};
let Some(base) = abs.file_name().and_then(|n| n.to_str()) else {
return fail("adapter has no usable filename".into());
};
adapter_inner = format!("/adapter/{base}");
args.push("-v".into());
args.push(format!("{}:{}:ro", abs.display(), adapter_inner));
}
for kv in &invoke.env {
args.push("-e".into());
args.push(kv.clone());
}
if let Some(ep) = &invoke.entrypoint {
args.push("--entrypoint".into());
args.push(ep.clone());
}
args.push(image.reference.clone());
for a in &invoke.argv {
if a == "{files}" {
args.extend(keys.iter().cloned());
} else {
args.push(a.replace("{adapter}", &adapter_inner));
}
}
let mut docker = Command::new("docker");
docker.args(&args);
let out = match crate::exec::captured(docker, "the container", timeout) {
Ok(out) => out,
Err(e) => return fail(format!("the batch of {} failed: {e}", files.len())),
};
let stdout = String::from_utf8_lossy(&out.stdout).into_owned();
let stderr = String::from_utf8_lossy(&out.stderr).into_owned();
raw.stdout = stdout.clone();
raw.stderr = stderr.clone();
parsers::parse_keyed(&invoke.parser, &stdout, &stderr, &keys)
.into_iter()
.map(|r| answered_or_died(r, &out, &entry.name))
.collect()
}
pub fn read_one(
entry: &Entry,
fixture: &Path,
timeout: Duration,
adapter_roots: &[PathBuf],
) -> Reading {
read_one_observed(entry, fixture, timeout, adapter_roots, &mut Raw::default())
}
fn run_host_adapter(
entry: &Entry,
fixture: &Path,
timeout: Duration,
adapter_roots: &[PathBuf],
raw: &mut Raw,
) -> Reading {
let Some(invoke) = &entry.invoke else {
return Reading::Failed("entry declares no invoke block".into());
};
let Some(rel) = &invoke.adapter else {
return Reading::Failed("a host invoke needs an adapter".into());
};
if !fixture.is_file() {
return Reading::Failed(format!("fixture {} not found", fixture.display()));
}
let adapter = match crate::adapter::resolve(rel, adapter_roots) {
Ok(p) => p.display().to_string(),
Err(why) => return Reading::Failed(why.to_string()),
};
let (_scratch, staged) = match presented_copy(fixture) {
Ok(pair) => pair,
Err(why) => return Reading::Failed(why),
};
let file = staged.display().to_string();
let program = invoke
.entrypoint
.clone()
.unwrap_or_else(|| crate::DEFAULT_HOST_ENTRYPOINT.into());
let argv: Vec<String> = invoke
.argv
.iter()
.map(|a| a.replace("{adapter}", &adapter).replace("{file}", &file))
.collect();
let mut cmd = Command::new(&program);
cmd.args(&argv);
for kv in &invoke.env {
if let Some((k, v)) = kv.split_once('=') {
cmd.env(k, v);
}
}
match crate::exec::captured(cmd, &program, timeout) {
Ok(out) => {
raw.stdout = String::from_utf8_lossy(&out.stdout).into_owned();
raw.stderr = String::from_utf8_lossy(&out.stderr).into_owned();
parsers::parse(&invoke.parser, &raw.stdout.clone(), &raw.stderr.clone())
}
Err(e) => Reading::Failed(e),
}
}
pub fn run(entry: &Entry, fixtures_dir: &Path, adapter_roots: &[PathBuf]) -> Verified {
if entry.kind == stegobench_core::registry::Kind::Embedder {
return roundtrip::run(entry, fixtures_dir);
}
let Some(test) = &entry.selftest else {
return Verified::Skipped("no selftest declared".into());
};
if entry.invoke.is_none() {
return Verified::Skipped(
"no invoke block yet, so this tool cannot be driven by the host".into(),
);
}
let detect_path = fixtures_dir.join(strip_prefix(&test.must_detect));
let clear_path = fixtures_dir.join(strip_prefix(&test.must_clear));
for path in [&detect_path, &clear_path] {
if !path.is_file() {
return Verified::Skipped(format!(
"fixture {} was not found, so this tool was never asked. \
Name the directory with --fixtures <DIR>",
path.display()
));
}
}
if let Some(declared) = entry.invoke.as_ref().and_then(|i| i.adapter.as_deref()) {
if let Err(why) = crate::adapter::resolve(declared, adapter_roots) {
return Verified::Skipped(format!("{why}, so this tool was never asked"));
}
}
let threshold = test.threshold;
let higher = entry.emits.higher_means_stego;
let on_stego = read_one(
entry,
&detect_path,
crate::exec::ITEM_TIMEOUT,
adapter_roots,
);
let on_clean = read_one(entry, &clear_path, crate::exec::ITEM_TIMEOUT, adapter_roots);
let subject = entry.maintainer == stegobench_core::registry::Maintainer::Subject;
match (
on_stego.says_stego(higher, threshold),
on_clean.says_stego(higher, threshold),
) {
(Some(true), Some(false)) => Verified::Passed,
(Some(_), Some(_)) if subject => Verified::Answered(
"reachable and answering; it did not separate the fixtures, which \
is a result rather than a fault"
.into(),
),
(Some(true), Some(true)) => Verified::Failed(
"says stego on the clean fixture too, so it answers yes to everything".into(),
),
(Some(false), Some(false)) => Verified::Failed(
"says clean on the stego fixture too, so it answers no to everything".into(),
),
(Some(false), Some(true)) => Verified::Failed("has both answers exactly backwards".into()),
(None, _) => Verified::Failed(format!("could not read the stego fixture: {on_stego:?}")),
(_, None) => Verified::Failed(format!("could not read the clean fixture: {on_clean:?}")),
}
}
fn strip_prefix(name: &str) -> &str {
name.strip_prefix("fixtures/").unwrap_or(name)
}
#[cfg(test)]
mod tests {
use super::*;
fn run(entry: &Entry, fixtures_dir: &Path) -> Verified {
super::run(entry, fixtures_dir, &[])
}
fn entry(extra: &str) -> Entry {
toml::from_str(&format!(
"name = \"x\"\nkind = \"detector\"\nlicence = \"MIT\"\n{extra}\n\
[selftest]\nmust_detect = \"fixtures/a.png\"\nmust_clear = \"fixtures/b.png\"\n"
))
.expect("parses")
}
#[cfg(unix)]
#[test]
fn a_binary_handed_the_batch_placeholder_still_receives_a_path() {
use std::os::unix::fs::PermissionsExt;
let tmp = tempfile::tempdir().expect("tmp");
let script = tmp.path().join("echoer.sh");
std::fs::write(
&script,
"#!/bin/sh\n[ \"$1\" = \"--version\" ] && { echo v1; exit 0; }\n\
case \"$1\" in *.png) echo 0.9 ;; *) echo 0.1 ;; esac\n",
)
.unwrap();
let mut perms = std::fs::metadata(&script).unwrap().permissions();
perms.set_mode(0o755);
std::fs::set_permissions(&script, perms).unwrap();
let e = entry(&format!(
"[binary]\ncommand = [{:?}]\nversion_args = [\"--version\"]\n\
[emits]\noutput = \"score\"\nhigher_means_stego = true\n\
[invoke]\nargv = [\"{{files}}\"]\nparser = \"number\"",
script.display().to_string()
));
let fixture = tmp.path().join("a.png");
std::fs::write(&fixture, b"\x89PNG\r\n\x1a\n").unwrap();
assert_eq!(
read_one(&e, &fixture, Duration::from_secs(5), &[]),
Reading::Score(0.9),
"the tool was handed the placeholder instead of the image"
);
}
#[cfg(unix)]
fn stand_in(dir: &Path, name: &str, body: &str) -> Entry {
use std::os::unix::fs::PermissionsExt;
let script = dir.join(name);
std::fs::write(
&script,
format!(
"#!/bin/sh\n[ \"$1\" = \"--version\" ] && {{ echo 'stand-in 1.0'; exit 0; }}\n\
{body}\n"
),
)
.expect("script");
let mut perms = std::fs::metadata(&script).expect("meta").permissions();
perms.set_mode(0o755);
std::fs::set_permissions(&script, perms).expect("chmod");
entry(&format!(
"[binary]\ncommand = [{:?}]\nversion_args = [\"--version\"]\n\
[emits]\noutput = \"score\"\nhigher_means_stego = true\n\
[invoke]\nargv = [\"{{file}}\"]\nparser = \"number\"",
script.display().to_string()
))
}
#[cfg(unix)]
fn two_fixtures(dir: &Path) -> &Path {
std::fs::write(dir.join("a.png"), b"\x89PNG\r\n\x1a\nPAYLOAD").expect("stego fixture");
std::fs::write(dir.join("b.png"), b"\x89PNG\r\n\x1a\nnothing").expect("clean fixture");
dir
}
#[cfg(unix)]
#[test]
fn a_detector_that_keys_on_the_fixture_name_can_no_longer_pass() {
let tmp = tempfile::tempdir().expect("tmp");
let e = stand_in(
tmp.path(),
"cheat.sh",
"b=$(basename \"$1\")\n\
case \"$b\" in *lsb*|*appended*|*a.png) echo 0.990 ;; *) echo 0.010 ;; esac",
);
let fx = tempfile::tempdir().expect("fixtures");
match run(&e, two_fixtures(fx.path())) {
Verified::Failed(why) => assert!(why.contains("no to everything"), "got {why}"),
other => panic!("a name-keyed plugin was verified: {other:?}"),
}
}
#[cfg(unix)]
#[test]
fn a_detector_that_reads_the_bytes_still_passes() {
let tmp = tempfile::tempdir().expect("tmp");
let e = stand_in(
tmp.path(),
"honest.sh",
"if grep -q PAYLOAD \"$1\"; then echo 0.9; else echo 0.1; fi",
);
let fx = tempfile::tempdir().expect("fixtures");
match run(&e, two_fixtures(fx.path())) {
Verified::Passed => {}
other => panic!("an honest plugin stopped passing: {other:?}"),
}
}
#[cfg(unix)]
#[test]
fn two_runs_reach_the_same_verdict_although_the_path_differs() {
let tmp = tempfile::tempdir().expect("tmp");
let honest = stand_in(
tmp.path(),
"honest.sh",
"if grep -q PAYLOAD \"$1\"; then echo 0.9; else echo 0.1; fi",
);
let cheat = stand_in(
tmp.path(),
"cheat.sh",
"case \"$(basename \"$1\")\" in *a.png) echo 0.9 ;; *) echo 0.1 ;; esac",
);
let fx = tempfile::tempdir().expect("fixtures");
let dir = two_fixtures(fx.path());
for e in [&honest, &cheat] {
assert_eq!(run(e, dir), run(e, dir), "the same tool answered twice");
}
}
#[cfg(unix)]
#[test]
fn a_tool_is_handed_a_neutral_stem_and_its_own_extension() {
let tmp = tempfile::tempdir().expect("tmp");
let e = stand_in(
tmp.path(),
"reporter.sh",
"case \"$(basename \"$1\")\" in image.png) echo 0.9 ;; *) echo 0.1 ;; esac",
);
let fixture = tmp.path().join("lsb-0.4bpp.png");
std::fs::write(&fixture, b"\x89PNG\r\n\x1a\n").expect("fixture");
assert_eq!(
read_one(&e, &fixture, Duration::from_secs(5), &[]),
Reading::Score(0.9),
"the tool saw a name other than the neutral one"
);
}
#[cfg(unix)]
#[test]
fn a_label_file_beside_the_image_cannot_be_read() {
let tmp = tempfile::tempdir().expect("tmp");
let e = stand_in(
tmp.path(),
"peeker.sh",
"if [ -f \"$(dirname \"$1\")/lsb-0.4bpp.json\" ]; then echo 0.9; else echo 0.1; fi",
);
let fixture = tmp.path().join("lsb-0.4bpp.png");
std::fs::write(&fixture, b"\x89PNG\r\n\x1a\n").expect("fixture");
std::fs::write(tmp.path().join("lsb-0.4bpp.json"), b"{\"arm\":\"lsb\"}").expect("sidecar");
assert_eq!(
read_one(&e, &fixture, Duration::from_secs(5), &[]),
Reading::Score(0.1),
"the tool could still read the label beside the image"
);
}
#[cfg(unix)]
#[test]
fn a_host_adapter_is_handed_the_neutral_name_too() {
use std::os::unix::fs::PermissionsExt;
let tmp = tempfile::tempdir().expect("tmp");
let adapter = tmp.path().join("adapter.sh");
std::fs::write(
&adapter,
"#!/bin/sh\ncase \"$(basename \"$1\")\" in image.png) echo 0.9 ;; *) echo 0.1 ;; esac\n",
)
.expect("adapter");
let mut perms = std::fs::metadata(&adapter).expect("meta").permissions();
perms.set_mode(0o755);
std::fs::set_permissions(&adapter, perms).expect("chmod");
let e = entry(&format!(
"[image]\nreference = \"x@sha256:a\"\nsize_mb = 1\nbundled = true\n\
[emits]\noutput = \"score\"\nhigher_means_stego = true\n\
[invoke]\nhost = true\nentrypoint = \"sh\"\nadapter = {:?}\n\
argv = [\"{{adapter}}\", \"{{file}}\"]\nparser = \"number\"",
adapter.display().to_string()
));
let fixture = tmp.path().join("lsb-0.4bpp.png");
std::fs::write(&fixture, b"\x89PNG\r\n\x1a\n").expect("fixture");
assert_eq!(
read_one(&e, &fixture, Duration::from_secs(5), &[]),
Reading::Score(0.9),
"the adapter saw a name other than the neutral one"
);
}
#[test]
fn the_presented_name_keeps_the_extension_and_drops_everything_else() {
assert_eq!(presented_name(Path::new("/x/lsb-0.4bpp.png")), "image.png");
assert_eq!(presented_name(Path::new("/x/clean.JPG")), "image.JPG");
assert_eq!(presented_name(Path::new("/x/appended")), "image");
assert_eq!(
presented_name(Path::new("/x/a.png")),
presented_name(Path::new("/y/b.png"))
);
}
#[test]
fn a_tool_with_no_invoke_block_is_skipped_not_failed() {
let e = entry("[image]\nreference = \"x@sha256:a\"\nsize_mb = 10\nbundled = true");
assert!(matches!(run(&e, Path::new(".")), Verified::Skipped(_)));
}
#[test]
fn a_binary_tool_is_now_run_rather_than_skipped() {
let e = entry(
"[binary]\ncommand = [\"definitely-not-real-xyzzy\"]\nversion_args = [\"-v\"]\n\
[invoke]\nargv = [\"/work\"]\nwritable_workdir = true\nparser = \"stegcore\"",
);
let fx = fixtures_present();
match run(&e, fx.path()) {
Verified::Failed(r) => assert!(r.contains("not on PATH"), "got {r}"),
other => panic!("expected Failed, got {other:?}"),
}
}
fn fixtures_present() -> tempfile::TempDir {
let dir = tempfile::tempdir().expect("scratch directory");
for name in ["a.png", "b.png"] {
std::fs::write(dir.path().join(name), b"not really a png").expect("fixture written");
}
dir
}
#[test]
fn a_host_adapter_entry_is_dispatched_to_the_host_not_the_image() {
let e = entry(
"[image]\nreference = \"x@sha256:a\"\nsize_mb = 10\nbundled = true\n\
[invoke]\nhost = true\nargv = [\"{adapter}\"]\nparser = \"number\"",
);
let fx = fixtures_present();
match run(&e, fx.path()) {
Verified::Failed(r) => assert!(r.contains("adapter"), "got {r}"),
other => panic!("expected Failed about the adapter, got {other:?}"),
}
}
#[test]
fn the_fixtures_prefix_is_not_doubled() {
assert_eq!(strip_prefix("fixtures/clean.png"), "clean.png");
assert_eq!(strip_prefix("clean.png"), "clean.png");
}
#[test]
fn a_missing_fixture_is_skipped_with_the_path_and_never_called_a_failure() {
let e = entry(
"[image]\nreference = \"x@sha256:a\"\nsize_mb = 10\nbundled = true\n\
[invoke]\nargv = [\"true\"]\nparser = \"zsteg\"",
);
match run(&e, Path::new("/definitely/not/here")) {
Verified::Skipped(r) => {
assert!(r.contains("not found"), "got {r}");
assert!(r.contains("/definitely/not/here"), "the path is named: {r}");
assert!(r.contains("--fixtures"), "the way out is named: {r}");
}
other => panic!("expected Skipped, got {other:?}"),
}
}
#[test]
fn the_roundtrip_payload_stays_inside_the_measured_band() {
assert_eq!(
roundtrip::PAYLOAD.len(),
128,
"the round trip payload must be 128 bytes: read the comment on \
roundtrip::PAYLOAD before changing it"
);
}
}
#[cfg(all(test, unix))]
mod local_roundtrip_tests {
use std::io::Write;
use std::os::unix::fs::PermissionsExt;
use std::path::Path;
use stegobench_core::registry::Entry;
use super::{read_one_observed, roundtrip, Raw, Verified};
use crate::parsers::Reading;
use std::time::Duration;
fn fake_tool(dir: &Path, name: &str, body: &str) -> String {
let path = dir.join(name);
let mut f = std::fs::File::create(&path).expect("script");
write!(f, "#!/bin/sh\nset -e\n{body}\n").expect("written");
let mut perms = f.metadata().expect("metadata").permissions();
perms.set_mode(0o755);
std::fs::set_permissions(&path, perms).expect("chmod");
path.display().to_string()
}
fn entry(command: &str) -> Entry {
toml::from_str(&format!(
"name = \"x\"\nkind = \"embedder\"\nlicence = \"MIT\"\n\
[binary]\ncommand = [{command}]\n\
[roundtrip]\ncover = \"fixtures/clean.png\"\n\
embed_argv = [\"embed\", \"{{cover}}\", \"{{payload}}\", \"{{stego}}\"]\n\
extract_argv = [\"extract\", \"{{stego}}\", \"{{recovered}}\"]\n\
[selftest]\nmust_detect = \"fixtures/a.png\"\nmust_clear = \"fixtures/b.png\"\n"
))
.expect("parses")
}
fn fixtures(dir: &Path) -> &Path {
std::fs::write(dir.join("clean.png"), b"\x89PNG\r\n\x1a\ncover bytes").expect("cover");
dir
}
fn detector(command: &str) -> Entry {
toml::from_str(&format!(
"name = \"crasher\"\nkind = \"detector\"\nlicence = \"MIT\"\n\
[binary]\ncommand = [{command}]\nversion_args = [\"-v\"]\n\
[emits]\noutput = \"verdict\"\n\
[invoke]\nargv = [\"{{file}}\"]\nparser = \"zsteg\"\n\
[selftest]\nmust_detect = \"a.png\"\nmust_clear = \"b.png\"\n"
))
.expect("parses")
}
#[test]
fn a_detector_that_exits_nonzero_saying_nothing_has_not_found_an_image_clean() {
let tmp = tempfile::tempdir().expect("tmp");
let tool = fake_tool(
tmp.path(),
"crash.sh",
"case \"$1\" in -v) echo 1.0; exit 0;; esac\n\
echo \"tempfile.rb:159: Read-only file system (Errno::EROFS)\" >&2\n\
exit 1",
);
let e = detector(&format!("{tool:?}"));
let img = tmp.path().join("exhibit.png");
std::fs::write(&img, b"\x89PNG\r\n\x1a\nnot really a png").expect("image");
let mut raw = Raw::default();
let reading = read_one_observed(&e, &img, Duration::from_secs(30), &[], &mut raw);
match reading {
Reading::Failed(why) => {
assert!(
why.contains("exited 1"),
"the exit code belongs in it: {why}"
);
assert!(
why.contains("not a clean result"),
"the whole point is that it is NOT clean: {why}"
);
}
other => panic!(
"a crash was read as an answer about the image: {other:?}. This is \
the defect that put four stack traces in front of a forensic \
analyst as evidence of absence"
),
}
}
#[test]
fn a_detector_that_reports_a_hit_and_then_dies_keeps_the_hit() {
let tmp = tempfile::tempdir().expect("tmp");
let tool = fake_tool(
tmp.path(),
"hit-then-crash.sh",
"case \"$1\" in -v) echo 1.0; exit 0;; esac\n\
echo \"[?] 29 bytes of extra data after image end\"\n\
echo \"then it fell over\" >&2\n\
exit 1",
);
let e = detector(&format!("{tool:?}"));
let img = tmp.path().join("exhibit.png");
std::fs::write(&img, b"\x89PNG\r\n\x1a\nnot really a png").expect("image");
let mut raw = Raw::default();
match read_one_observed(&e, &img, Duration::from_secs(30), &[], &mut raw) {
Reading::Verdict(true) => {}
other => panic!("a finding printed before the crash was thrown away: {other:?}"),
}
}
#[test]
fn a_local_tool_that_returns_the_payload_passes() {
let tmp = tempfile::tempdir().expect("tmp");
let tool = fake_tool(
tmp.path(),
"tool.sh",
"case \"$1\" in\n\
embed) { printf 'STEGO:'; cat \"$3\"; } > \"$4\" ;;\n\
extract) tail -c +7 \"$2\" > \"$3\" ;;\n\
esac",
);
let e = entry(&format!("{tool:?}"));
let fix = tempfile::tempdir().expect("fixtures");
match roundtrip::run(&e, fixtures(fix.path())) {
Verified::Passed => {}
other => panic!("expected Passed, got {other:?}"),
}
}
#[test]
fn a_local_tool_that_writes_nothing_fails_rather_than_passing_quietly() {
let tmp = tempfile::tempdir().expect("tmp");
let tool = fake_tool(tmp.path(), "quiet.sh", "exit 0");
let e = entry(&format!("{tool:?}"));
let fix = tempfile::tempdir().expect("fixtures");
match roundtrip::run(&e, fixtures(fix.path())) {
Verified::Failed(r) => assert!(r.contains("wrote no stego file"), "got {r}"),
other => panic!("expected Failed, got {other:?}"),
}
}
#[test]
fn a_local_tool_that_returns_the_wrong_bytes_fails() {
let tmp = tempfile::tempdir().expect("tmp");
let tool = fake_tool(
tmp.path(),
"lossy.sh",
"case \"$1\" in\n\
embed) cat \"$2\" \"$3\" > \"$4\" ;;\n\
extract) printf 'not the payload' > \"$3\" ;;\n\
esac",
);
let e = entry(&format!("{tool:?}"));
let fix = tempfile::tempdir().expect("fixtures");
match roundtrip::run(&e, fixtures(fix.path())) {
Verified::Failed(r) => {
assert!(r.contains("did not survive the round trip"), "got {r}");
assert!(
r.contains(&format!("{} bytes went in", roundtrip::PAYLOAD.len())),
"the payload size: {r}"
);
assert!(r.contains("15 came back"), "what came back: {r}");
}
other => panic!("expected Failed, got {other:?}"),
}
}
#[test]
fn a_tool_that_is_not_installed_fails_naming_the_program() {
let e = entry("\"definitely-not-installed-anywhere\"");
let fix = tempfile::tempdir().expect("fixtures");
match roundtrip::run(&e, fixtures(fix.path())) {
Verified::Failed(r) => {
assert!(r.contains("embed failed"), "got {r}");
assert!(r.contains("definitely-not-installed-anywhere"), "got {r}");
}
other => panic!("expected Failed, got {other:?}"),
}
}
#[test]
fn a_tool_that_never_answers_is_killed_rather_than_waited_on_for_ever() {
use std::time::Duration;
let mut sleeper = std::process::Command::new("sh");
sleeper.args(["-c", "sleep 30"]);
let started = std::time::Instant::now();
let outcome = crate::embed::bounded(sleeper, "sh", Duration::from_millis(200));
assert!(started.elapsed() < Duration::from_secs(10), "it waited");
match outcome {
Err(why) => assert!(why.contains("no answer in"), "got {why}"),
Ok(()) => panic!("a tool that never answered was reported as fine"),
}
}
#[test]
fn an_entry_with_neither_an_image_nor_a_binary_is_skipped() {
let e: Entry = toml::from_str(
"name = \"x\"\nkind = \"embedder\"\nlicence = \"MIT\"\n\
[roundtrip]\ncover = \"fixtures/clean.png\"\n\
embed_argv = [\"e\"]\nextract_argv = [\"x\"]\n\
[selftest]\nmust_detect = \"fixtures/a.png\"\nmust_clear = \"fixtures/b.png\"\n",
)
.expect("parses");
let fix = tempfile::tempdir().expect("fixtures");
match roundtrip::run(&e, fixtures(fix.path())) {
Verified::Skipped(r) => assert!(r.contains("no image and no binary"), "got {r}"),
other => panic!("expected Skipped, got {other:?}"),
}
}
}
pub mod roundtrip {
use std::path::Path;
use stegobench_core::registry::Entry;
use super::Verified;
pub const PAYLOAD: &[u8] = b"stegobench roundtrip fixture 2026: if you can \
read this, it survived. 128 bytes on purpose: do not shorten it, read \
the comment.";
pub fn run(entry: &Entry, fixtures_dir: &Path) -> Verified {
let Some(rt) = &entry.roundtrip else {
return Verified::Skipped("no roundtrip declared".into());
};
let cover_src = fixtures_dir.join(rt.cover.trim_start_matches("fixtures/"));
if !cover_src.is_file() {
return Verified::Skipped(format!(
"cover {} was not found, so no round trip was attempted. \
Name the directory with --fixtures <DIR>",
cover_src.display()
));
}
let Ok(dir) = tempfile::tempdir() else {
return Verified::Failed("no scratch directory".into());
};
let payload = dir.path().join("payload.bin");
if std::fs::write(&payload, PAYLOAD).is_err() {
return Verified::Failed("could not stage the payload".into());
}
let ext = cover_src
.extension()
.and_then(|e| e.to_str())
.unwrap_or("bin");
let out = dir.path().join(format!("stego.{ext}"));
match crate::embed::run(entry, &cover_src, &payload, &out, None, true) {
Err(e) if e.starts_with("no image and no binary") => Verified::Skipped(e),
Err(e) => Verified::Failed(e),
Ok(done) => match done.recovered {
Some(Ok(())) => Verified::Passed,
Some(Err(e)) => Verified::Failed(e),
None => Verified::Skipped("no extract argv to check the embed with".into()),
},
}
}
}