use stegobench_core::corpus::{CorpusEntry, LicenceStatus};
use stegobench_core::registry::{Entry, Kind};
use stegobench_plugin::availability::{Availability, Presence};
pub fn can_be_driven(entry: &Entry) -> bool {
entry.can_be_driven()
}
pub fn undrivable_because(entry: &Entry) -> String {
let block = match entry.kind {
Kind::Detector => "invoke",
Kind::Embedder => "roundtrip",
};
format!(
"declares no {block} block, so nothing in its registry entry says \
what command to launch and the host has no way to drive it"
)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Readiness {
Ready,
NeedsYou,
CannotRunHere,
CannotBeDriven,
Unknown,
}
impl Readiness {
pub fn word(self) -> &'static str {
match self {
Readiness::Ready => "ready",
Readiness::NeedsYou => "NEEDS YOU",
Readiness::CannotRunHere => "n/a here",
Readiness::CannotBeDriven => "undrivable",
Readiness::Unknown => "unknown",
}
}
pub fn code(self) -> &'static str {
match self {
Readiness::Ready => "ready",
Readiness::NeedsYou => "needs_you",
Readiness::CannotRunHere => "cannot_run_here",
Readiness::CannotBeDriven => "cannot_be_driven",
Readiness::Unknown => "unknown",
}
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct Step {
pub run: Option<String>,
pub why: String,
}
impl Step {
fn run(command: impl Into<String>, why: impl Into<String>) -> Self {
Step {
run: Some(command.into()),
why: why.into(),
}
}
fn just(why: impl Into<String>) -> Self {
Step {
run: None,
why: why.into(),
}
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct Needs {
pub readiness: Readiness,
pub steps: Vec<Step>,
}
impl Needs {
pub fn block(&self) -> String {
if self.steps.is_empty() {
return String::new();
}
let mut out = String::new();
for step in &self.steps {
match &step.run {
Some(cmd) => out.push_str(&format!(" {cmd}\n {}\n", step.why)),
None => out.push_str(&format!(" {}\n", step.why)),
}
}
out.pop();
out
}
pub fn to_json(&self) -> serde_json::Value {
serde_json::json!({
"readiness": self.readiness.code(),
"steps": self.steps.iter().map(|s| serde_json::json!({
"run": s.run,
"why": s.why,
})).collect::<Vec<_>>(),
})
}
}
pub fn runs_in_container(entry: &Entry) -> bool {
entry.image.is_some() && !entry.invoke.as_ref().is_some_and(|i| i.host)
}
pub fn of_tool(
entry: &Entry,
availability: &Availability,
adapter_roots: &[std::path::PathBuf],
) -> Needs {
if let Presence::Unsupported { reason } = &availability.presence {
return Needs {
readiness: Readiness::CannotRunHere,
steps: vec![Step::just(format!(
"{reason}. Nothing to install will change it."
))],
};
}
if !can_be_driven(entry) {
return Needs {
readiness: Readiness::CannotBeDriven,
steps: vec![Step::just(format!(
"This entry {}. Nothing you install will change that.",
undrivable_because(entry)
))],
};
}
let mut steps = Vec::new();
if !availability.presence.is_present() {
steps.extend(route_steps(entry, &availability.presence, adapter_roots));
}
for name in &availability.missing_secrets {
steps.push(Step::run(
format!("export {name}=<your value>"),
format!("{name} is not set. The registry never records a value"),
));
}
let readiness = if !steps.is_empty() {
Readiness::NeedsYou
} else if availability.presence.is_present() {
Readiness::Ready
} else {
Readiness::Unknown
};
Needs { readiness, steps }
}
fn route_steps(
entry: &Entry,
presence: &Presence,
adapter_roots: &[std::path::PathBuf],
) -> Vec<Step> {
let mut steps = Vec::new();
if runs_in_container(entry) {
let Some(image) = &entry.image else {
return vec![Step::just(
"this entry declares neither an image nor a binary. That is a \
fault in the registry entry, not something to install.",
)];
};
if matches!(presence, Presence::Unknown { .. }) {
steps.push(Step::just(
"No container runtime answered. Install Docker (or a drop-in \
replacement) and start the daemon.",
));
}
let size = match image.size_mb {
Some(mb) => format!("about {mb} MB, "),
None => String::new(),
};
steps.push(Step::run(
format!("docker pull {}", image.reference),
format!("{size}pinned by digest"),
));
return steps;
}
if let Some(invoke) = &entry.invoke {
if invoke.host {
if let Some(declared) = &invoke.adapter {
if let Err(why) = stegobench_plugin::adapter::resolve(declared, adapter_roots) {
steps.push(Step::just(why.to_string()));
}
}
let program = invoke
.entrypoint
.clone()
.unwrap_or_else(|| stegobench_plugin::DEFAULT_HOST_ENTRYPOINT.into());
if stegobench_plugin::which(&program).is_none() {
steps.push(Step::just(format!(
"{program} is not on PATH, and it runs the adapter. \
Install it."
)));
}
if let Some(name) = &invoke.endpoint_env {
let set =
std::env::var_os(name).is_some_and(|v| !v.to_string_lossy().trim().is_empty());
if !set {
steps.push(Step::run(
format!("export {name}=<the address of your own instance>"),
format!(
"{} is a service, scored by posting to an instance \
you started. No default: one would be scored \
against whatever answered. `stegobench describe \
{}` has the shape",
entry.name, entry.name
),
));
}
}
return steps;
}
}
let program = entry
.binary
.as_ref()
.and_then(|b| b.command.first())
.cloned();
match program {
Some(program) => {
let where_from = match &entry.upstream {
Some(url) => format!(" Upstream: {url}"),
None => String::new(),
};
steps.push(Step::just(format!(
"{program} is not on PATH. Install it until `which {program}` \
finds it.{where_from}"
)));
}
None => steps.push(Step::just(
"this entry declares neither an image nor a runnable command. \
That is a fault in the registry entry, not something to install.",
)),
}
steps
}
pub fn of_corpus(corpus: &CorpusEntry) -> Needs {
let mut steps = Vec::new();
if corpus.obtain.requires_acceptance {
steps.push(Step::just(format!(
"Accept {}'s terms yourself first. No script here can do it for \
you.",
corpus.id
)));
}
if let Some(instructions) = &corpus.obtain.instructions {
steps.push(Step::just(format!("How to obtain it: {instructions}")));
}
match (&corpus.obtain.doi, &corpus.obtain.url) {
(Some(doi), _) => steps.push(Step::just(format!("Download it, citing {doi}"))),
(None, Some(url)) => steps.push(Step::just(format!("Download it from {url}"))),
(None, None) if corpus.id == crate::STARTER_ID => steps.push(Step::just(
"No download: this one is compiled into the binary. \
`stegobench fetch stegobench-starter --tier nano` writes it out.",
)),
(None, None) => steps.push(Step::just(
"This entry records no download route; obtaining it is yours to \
arrange.",
)),
}
match corpus.licence.status {
LicenceStatus::Verified => {}
LicenceStatus::Unverified => steps.push(Step::just(format!(
"{}'s terms are UNVERIFIED here. Read them before publishing \
anything derived from it; an unknown is not a yes.",
corpus.id
))),
LicenceStatus::NoneGranted => steps.push(Step::just(format!(
"{} grants NO terms. Measuring against it privately may be \
possible; publishing anything derived from it is not.",
corpus.id
))),
}
let declares_digest = corpus
.integrity
.as_ref()
.is_some_and(|i| i.records_sha256.is_some());
if declares_digest {
steps.push(Step::run(
format!(
"stegobench score --corpus <your copy> --corpus-id {} \
--detector <name>",
corpus.id
),
"checks your copy against the declared digest first, so the \
result can be quoted beside anybody else's over the same corpus",
));
} else {
steps.push(Step::just(format!(
"{} declares no records digest, so a run over your copy is \
`custom`: comparable with itself alone.",
corpus.id
)));
}
Needs {
readiness: Readiness::Unknown,
steps,
}
}
#[cfg(test)]
mod tests {
use super::*;
fn entry(toml_text: &str) -> Entry {
toml::from_str(toml_text).expect("parses")
}
const SELFTEST: &str = "\n[selftest]\nmust_detect = \"a.png\"\nmust_clear = \"b.png\"\n";
const INVOKE: &str = "\n[invoke]\nargv = [\"{file}\"]\nparser = \"number\"\n";
fn availability(entry: &Entry) -> Availability {
stegobench_plugin::availability::check(entry, &[])
}
fn of_tool(entry: &Entry, availability: &Availability) -> Needs {
super::of_tool(entry, availability, &[])
}
#[test]
fn three_different_kinds_of_tool_answer_in_one_shape() {
let digest = "5".repeat(64);
let container = entry(&format!(
"name = \"c\"\nkind = \"detector\"\nlicence = \"MIT\"\n\
[image]\nreference = \"ghcr.io/x/y@sha256:{digest}\"\nsize_mb = 42\n{INVOKE}{SELFTEST}"
));
let binary = entry(&format!(
"name = \"b\"\nkind = \"detector\"\nlicence = \"MIT\"\n\
upstream = \"https://example.invalid/b\"\n\
[binary]\ncommand = [\"definitely-not-installed-xyzzy\"]\n\
version_args = [\"--version\"]\n{INVOKE}{SELFTEST}"
));
let service = entry(&format!(
"name = \"s\"\nkind = \"detector\"\nlicence = \"MIT\"\n\
[image]\nreference = \"ghcr.io/x/s@sha256:{digest}\"\n\
[invoke]\nhost = true\nadapter = \"plugins/adapters/stegashield_one.py\"\n\
entrypoint = \"python3\"\nargv = [\"{{adapter}}\", \"{{file}}\"]\n\
endpoint_env = \"STEGOBENCH_TEST_ENDPOINT_UNSET\"\nparser = \"number\"\n{SELFTEST}"
));
for e in [&container, &binary, &service] {
let needs = of_tool(e, &availability(e));
assert!(
!needs.steps.is_empty(),
"{} offered no next step at all",
e.name
);
assert!(!needs.block().is_empty(), "{} rendered nothing", e.name);
assert!(needs.to_json()["readiness"].is_string());
}
}
#[test]
fn a_container_is_told_to_pull_the_digest_and_not_a_tag() {
let digest = "5".repeat(64);
let e = entry(&format!(
"name = \"c\"\nkind = \"detector\"\nlicence = \"MIT\"\n\
[image]\nreference = \"ghcr.io/x/y@sha256:{digest}\"\nsize_mb = 42\n\
{INVOKE}{SELFTEST}"
));
let av = Availability {
name: e.name.clone(),
presence: Presence::Absent {
reason: "not pulled".into(),
},
verified: None,
missing_secrets: vec![],
};
let needs = of_tool(&e, &av);
assert_eq!(needs.readiness, Readiness::NeedsYou);
let line = needs.steps[0].run.as_deref().expect("a command to type");
assert!(line.starts_with("docker pull "), "{line}");
assert!(
line.contains(&digest),
"the pull must name the digest: {line}"
);
assert!(
needs.steps[0].why.contains("42 MB"),
"{}",
needs.steps[0].why
);
}
#[test]
fn a_service_is_never_told_to_pull_the_image_that_only_names_it() {
let digest = "5".repeat(64);
let e = entry(&format!(
"name = \"s\"\nkind = \"detector\"\nlicence = \"MIT\"\n\
[image]\nreference = \"ghcr.io/x/s@sha256:{digest}\"\n\
[invoke]\nhost = true\nadapter = \"plugins/adapters/stegashield_one.py\"\n\
entrypoint = \"python3\"\nargv = [\"{{adapter}}\", \"{{file}}\"]\n\
endpoint_env = \"STEGOBENCH_TEST_ENDPOINT_UNSET\"\nparser = \"number\"\n{SELFTEST}"
));
let needs = of_tool(&e, &availability(&e));
let rendered = needs.block();
assert!(
!rendered.contains("docker pull"),
"a service was told to pull the image that only names it:\n{rendered}"
);
assert!(
rendered.contains("export STEGOBENCH_TEST_ENDPOINT_UNSET="),
"{rendered}"
);
assert_eq!(needs.readiness, Readiness::NeedsYou);
}
#[test]
fn an_endpoint_is_named_and_never_filled_in() {
let key = "STEGOBENCH_TEST_ENDPOINT_UNSET";
let digest = "5".repeat(64);
let e = entry(&format!(
"name = \"s\"\nkind = \"detector\"\nlicence = \"MIT\"\n\
[image]\nreference = \"ghcr.io/x/s@sha256:{digest}\"\n\
[invoke]\nhost = true\nadapter = \"plugins/adapters/stegashield_one.py\"\n\
entrypoint = \"python3\"\nargv = [\"{{adapter}}\", \"{{file}}\"]\n\
endpoint_env = \"{key}\"\nparser = \"number\"\n{SELFTEST}"
));
let rendered = of_tool(&e, &availability(&e)).block();
assert!(
!rendered.contains("http://") && !rendered.contains("https://"),
"an address reached the output:\n{rendered}"
);
assert!(
rendered.contains("<the address of your own instance>"),
"{rendered}"
);
}
#[test]
fn a_missing_secret_is_a_step_whatever_the_presence_answer_was() {
let e = entry(&format!(
"name = \"x\"\nkind = \"detector\"\nlicence = \"MIT\"\n\
secrets = [\"STEGOBENCH_TEST_TOKEN_UNSET\"]\n\
[binary]\ncommand = [\"sh\"]\nversion_args = [\"--version\"]\n{INVOKE}{SELFTEST}"
));
let av = Availability {
name: e.name.clone(),
presence: Presence::Present { pin: "x".into() },
verified: None,
missing_secrets: vec!["STEGOBENCH_TEST_TOKEN_UNSET".into()],
};
let needs = of_tool(&e, &av);
assert_eq!(
needs.readiness,
Readiness::NeedsYou,
"a present tool with an unset token was reported ready"
);
assert!(needs
.block()
.contains("export STEGOBENCH_TEST_TOKEN_UNSET="));
assert!(!needs.block().contains("docker pull"));
}
#[test]
fn a_tool_with_everything_in_place_says_so_and_prints_nothing() {
let e = entry(&format!(
"name = \"x\"\nkind = \"detector\"\nlicence = \"MIT\"\n\
[binary]\ncommand = [\"sh\"]\nversion_args = [\"--version\"]\n{INVOKE}{SELFTEST}"
));
let av = Availability {
name: e.name.clone(),
presence: Presence::Present { pin: "x".into() },
verified: None,
missing_secrets: vec![],
};
let needs = of_tool(&e, &av);
assert_eq!(needs.readiness, Readiness::Ready);
assert_eq!(needs.block(), "");
}
#[test]
fn a_tool_that_cannot_run_here_offers_nothing_to_type() {
let e = entry(&format!(
"name = \"x\"\nkind = \"detector\"\nlicence = \"MIT\"\n\
[binary]\ncommand = [\"sh\"]\n{SELFTEST}"
));
let av = Availability {
name: e.name.clone(),
presence: Presence::Unsupported {
reason: "this entry runs on windows and this machine is linux".into(),
},
verified: None,
missing_secrets: vec![],
};
let needs = of_tool(&e, &av);
assert_eq!(needs.readiness, Readiness::CannotRunHere);
assert!(needs.steps.iter().all(|s| s.run.is_none()));
}
#[test]
fn every_shipped_entry_answers_the_question() {
let dir =
std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../plugins/registry");
let reg = stegobench_core::registry::Registry::load(&dir).expect("the real registry loads");
assert!(
reg.entries.len() >= 10,
"walked {} entries; this registry ships thirteen",
reg.entries.len()
);
for e in reg.entries.values() {
let needs = of_tool(e, &availability(e));
if needs.readiness == Readiness::NeedsYou {
assert!(
!needs.steps.is_empty(),
"{} says it needs something and names nothing",
e.name
);
}
}
assert!(!reg.corpora.is_empty());
for c in reg.corpora.values() {
let needs = of_corpus(c);
assert!(!needs.steps.is_empty(), "{} offered no next step", c.id);
assert!(!needs.block().is_empty());
}
}
#[test]
fn a_corpus_that_needs_terms_accepting_says_so_before_it_says_download() {
let dir =
std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../plugins/registry");
let reg = stegobench_core::registry::Registry::load(&dir).expect("loads");
let mut saw_acceptance = false;
for c in reg.corpora.values() {
if !c.obtain.requires_acceptance {
continue;
}
saw_acceptance = true;
let block = of_corpus(c).block();
let accept = block.find("Accept").expect("the acceptance line");
let download = block.find("Download").or_else(|| block.find("Download it"));
if let Some(download) = download {
assert!(
accept < download,
"{} tells the reader to download before it tells them to \
accept the terms:\n{block}",
c.id
);
}
}
assert!(
saw_acceptance,
"no shipped corpus requires acceptance, so this test verified \
nothing. If that is now true of the registry, delete it rather \
than leaving a check that cannot fail"
);
}
}