use std::path::PathBuf;
use std::process::Command;
use std::time::Duration;
use stegobench_core::registry::{Entry, Invoke};
use crate::{binary_version, hash_file, which};
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Presence {
Present { pin: String },
Absent { reason: String },
Unknown { reason: String },
Unsupported { reason: String },
}
impl Presence {
pub fn is_present(&self) -> bool {
matches!(self, Presence::Present { .. })
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct Availability {
pub name: String,
pub presence: Presence,
pub verified: Option<bool>,
pub missing_secrets: Vec<String>,
}
impl Availability {
pub fn summary(&self) -> String {
let state = match &self.presence {
Presence::Present { pin } => {
let short: String = pin.chars().take(23).collect();
format!("present {short}")
}
Presence::Absent { reason } => format!("MISSING {reason}"),
Presence::Unknown { reason } => format!("unknown {reason}"),
Presence::Unsupported { reason } => format!("n/a {reason}"),
};
let verified = match self.verified {
Some(true) => " verified",
Some(false) => " SELF TEST FAILED",
None => " not verified",
};
let secrets = if self.missing_secrets.is_empty() {
String::new()
} else {
format!(" needs {}", self.missing_secrets.join(", "))
};
format!("{:<16} {state}{verified}{secrets}", self.name)
}
}
const DOCKER_TIMEOUT: Duration = Duration::from_secs(20);
fn image_present(reference: &str) -> Presence {
if which("docker").is_none() {
return Presence::Unknown {
reason: "no container runtime: docker is not on PATH".to_string(),
};
}
let mut cmd = Command::new("docker");
cmd.args(["image", "inspect", "--format", "{{.Id}}", reference]);
let out = crate::exec::captured(cmd, "docker image inspect", DOCKER_TIMEOUT);
match out {
Ok(o) if o.status.success() => Presence::Present {
pin: String::from_utf8_lossy(&o.stdout).trim().to_string(),
},
Ok(_) => Presence::Absent {
reason: format!("not pulled. docker pull {reference}"),
},
Err(e) => Presence::Unknown {
reason: format!("could not ask the container runtime: {e}"),
},
}
}
const REGISTRY_TIMEOUT: Duration = Duration::from_secs(45);
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Reach {
Fetchable,
Absent { reason: String },
NotRun { reason: String },
}
pub fn registry_reach(reference: &str) -> Reach {
if which("docker").is_none() {
return Reach::NotRun {
reason: "no container runtime: docker is not on PATH".to_string(),
};
}
let mut cmd = Command::new("docker");
cmd.args(["manifest", "inspect", reference]);
match crate::exec::captured(cmd, "docker manifest inspect", REGISTRY_TIMEOUT) {
Ok(o) if o.status.success() => Reach::Fetchable,
Ok(o) => Reach::Absent {
reason: refusal_reason(&String::from_utf8_lossy(&o.stderr)),
},
Err(e) => Reach::NotRun {
reason: format!("could not ask the registry: {e}"),
},
}
}
fn refusal_reason(stderr: &str) -> String {
stderr
.lines()
.rev()
.find(|line| !line.trim().is_empty())
.map(|line| line.trim().to_string())
.unwrap_or_else(|| "the registry refused and gave no reason".to_string())
}
fn binary_present(entry: &Entry, program: &str) -> Presence {
let Some(path) = which(program) else {
return Presence::Absent {
reason: format!("{program} is not on PATH"),
};
};
match hash_file(&path) {
Ok(pin) => {
let version = binary_version(entry);
Presence::Present {
pin: format!("{} {pin}", version.pin_label()),
}
}
Err(e) => Presence::Unknown {
reason: format!("{} could not be read: {e}", path.display()),
},
}
}
pub fn check(entry: &Entry, adapter_roots: &[PathBuf]) -> Availability {
let presence = match unsupported_here(entry) {
Some(p) => p,
None => present_here(entry, adapter_roots),
};
let missing_secrets = entry
.secrets
.iter()
.filter(|name| std::env::var_os(name).is_none())
.cloned()
.collect();
Availability {
name: entry.name.clone(),
presence,
verified: None,
missing_secrets,
}
}
fn unsupported_here(entry: &Entry) -> Option<Presence> {
let runs_on_this_machine =
entry.binary.is_some() || entry.invoke.as_ref().is_some_and(|i| i.host);
if entry.platforms.is_empty() || !runs_on_this_machine {
return None;
}
let here = std::env::consts::OS;
if entry.platforms.iter().any(|p| p == here) {
return None;
}
Some(Presence::Unsupported {
reason: format!(
"this entry runs on {} and this machine is {here}, so nothing to \
install would make it available",
entry.platforms.join(" and ")
),
})
}
fn host_adapter_present(invoke: &Invoke, adapter_roots: &[PathBuf]) -> Presence {
let Some(rel) = &invoke.adapter else {
return Presence::Unknown {
reason: "invoke.host is set but the entry declares no adapter, so \
it names no program to run"
.into(),
};
};
let adapter = match crate::adapter::resolve(rel, adapter_roots) {
Ok(p) => p,
Err(why) => {
return Presence::Absent {
reason: why.to_string(),
}
}
};
let program = invoke
.entrypoint
.clone()
.unwrap_or_else(|| crate::DEFAULT_HOST_ENTRYPOINT.into());
if which(&program).is_none() {
return Presence::Absent {
reason: format!("{program} is not on PATH, and it is what runs the adapter"),
};
}
if let Some(name) = &invoke.endpoint_env {
let supplied =
std::env::var_os(name).is_some_and(|v| !v.to_string_lossy().trim().is_empty());
if !supplied {
return Presence::Unknown {
reason: format!(
"{name} is not set to an address, so there is no instance \
to ask. Start your own and export it; there is no default"
),
};
}
}
match hash_file(&adapter) {
Ok(pin) => Presence::Present {
pin: format!("adapter {pin}"),
},
Err(e) => Presence::Unknown {
reason: format!("{} could not be read: {e}", adapter.display()),
},
}
}
fn present_here(entry: &Entry, adapter_roots: &[PathBuf]) -> Presence {
if let Some(invoke) = &entry.invoke {
if invoke.host {
return host_adapter_present(invoke, adapter_roots);
}
}
match (&entry.image, &entry.binary) {
(Some(img), _) => image_present(&img.reference),
(_, Some(bin)) => match bin.command.first() {
Some(program) => binary_present(entry, program),
None => Presence::Unknown {
reason: "entry declares an empty command".into(),
},
},
_ => Presence::Unknown {
reason: "entry declares neither an image nor a binary".into(),
},
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_registry_refusal_keeps_the_line_that_says_why() {
let said = "Get \"https://registry-1.docker.io/v2/\": unauthorized\n\
unauthorized: authentication required\n";
assert_eq!(
refusal_reason(said),
"unauthorized: authentication required"
);
}
#[test]
fn a_missing_repository_reads_differently_from_a_forbidden_one() {
assert_eq!(
refusal_reason("denied: requested access to the resource is denied\n"),
"denied: requested access to the resource is denied"
);
}
#[test]
fn a_runtime_that_refused_silently_still_gives_a_reason() {
assert!(!refusal_reason("").is_empty());
assert!(!refusal_reason(" \n\n \n").is_empty());
}
#[test]
fn a_check_that_could_not_run_is_not_a_check_that_passed() {
let could_not = Reach::NotRun {
reason: "no container runtime: docker is not on PATH".into(),
};
let answered_no = Reach::Absent {
reason: "denied: requested access to the resource is denied".into(),
};
assert_ne!(could_not, answered_no);
assert_ne!(could_not, Reach::Fetchable);
assert_ne!(answered_no, Reach::Fetchable);
}
fn check(entry: &Entry) -> Availability {
super::check(entry, &[])
}
fn entry(toml_text: &str) -> Entry {
toml::from_str(toml_text).expect("parses")
}
const SELFTEST: &str = "\n[selftest]\nmust_detect = \"a.png\"\nmust_clear = \"b.png\"\n";
#[cfg(unix)]
#[test]
fn a_version_probe_answered_with_a_score_says_so_in_the_pin() {
use std::os::unix::fs::PermissionsExt;
let dir = tempfile::tempdir().expect("tmp");
let script = dir.path().join("cheat.sh");
std::fs::write(&script, "#!/bin/sh\necho 0.010\n").expect("script");
let mut perms = std::fs::metadata(&script).expect("meta").permissions();
perms.set_mode(0o755);
std::fs::set_permissions(&script, perms).expect("chmod");
let e = entry(&format!(
"name = \"cheat\"\nkind = \"detector\"\nlicence = \"X\"\n\
[binary]\ncommand = [{:?}]\nversion_args = [\"--version\"]\n\
[emits]\noutput = \"score\"\n\
[invoke]\nargv = [\"{{file}}\"]\nparser = \"number\"{SELFTEST}",
script.display().to_string()
));
match check(&e).presence {
Presence::Present { pin } => {
assert!(pin.contains("not a version"), "got {pin}");
assert!(pin.contains("sha256:"), "the hash still pins it: {pin}");
}
other => panic!("the program is on PATH, so it is present: {other:?}"),
}
}
#[test]
fn a_tool_that_cannot_run_here_is_not_reported_as_missing() {
let elsewhere = match std::env::consts::OS {
"linux" => "windows",
_ => "linux",
};
let e = entry(&format!(
"name = \"x\"\nkind = \"detector\"\nlicence = \"X\"\n\
platforms = [\"{elsewhere}\"]\n\
[binary]\ncommand = [\"definitely-not-installed\"]\n{SELFTEST}"
));
let got = check(&e);
match &got.presence {
Presence::Unsupported { reason } => {
assert!(reason.contains(elsewhere), "{reason}");
assert!(reason.contains(std::env::consts::OS), "{reason}");
}
other => panic!("reported as {other:?} rather than unsupported"),
}
assert!(!got.presence.is_present());
}
#[test]
fn a_tool_listing_this_platform_is_checked_normally() {
let e = entry(&format!(
"name = \"sh\"\nkind = \"detector\"\nlicence = \"X\"\n\
platforms = [\"{}\"]\n\
[binary]\ncommand = [\"sh\"]\nversion_args = [\"--version\"]\n{SELFTEST}",
std::env::consts::OS
));
assert!(
!matches!(check(&e).presence, Presence::Unsupported { .. }),
"a tool that names this platform was ruled out on it"
);
}
#[test]
fn a_platform_list_does_not_rule_out_a_container() {
let e = entry(
"name = \"x\"\nkind = \"detector\"\nlicence = \"X\"\n\
platforms = [\"plan9\"]\n\
[image]\nreference = \"r@sha256:abc\"\nsize_mb = 1\nbundled = true\n",
);
assert!(!matches!(check(&e).presence, Presence::Unsupported { .. }));
}
#[test]
fn a_binary_on_path_is_present_and_pinned_by_its_hash() {
let e = entry(&format!(
"name = \"sh\"\nkind = \"detector\"\nlicence = \"X\"\n\
[binary]\ncommand = [\"sh\"]\nversion_args = [\"--version\"]\n{SELFTEST}"
));
let a = check(&e);
assert!(a.presence.is_present(), "got {:?}", a.presence);
if let Presence::Present { pin } = &a.presence {
assert!(pin.contains("sha256:"), "pin should hash the bytes: {pin}");
}
}
#[test]
fn a_missing_binary_says_what_is_missing() {
let e = entry(&format!(
"name = \"nope\"\nkind = \"detector\"\nlicence = \"X\"\n\
[binary]\ncommand = [\"definitely-not-real-xyzzy\"]\nversion_args = [\"-v\"]\n{SELFTEST}"
));
match check(&e).presence {
Presence::Absent { reason } => assert!(reason.contains("not on PATH")),
other => panic!("expected Absent, got {other:?}"),
}
}
#[test]
fn presence_never_claims_the_tool_was_verified() {
let e = entry(&format!(
"name = \"sh\"\nkind = \"detector\"\nlicence = \"X\"\n\
[binary]\ncommand = [\"sh\"]\nversion_args = [\"--version\"]\n{SELFTEST}"
));
let a = check(&e);
assert_eq!(a.verified, None);
assert!(a.summary().contains("not verified"));
}
#[test]
fn an_unset_secret_is_reported_by_name_only() {
let e = entry(&format!(
"name = \"x\"\nkind = \"detector\"\nlicence = \"X\"\n\
secrets = [\"STEGOBENCH_TEST_UNSET_VAR\"]\n\
[binary]\ncommand = [\"sh\"]\nversion_args = [\"--version\"]\n{SELFTEST}"
));
let a = check(&e);
assert_eq!(a.missing_secrets, ["STEGOBENCH_TEST_UNSET_VAR"]);
assert!(a.summary().contains("STEGOBENCH_TEST_UNSET_VAR"));
}
#[test]
fn a_set_secret_is_not_reported_as_missing_and_its_value_never_appears() {
std::env::set_var("STEGOBENCH_TEST_SET_VAR", "swordfish-do-not-print");
let e = entry(&format!(
"name = \"x\"\nkind = \"detector\"\nlicence = \"X\"\n\
secrets = [\"STEGOBENCH_TEST_SET_VAR\"]\n\
[binary]\ncommand = [\"sh\"]\nversion_args = [\"--version\"]\n{SELFTEST}"
));
let a = check(&e);
assert!(a.missing_secrets.is_empty());
assert!(
!format!("{a:?}").contains("swordfish"),
"a secret value must never reach the output"
);
std::env::remove_var("STEGOBENCH_TEST_SET_VAR");
}
#[test]
fn the_stegashield_licence_token_never_leaks_on_either_path() {
let registry_dir =
std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../plugins/registry");
let reg = stegobench_core::registry::Registry::load(®istry_dir)
.expect("the real registry loads");
let stegashield = reg
.entries
.get("stegashield")
.expect("stegashield is registered");
assert_eq!(stegashield.secret_names(), ["STEGASHIELD_LICENCE"]);
std::env::remove_var("STEGASHIELD_LICENCE");
let absent = check(stegashield);
assert_eq!(absent.missing_secrets, ["STEGASHIELD_LICENCE"]);
const SENTINEL: &str = "sk-test-sentinel-do-not-print-9f31c2";
std::env::set_var("STEGASHIELD_LICENCE", SENTINEL);
let present = check(stegashield);
std::env::remove_var("STEGASHIELD_LICENCE");
assert!(present.missing_secrets.is_empty());
let debug_repr = format!("{present:?}");
assert!(
!debug_repr.contains(SENTINEL),
"the token leaked into Availability's Debug output: {debug_repr}"
);
assert!(
!present.summary().contains(SENTINEL),
"the token leaked into the doctor summary line"
);
let describe_json = serde_json::to_string(stegashield).expect("Entry serialises");
assert!(
!describe_json.contains(SENTINEL),
"the token leaked into what `stegobench describe stegashield` would print"
);
}
struct Service {
_dir: tempfile::TempDir,
adapter: std::path::PathBuf,
}
impl Service {
fn new() -> Self {
let dir = tempfile::tempdir().expect("a temporary directory");
let adapter = dir.path().join("service_one.sh");
std::fs::write(&adapter, b"#!/bin/sh\necho 0.9\n").expect("the adapter is written");
Service { _dir: dir, adapter }
}
fn entry(&self, top: &str, invoke_extra: &str) -> Entry {
let e = entry(&format!(
"name = \"svc\"\nkind = \"detector\"\nlicence = \"X\"\n{top}\
[image]\nreference = \"r@sha256:abc\"\nsize_mb = 1250\nbundled = false\n\
[invoke]\nhost = true\nentrypoint = \"sh\"\n\
adapter = \"{}\"\nargv = [\"{{adapter}}\", \"{{file}}\"]\n\
parser = \"number\"\n{invoke_extra}{SELFTEST}",
self.adapter.display()
));
assert!(e.invoke.as_ref().expect("an invoke block").host);
e
}
fn with_endpoint(&self, var: &str) -> Entry {
let e = self.entry("", &format!("endpoint_env = \"{var}\"\n"));
assert_eq!(
e.invoke.as_ref().unwrap().endpoint_env.as_deref(),
Some(var),
"the fixture did not actually declare the variable"
);
e
}
}
#[test]
fn a_service_entry_is_answered_by_its_adapter_and_not_by_its_image() {
let svc = Service::new();
let e = svc.entry("", "");
assert!(
!image_present(&e.image.as_ref().unwrap().reference).is_present(),
"the fixture's image turned out to be pulled, so this proves nothing"
);
let got = check(&e);
assert!(got.presence.is_present(), "got {:?}", got.presence);
}
#[test]
fn a_ready_service_is_pinned_by_the_adapters_bytes_and_says_so() {
let svc = Service::new();
let a = check(&svc.entry("", "")).presence;
let Presence::Present { pin } = &a else {
panic!("expected Present, got {a:?}");
};
assert!(
pin.starts_with("adapter sha256:"),
"the pin must say what it pinned, or it reads as the image digest: {pin}"
);
assert_eq!(
pin,
&format!("adapter {}", hash_file(&svc.adapter).expect("hashes")),
"the pin should be the bytes that actually run here"
);
assert!(check(&svc.entry("", ""))
.summary()
.contains("adapter sha256"));
}
#[test]
fn a_service_whose_endpoint_variable_is_unset_says_which_variable_and_why() {
const VAR: &str = "STEGOBENCH_TEST_UNSET_ENDPOINT_A";
std::env::remove_var(VAR);
let svc = Service::new();
let e = svc.with_endpoint(VAR);
match check(&e).presence {
Presence::Unknown { reason } => {
assert!(reason.contains(VAR), "the variable is not named: {reason}");
assert!(
reason.contains("no default"),
"it should say why there is no default: {reason}"
);
}
other => panic!("expected Unknown naming the variable, got {other:?}"),
}
}
#[test]
fn an_empty_or_blank_endpoint_variable_counts_as_unset() {
for (i, blank) in ["", " ", "\t\n"].iter().enumerate() {
let var = format!("STEGOBENCH_TEST_BLANK_ENDPOINT_{i}");
std::env::set_var(&var, blank);
let svc = Service::new();
let got = check(&svc.with_endpoint(&var)).presence;
std::env::remove_var(&var);
assert!(
matches!(got, Presence::Unknown { .. }),
"{blank:?} was taken for an address: {got:?}"
);
}
}
#[test]
fn a_service_with_an_endpoint_set_is_present_without_anything_being_contacted() {
const VAR: &str = "STEGOBENCH_TEST_SET_ENDPOINT";
std::env::set_var(VAR, "http://198.51.100.7:3000/api/analyze");
let svc = Service::new();
let started = std::time::Instant::now();
let got = check(&svc.with_endpoint(VAR));
std::env::remove_var(VAR);
assert!(got.presence.is_present(), "got {:?}", got.presence);
assert!(
started.elapsed() < std::time::Duration::from_secs(1),
"a presence check that takes a second is one that went and asked"
);
assert_eq!(
got.verified, None,
"present must never be allowed to stand in for working"
);
assert!(got.summary().contains("not verified"));
}
#[test]
fn the_endpoints_value_never_reaches_the_output() {
const VAR: &str = "STEGOBENCH_TEST_ENDPOINT_VALUE";
const SENTINEL: &str = "http://internal-host-do-not-print.invalid:3000/api/analyze";
std::env::set_var(VAR, SENTINEL);
let svc = Service::new();
let got = check(&svc.with_endpoint(VAR));
std::env::remove_var(VAR);
assert!(
!format!("{got:?}").contains("internal-host-do-not-print"),
"the address leaked into the output: {got:?}"
);
}
#[test]
fn a_service_whose_adapter_is_missing_says_where_it_looked_and_not_which_clone() {
let root = tempfile::tempdir().expect("tmp");
let svc = Service::new();
let mut e = svc.entry("", "");
e.invoke.as_mut().unwrap().adapter = Some("plugins/adapters/not-here-xyzzy.py".into());
match super::check(&e, &[root.path().to_path_buf()]).presence {
Presence::Absent { reason } => {
assert!(reason.contains("not-here-xyzzy.py"), "{reason}");
assert!(
reason.contains(&root.path().display().to_string()),
"it does not say where it looked: {reason}"
);
assert!(
!reason.contains("clone"),
"a packaged install is still being sent to a clone: {reason}"
);
}
other => panic!("expected Absent, got {other:?}"),
}
}
#[test]
fn a_service_whose_adapter_is_under_the_root_is_present() {
let root = tempfile::tempdir().expect("tmp");
let adapter = root.path().join("plugins").join("adapters").join("one.py");
std::fs::create_dir_all(adapter.parent().expect("parent")).expect("dirs");
std::fs::write(&adapter, b"#!/usr/bin/env python3\n").expect("written");
let svc = Service::new();
let mut e = svc.entry("", "");
e.invoke.as_mut().unwrap().adapter = Some("plugins/adapters/one.py".into());
assert!(matches!(
check(&e).presence,
Presence::Absent { .. } | Presence::Unknown { .. }
));
match super::check(&e, &[root.path().to_path_buf()]).presence {
Presence::Present { pin } => assert!(pin.starts_with("adapter "), "{pin}"),
other => panic!("expected Present, got {other:?}"),
}
}
#[test]
fn a_service_whose_interpreter_is_not_installed_names_the_interpreter() {
let svc = Service::new();
let mut e = svc.entry("", "");
e.invoke.as_mut().unwrap().entrypoint = Some("definitely-not-a-runtime-xyzzy".into());
match check(&e).presence {
Presence::Absent { reason } => {
assert!(
reason.contains("definitely-not-a-runtime-xyzzy"),
"{reason}"
);
assert!(reason.contains("not on PATH"), "{reason}");
}
other => panic!("expected Absent, got {other:?}"),
}
}
#[test]
fn a_host_entry_with_no_adapter_at_all_is_unknown() {
let svc = Service::new();
let mut e = svc.entry("", "");
e.invoke.as_mut().unwrap().adapter = None;
match check(&e).presence {
Presence::Unknown { reason } => assert!(reason.contains("no adapter"), "{reason}"),
other => panic!("expected Unknown, got {other:?}"),
}
}
#[test]
fn a_platform_list_does_rule_out_a_host_adapter_even_though_it_names_an_image() {
let elsewhere = match std::env::consts::OS {
"linux" => "windows",
_ => "linux",
};
let svc = Service::new();
let e = svc.entry(&format!("platforms = [\"{elsewhere}\"]\n"), "");
match check(&e).presence {
Presence::Unsupported { reason } => assert!(reason.contains(elsewhere), "{reason}"),
other => panic!("expected Unsupported, got {other:?}"),
}
}
#[test]
fn a_host_adapter_listing_this_platform_is_checked_normally() {
let svc = Service::new();
let e = svc.entry(&format!("platforms = [\"{}\"]\n", std::env::consts::OS), "");
assert!(check(&e).presence.is_present(), "{:?}", check(&e).presence);
}
#[test]
fn a_container_entry_without_host_is_unaffected() {
let e = entry(
"name = \"x\"\nkind = \"detector\"\nlicence = \"X\"\n\
[image]\nreference = \"definitely-not-pulled-xyzzy@sha256:abc\"\n\
size_mb = 1\nbundled = true\n\
[invoke]\nadapter = \"plugins/adapters/x.py\"\nargv = [\"{file}\"]\n\
parser = \"number\"\n",
);
match check(&e).presence {
Presence::Absent { reason } => assert!(reason.contains("docker pull"), "{reason}"),
Presence::Unknown { reason } => {
assert!(reason.contains("no container runtime"), "{reason}")
}
other => panic!("a container entry took another route: {other:?}"),
}
}
#[test]
fn a_binary_entry_without_host_is_unaffected() {
let e = entry(&format!(
"name = \"sh\"\nkind = \"detector\"\nlicence = \"X\"\n\
[binary]\ncommand = [\"sh\"]\nversion_args = [\"--version\"]\n\
[invoke]\nargv = [\"{{file}}\"]\nparser = \"number\"\n{SELFTEST}"
));
let a = check(&e).presence;
assert!(a.is_present(), "got {a:?}");
if let Presence::Present { pin } = &a {
assert!(
!pin.starts_with("adapter "),
"a binary entry was pinned as an adapter: {pin}"
);
}
}
#[test]
fn an_entry_with_neither_image_nor_binary_is_unknown_not_absent() {
let mut e = entry(&format!(
"name = \"x\"\nkind = \"detector\"\nlicence = \"X\"\n\
[binary]\ncommand = [\"sh\"]\nversion_args = [\"-v\"]\n{SELFTEST}"
));
e.binary = None;
assert!(matches!(check(&e).presence, Presence::Unknown { .. }));
}
}