stegobench_plugin/lib.rs
1// Author: Daniel Iwugo
2// Comment: Christ is King
3// SPDX-License-Identifier: AGPL-3.0-or-later
4// Copyright (C) 2026 Daniel Iwugo
5//! Running a plugin, and finding out whether one can be run at all.
6//!
7//! THE PROTOCOL, IN FULL
8//! ---------------------
9//! A plugin is a program, or a container image holding one, that answers two
10//! verbs and nothing else.
11//!
12//! `describe` prints one JSON object and exits 0. `run` reads newline
13//! delimited work items on stdin and prints one record per item on stdout,
14//! with progress on stderr.
15//!
16//! Two verbs is the whole contract. Sharding, timeouts, memory caps, container
17//! lifecycle, retries and resumption are the host's job, and no plugin author
18//! ever thinks about them. That is what makes "contribute a detector" mean
19//! writing thirty lines in any language rather than learning our internals.
20//!
21//! AN ERROR IS A RECORD, NOT AN ABSENCE
22//! ------------------------------------
23//! A plugin that cannot score an image says so, in a record carrying the same
24//! id. On 2026-09-17 resume logic here treated an empty failed record as
25//! complete, so a re-run scored nothing while printing "resuming: 1480 files
26//! already scored". A record is done when it carries a score **or** an error,
27//! never when it merely exists.
28
29use std::path::{Path, PathBuf};
30use std::process::Command;
31use std::time::Duration;
32
33use serde::{Deserialize, Serialize};
34use sha2::{Digest, Sha256};
35use stegobench_core::registry::Entry;
36
37pub mod adapter;
38pub mod availability;
39pub mod embed;
40pub mod exec;
41pub mod parsers;
42pub mod runner;
43pub mod selftest;
44
45pub use availability::{registry_reach, Availability, Presence, Reach};
46pub use parsers::Reading;
47pub use selftest::Verified;
48
49/// One item of work handed to a plugin.
50#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
51pub struct WorkItem {
52 /// Stable across the run, and the only thing tying a record back to a file.
53 pub id: String,
54 pub path: PathBuf,
55}
56
57/// What a plugin says about one item.
58///
59/// Exactly one of `score`, `verdict` or `error` is meaningful, and the
60/// [`Record::is_complete`] rule is what resume depends on.
61#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
62pub struct Record {
63 pub id: String,
64 #[serde(default, skip_serializing_if = "Option::is_none")]
65 pub score: Option<f64>,
66 #[serde(default, skip_serializing_if = "Option::is_none")]
67 pub verdict: Option<bool>,
68 #[serde(default, skip_serializing_if = "Option::is_none")]
69 pub error: Option<String>,
70 #[serde(default, skip_serializing_if = "Option::is_none")]
71 pub elapsed_ms: Option<u64>,
72}
73
74impl Record {
75 /// Whether this item needs doing again.
76 ///
77 /// An answer OR an error counts. A record with neither is a placeholder
78 /// the plugin wrote before it worked, and treating that as done is the
79 /// exact bug that made a re-run score nothing while reporting success.
80 pub fn is_complete(&self) -> bool {
81 self.score.is_some() || self.verdict.is_some() || self.error.is_some()
82 }
83}
84
85/// How a plugin describes itself.
86#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
87pub struct Describe {
88 pub protocol: u32,
89 pub name: String,
90 pub version: String,
91}
92
93#[derive(Debug, thiserror::Error)]
94pub enum PluginError {
95 #[error("{name} is not installed: {reason}")]
96 NotInstalled { name: String, reason: String },
97 #[error("{name} could not be run: {source}")]
98 Spawn {
99 name: String,
100 #[source]
101 source: std::io::Error,
102 },
103 #[error("{name} produced output this version cannot read: {reason}")]
104 BadOutput { name: String, reason: String },
105}
106
107/// What runs a host adapter when its entry names no entrypoint.
108///
109/// One constant rather than two literals, because the availability check and
110/// the runner have to agree about which program they are talking about: a
111/// `doctor` line saying the tool is here, against an interpreter a run then
112/// fails to launch, is the exact drift this file's neighbours exist to stop.
113pub const DEFAULT_HOST_ENTRYPOINT: &str = "python3";
114
115/// The SHA256 of a file, which is how a binary plugin is pinned.
116///
117/// Stronger than a tag: a tag can be moved under you by whoever published it,
118/// and a hash of the bytes actually executed cannot.
119pub fn hash_file(path: &Path) -> std::io::Result<String> {
120 let bytes = std::fs::read(path)?;
121 let mut h = Sha256::new();
122 h.update(&bytes);
123 Ok(format!("sha256:{:x}", h.finalize()))
124}
125
126/// Finds an executable the way a shell would, so a registry entry can name
127/// `stegcore` rather than an absolute path that differs per machine.
128pub fn which(program: &str) -> Option<PathBuf> {
129 if program.contains('/') {
130 let p = PathBuf::from(program);
131 return p.is_file().then_some(p);
132 }
133 std::env::var_os("PATH").and_then(|paths| {
134 std::env::split_paths(&paths)
135 .map(|dir| dir.join(program))
136 .find(|p| p.is_file())
137 })
138}
139
140/// How long a tool gets to say what version it is.
141///
142/// A version probe is the cheapest question there is, so the ceiling is for
143/// the tool that does not answer it rather than for the one that is slow.
144const VERSION_TIMEOUT: Duration = Duration::from_secs(10);
145
146/// How many lines of a version probe's output are read before giving up.
147///
148/// Real tools print a banner and put the version on the second or third line.
149/// Nothing prints it on the fortieth, and an unbounded scan over a tool that
150/// answered with its whole help text is a parser with no resource cap.
151const VERSION_LINES_SCANNED: usize = 10;
152
153/// The longest a line may be and still be a version.
154///
155/// Generous on purpose: `zsteg 0.2.13 (c) 2019 ...` is a real version line and
156/// a cap tight enough to be elegant would throw it away. A line longer than
157/// this is prose, a traceback, or a usage message.
158const VERSION_LINE_MAX: usize = 200;
159
160/// What a version probe produced, and whether it can be believed.
161///
162/// Three answers rather than an `Option`, because "it refused to say" and "it
163/// said something that cannot be a version" send a reader to different
164/// problems, and the second used to be recorded as though it were the first.
165#[derive(Debug, Clone, PartialEq, Eq)]
166pub enum VersionProbe {
167 /// It answered, and the answer can be a version.
168 Reported(String),
169 /// It answered, and the answer cannot be recorded as a version. The string
170 /// is a short reason that names what the tool actually printed.
171 Refused(String),
172 /// It could not be asked at all: no binary, not on PATH, no answer inside
173 /// the deadline.
174 NotAsked,
175}
176
177impl VersionProbe {
178 /// The short string that goes in front of the hash in a presence pin.
179 ///
180 /// Kept under forty characters because `doctor` renders it in a column,
181 /// and it carries a digit wherever the tool's own answer did, so the
182 /// renderer prints it rather than collapsing it to "probe failed".
183 pub fn pin_label(&self) -> &str {
184 match self {
185 VersionProbe::Reported(v) => v,
186 VersionProbe::Refused(why) => why,
187 VersionProbe::NotAsked => "unknown version",
188 }
189 }
190}
191
192/// Asks a binary plugin what version it is, using the arguments its entry
193/// declares, and refuses an answer that cannot be a version.
194///
195/// WHY THE ANSWER IS CHECKED AND NOT JUST RECORDED
196/// -----------------------------------------------
197/// An adversarial researcher walked this build on 2026-10-02 with a shell
198/// script that printed a score for every input it was given. Asked `--version`
199/// it printed `0.010`, and the harness wrote `0.010` down as the version of the
200/// thing that produced a number. Nothing anywhere asked whether a version
201/// string looked like a version.
202///
203/// Where the line sits, and why it sits there. Real tools answer in wildly
204/// different shapes (`zsteg 0.2.13`, `v1.4`, `StegExpose 0.0.2`, a bare `1.0`,
205/// a banner with the version on the third line, sometimes on stderr), and a
206/// check strict enough to catch every cheat would throw honest versions away,
207/// which is the worse failure. So only two things are refused:
208///
209/// 1. Output with no version-shaped token anywhere in the lines scanned. A
210/// Python tool once answered `Traceback (most recent call last):` and that
211/// sat in the column every other row uses for a digest. A token is dotted
212/// digits, optionally `v` prefixed, which is what every shape above has and
213/// what a traceback, a usage line and a copyright year do not.
214/// 2. An answer with no letter in it that the tool's OWN output parser reads as
215/// a score. `0.010` is equally a version and this tool's answer about an
216/// image, nothing here can tell which was meant, and guessing is how a
217/// misread becomes a published figure. The cost is that a tool whose entire
218/// version output is a bare `1.0` is refused too; it keeps working, and its
219/// result is pinned by the hash of its bytes rather than by what it says
220/// about itself, which is the stronger pin anyway.
221///
222/// Bounded through [`exec::captured`] rather than `Command::output()`, which
223/// reads both pipes to end of file with no deadline. Measured 2026-09-29: a
224/// tool that prints its version, exits 0, and leaves one background child
225/// holding the write end of the pipe makes `output()` block forever, because
226/// end of file arrives when the LAST holder closes it rather than when the
227/// process we spawned exits. The tool had already answered. Forking a helper
228/// and returning is ordinary behaviour for a shell wrapper, which is what
229/// several registry entries point at.
230pub fn binary_version(entry: &Entry) -> VersionProbe {
231 let Some(bin) = entry.binary.as_ref() else {
232 return VersionProbe::NotAsked;
233 };
234 let Some(program) = bin.command.first() else {
235 return VersionProbe::NotAsked;
236 };
237 let Some(path) = which(program) else {
238 return VersionProbe::NotAsked;
239 };
240 let mut cmd = Command::new(&path);
241 cmd.args(&bin.version_args);
242 let Ok(out) = exec::captured(cmd, program, VERSION_TIMEOUT) else {
243 return VersionProbe::NotAsked;
244 };
245 let text = String::from_utf8_lossy(if out.stdout.is_empty() {
246 &out.stderr
247 } else {
248 &out.stdout
249 });
250 let parser = entry.invoke.as_ref().map(|i| i.parser.as_str());
251 read_version(&text, parser)
252}
253
254/// The version in a probe's output, or the reason there is not one.
255///
256/// Separated from the running so that every shape a real tool prints is a test
257/// rather than a claim. See [`binary_version`] for where the line sits.
258fn read_version(text: &str, parser: Option<&str>) -> VersionProbe {
259 let candidate = text
260 .lines()
261 .take(VERSION_LINES_SCANNED)
262 .map(str::trim)
263 .filter(|line| line.len() <= VERSION_LINE_MAX && !line.chars().any(char::is_control))
264 .find(|line| line.split_whitespace().any(is_version_token));
265
266 let Some(line) = candidate else {
267 // Named rather than summarised: the operator has to be able to see
268 // that the tool answered and what it said, or they go looking for an
269 // installation problem that is not there.
270 let first = text.lines().next().unwrap_or("").trim();
271 return VersionProbe::Refused(format!("no version in: {}", clip(first, 22)));
272 };
273
274 // The cheat's answer: a number and nothing else, which this tool's own
275 // parser reads as its answer about an image. Refused rather than resolved,
276 // and the message says which two readings it could not choose between.
277 let bare_number = !line.chars().any(|c| c.is_ascii_alphabetic());
278 if bare_number {
279 let reads_as_an_answer = parser.is_some_and(|p| {
280 matches!(
281 parsers::parse(p, line, ""),
282 Reading::Score(_) | Reading::Verdict(true)
283 )
284 });
285 if reads_as_an_answer {
286 return VersionProbe::Refused(format!("{} is a score, not a version", clip(line, 12)));
287 }
288 }
289 VersionProbe::Reported(line.to_string())
290}
291
292/// Is this token shaped like a version: `1.4`, `v2.0.0`, `0.2.13-rc1`?
293///
294/// Dotted digits, because that is the one thing every real shape has in common
295/// and the thing a traceback, a usage line and a copyright year all lack. A
296/// bare integer and a date are not accepted, and a tool versioned that way is
297/// refused with its answer named rather than having the answer guessed at.
298fn is_version_token(token: &str) -> bool {
299 let trimmed = token.trim_matches(|c: char| !c.is_ascii_alphanumeric());
300 let stripped = trimmed
301 .strip_prefix('v')
302 .or_else(|| trimmed.strip_prefix('V'))
303 .unwrap_or(trimmed);
304 let head: String = stripped
305 .chars()
306 .take_while(|c| c.is_ascii_digit() || *c == '.')
307 .collect();
308 let mut groups = head.split('.').peekable();
309 let mut count = 0usize;
310 while let Some(group) = groups.next() {
311 // An empty group means two dots in a row or a gap: `1..2` is not a
312 // number anybody versions with. A trailing dot never reaches here,
313 // because the punctuation at either end was trimmed above.
314 if group.is_empty() {
315 return groups.peek().is_none() && count >= 2;
316 }
317 if !group.chars().all(|c| c.is_ascii_digit()) {
318 return false;
319 }
320 count += 1;
321 }
322 count >= 2
323}
324
325/// A string cut to a length that fits a column, with the cut made visible.
326fn clip(text: &str, max: usize) -> String {
327 if text.chars().count() <= max {
328 return text.to_string();
329 }
330 let head: String = text.chars().take(max.saturating_sub(3)).collect();
331 format!("{head}...")
332}
333
334#[cfg(test)]
335mod tests {
336 use super::*;
337
338 /// The hang this whole module's timeout discipline exists to stop, driven
339 /// rather than described.
340 ///
341 /// A tool that prints its version, exits 0, and leaves one background
342 /// child alive is a shell wrapper doing something ordinary. The write end
343 /// of the pipe is inherited by that child, so end of file never arrives,
344 /// and `Command::output()` waits on it for as long as the machine stays
345 /// up. Measured 2026-09-29 with a five line script: `output()` had not
346 /// returned after 25 seconds although the tool answered immediately.
347 ///
348 /// Written to fail rather than to hang if the bound is ever removed: the
349 /// probe runs on its own thread and the assertion is on a channel with a
350 /// deadline, so a regression reports a failure in seconds instead of
351 /// wedging the suite, which is what a test for a hang has to do.
352 #[cfg(unix)]
353 #[test]
354 fn a_version_probe_returns_even_when_the_tool_leaves_a_child_holding_the_pipe() {
355 let dir = tempfile::tempdir().expect("tmp");
356 let script = dir.path().join("forker.sh");
357 std::fs::write(
358 &script,
359 "#!/bin/sh\nsleep 120 &\necho 'forker 9.9'\nexit 0\n",
360 )
361 .expect("written");
362 let mut perms = std::fs::metadata(&script).expect("stat").permissions();
363 std::os::unix::fs::PermissionsExt::set_mode(&mut perms, 0o755);
364 std::fs::set_permissions(&script, perms).expect("chmod");
365
366 let entry: Entry = toml::from_str(&format!(
367 "name = \"forker\"\nkind = \"detector\"\nlicence = \"MIT\"\n\
368 [binary]\ncommand = [\"{}\"]\nversion_args = [\"--version\"]\n\
369 [selftest]\nmust_detect = \"a.png\"\nmust_clear = \"b.png\"\n",
370 script.display()
371 ))
372 .expect("the entry parses");
373
374 let (tx, rx) = std::sync::mpsc::channel();
375 std::thread::spawn(move || {
376 let _ = tx.send(binary_version(&entry));
377 });
378
379 // Comfortably above VERSION_TIMEOUT and far below the 120s the
380 // grandchild lives for, so this can only pass because the probe is
381 // bounded.
382 match rx.recv_timeout(Duration::from_secs(VERSION_TIMEOUT.as_secs() + 20)) {
383 Ok(version) => assert_eq!(
384 version,
385 VersionProbe::Reported("forker 9.9".into()),
386 "the tool did answer, so its answer should survive the bound"
387 ),
388 Err(_) => panic!(
389 "the version probe never returned: a child holding the pipe \
390 open is enough to hang it again"
391 ),
392 }
393 }
394
395 /// Every shape a real tool was observed to answer in, kept together so
396 /// that tightening the check has to face all of them at once.
397 #[test]
398 fn the_shapes_real_tools_answer_in_are_all_recorded() {
399 for answer in [
400 "zsteg 0.2.13",
401 "v1.4",
402 "StegExpose 0.0.2",
403 "stegcore 1.2.3-rc1",
404 "zsteg 0.2.13 (c) 2019 Andrey Zaikin, all rights reserved",
405 "tool version 1.4.",
406 ] {
407 assert_eq!(
408 read_version(answer, Some("number")),
409 VersionProbe::Reported(answer.trim().to_string()),
410 "an honest version was refused: {answer:?}"
411 );
412 }
413 }
414
415 #[test]
416 fn a_version_on_the_third_line_of_a_banner_is_found() {
417 // A banner is ordinary. The first line carries no version-shaped
418 // token, so the scan goes on rather than recording the product name.
419 assert_eq!(
420 read_version("MyTool\nCopyright (c) 2019 nobody\n1.4.2\n", Some("number")),
421 VersionProbe::Reported("1.4.2".into()),
422 "the copyright year must not be mistaken for the version either"
423 );
424 }
425
426 #[test]
427 fn a_traceback_is_refused_rather_than_recorded_as_a_version() {
428 // The failure this check was written beside: a Python tool answered
429 // with a traceback and it sat in the column every other row uses for
430 // a digest. The second line carries a digit, so a looser rule would
431 // have recorded `File "x.py", line 1` as the version.
432 match read_version(
433 "Traceback (most recent call last):\n File \"x.py\", line 1, in <module>\n",
434 Some("number"),
435 ) {
436 VersionProbe::Refused(why) => assert!(why.contains("Traceback"), "got {why}"),
437 other => panic!("expected a refusal, got {other:?}"),
438 }
439 }
440
441 #[test]
442 fn a_tool_that_answers_the_version_probe_with_its_score_is_refused() {
443 // Rung 5, 2026-10-02: a shell script that printed a score for every
444 // input printed `0.010` for `--version` too, and the harness wrote
445 // that down as the version of the thing that produced a number.
446 match read_version("0.010\n", Some("number")) {
447 VersionProbe::Refused(why) => {
448 assert!(why.contains("0.010"), "the answer is named: {why}");
449 assert!(why.contains("score"), "and why it was refused: {why}");
450 }
451 other => panic!("expected a refusal, got {other:?}"),
452 }
453 }
454
455 #[test]
456 fn a_bare_number_is_refused_only_where_the_tools_own_parser_reads_it() {
457 // The line is drawn at ambiguity, not at shape. `1.0` is a perfectly
458 // ordinary version, and it is refused for an entry whose parser reads
459 // a bare number as an answer about an image, because nothing here can
460 // tell which of the two readings was meant. For a tool that answers in
461 // JSON there is no second reading, so it is recorded.
462 assert!(matches!(
463 read_version("1.0\n", Some("number")),
464 VersionProbe::Refused(_)
465 ));
466 assert_eq!(
467 read_version("1.0\n", Some("stegcore")),
468 VersionProbe::Reported("1.0".into())
469 );
470 }
471
472 #[test]
473 fn a_usage_message_and_an_empty_answer_are_both_refused() {
474 for answer in ["", "usage: tool [-h] [--version]\n", " \n"] {
475 assert!(
476 matches!(
477 read_version(answer, Some("number")),
478 VersionProbe::Refused(_)
479 ),
480 "this is not a version: {answer:?}"
481 );
482 }
483 }
484
485 #[test]
486 fn a_probe_that_floods_the_output_is_bounded() {
487 // A tool that answers the version flag with its whole manual. The
488 // scan stops after a fixed number of lines, so the version buried
489 // below is not found and the answer is refused rather than the parser
490 // reading megabytes to find out.
491 let mut flood = "no version here\n".repeat(VERSION_LINES_SCANNED + 5);
492 flood.push_str("1.2.3\n");
493 assert!(matches!(
494 read_version(&flood, Some("number")),
495 VersionProbe::Refused(_)
496 ));
497 }
498
499 #[test]
500 fn an_overlong_line_is_not_a_version_even_with_a_number_in_it() {
501 let long = format!("{} 1.2.3", "x".repeat(VERSION_LINE_MAX));
502 assert!(matches!(
503 read_version(&long, Some("number")),
504 VersionProbe::Refused(_)
505 ));
506 }
507
508 #[test]
509 fn a_pin_label_is_short_enough_for_the_column_it_is_printed_in() {
510 // `doctor` renders this in a forty character column. A refusal that
511 // overflows it is a refusal nobody reads.
512 for probe in [
513 read_version("0.010\n", Some("number")),
514 read_version("Traceback (most recent call last):\n", Some("number")),
515 VersionProbe::NotAsked,
516 ] {
517 let label = probe.pin_label();
518 assert!(
519 label.chars().count() <= 40,
520 "{label:?} is {} characters",
521 label.chars().count()
522 );
523 }
524 }
525
526 #[test]
527 fn a_record_with_a_score_is_complete() {
528 let r = Record {
529 id: "1".into(),
530 score: Some(0.7),
531 verdict: None,
532 error: None,
533 elapsed_ms: None,
534 };
535 assert!(r.is_complete());
536 }
537
538 #[test]
539 fn a_record_with_an_error_is_also_complete() {
540 // An error is an answer. Re-running it would fail the same way, and
541 // treating it as outstanding makes a resume loop forever.
542 let r = Record {
543 id: "1".into(),
544 score: None,
545 verdict: None,
546 error: Some("decode failed".into()),
547 elapsed_ms: None,
548 };
549 assert!(r.is_complete());
550 }
551
552 #[test]
553 fn an_empty_placeholder_is_not_complete() {
554 // The 2026-09-17 bug, pinned. The panel writes a record before it
555 // scores, and counting that as done made a re-run print "resuming:
556 // 1480 files already scored" and then score nothing.
557 let r = Record {
558 id: "1".into(),
559 score: None,
560 verdict: None,
561 error: None,
562 elapsed_ms: None,
563 };
564 assert!(!r.is_complete());
565 }
566
567 #[test]
568 fn a_verdict_only_plugin_still_produces_complete_records() {
569 // zsteg answers yes or no and never a number. That is a property of
570 // the tool, not a failure, and resume must not treat it as unfinished.
571 let r = Record {
572 id: "1".into(),
573 score: None,
574 verdict: Some(true),
575 error: None,
576 elapsed_ms: None,
577 };
578 assert!(r.is_complete());
579 }
580
581 #[test]
582 fn hashing_a_file_is_stable_and_prefixed() {
583 let dir = tempfile::tempdir().unwrap();
584 let p = dir.path().join("x");
585 std::fs::write(&p, b"hello").unwrap();
586 let a = hash_file(&p).unwrap();
587 assert_eq!(a, hash_file(&p).unwrap());
588 assert!(a.starts_with("sha256:"));
589 }
590
591 #[test]
592 fn hashing_notices_a_changed_byte() {
593 let dir = tempfile::tempdir().unwrap();
594 let p = dir.path().join("x");
595 std::fs::write(&p, b"hello").unwrap();
596 let before = hash_file(&p).unwrap();
597 std::fs::write(&p, b"hellp").unwrap();
598 assert_ne!(before, hash_file(&p).unwrap());
599 }
600
601 #[cfg(unix)]
602 #[test]
603 fn which_finds_something_that_exists_and_not_something_that_does_not() {
604 assert!(which("sh").is_some(), "sh should be on PATH");
605 assert!(which("definitely-not-a-real-program-xyzzy").is_none());
606 }
607
608 #[cfg(unix)]
609 #[test]
610 fn an_absolute_path_is_used_directly() {
611 assert!(which("/bin/sh").is_some() || which("/usr/bin/sh").is_some());
612 assert!(which("/nope/nothing").is_none());
613 }
614
615 #[test]
616 fn records_round_trip_and_omit_what_is_absent() {
617 let r = Record {
618 id: "42".into(),
619 score: Some(0.5),
620 verdict: None,
621 error: None,
622 elapsed_ms: Some(10),
623 };
624 let text = serde_json::to_string(&r).unwrap();
625 assert!(
626 !text.contains("verdict"),
627 "absent fields should not be written"
628 );
629 assert_eq!(serde_json::from_str::<Record>(&text).unwrap(), r);
630 }
631}