Skip to main content

stegobench_plugin/
lib.rs

1// Author:  Daniel Iwugo
2// Comment: Christ is King
3// SPDX-License-Identifier: AGPL-3.0-or-later
4// Copyright (C) 2026 Daniel Iwugo
5//! Running a plugin, and finding out whether one can be run at all.
6//!
7//! THE PROTOCOL, IN FULL
8//! ---------------------
9//! A plugin is a program, or a container image holding one, that answers two
10//! verbs and nothing else.
11//!
12//! `describe` prints one JSON object and exits 0. `run` reads newline
13//! delimited work items on stdin and prints one record per item on stdout,
14//! with progress on stderr.
15//!
16//! Two verbs is the whole contract. Sharding, timeouts, memory caps, container
17//! lifecycle, retries and resumption are the host's job, and no plugin author
18//! ever thinks about them. That is what makes "contribute a detector" mean
19//! writing thirty lines in any language rather than learning our internals.
20//!
21//! AN ERROR IS A RECORD, NOT AN ABSENCE
22//! ------------------------------------
23//! A plugin that cannot score an image says so, in a record carrying the same
24//! id. On 2026-09-17 resume logic here treated an empty failed record as
25//! complete, so a re-run scored nothing while printing "resuming: 1480 files
26//! already scored". A record is done when it carries a score **or** an error,
27//! never when it merely exists.
28
29use std::path::{Path, PathBuf};
30use std::process::Command;
31use std::time::Duration;
32
33use serde::{Deserialize, Serialize};
34use sha2::{Digest, Sha256};
35use stegobench_core::registry::Entry;
36
37pub mod adapter;
38pub mod availability;
39pub mod embed;
40pub mod exec;
41pub mod parsers;
42pub mod runner;
43pub mod selftest;
44
45pub use availability::{registry_reach, Availability, Presence, Reach};
46pub use parsers::Reading;
47pub use selftest::Verified;
48
49/// One item of work handed to a plugin.
50#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
51pub struct WorkItem {
52    /// Stable across the run, and the only thing tying a record back to a file.
53    pub id: String,
54    pub path: PathBuf,
55}
56
57/// What a plugin says about one item.
58///
59/// Exactly one of `score`, `verdict` or `error` is meaningful, and the
60/// [`Record::is_complete`] rule is what resume depends on.
61#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
62pub struct Record {
63    pub id: String,
64    #[serde(default, skip_serializing_if = "Option::is_none")]
65    pub score: Option<f64>,
66    #[serde(default, skip_serializing_if = "Option::is_none")]
67    pub verdict: Option<bool>,
68    #[serde(default, skip_serializing_if = "Option::is_none")]
69    pub error: Option<String>,
70    #[serde(default, skip_serializing_if = "Option::is_none")]
71    pub elapsed_ms: Option<u64>,
72}
73
74impl Record {
75    /// Whether this item needs doing again.
76    ///
77    /// An answer OR an error counts. A record with neither is a placeholder
78    /// the plugin wrote before it worked, and treating that as done is the
79    /// exact bug that made a re-run score nothing while reporting success.
80    pub fn is_complete(&self) -> bool {
81        self.score.is_some() || self.verdict.is_some() || self.error.is_some()
82    }
83}
84
85/// How a plugin describes itself.
86#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
87pub struct Describe {
88    pub protocol: u32,
89    pub name: String,
90    pub version: String,
91}
92
93#[derive(Debug, thiserror::Error)]
94pub enum PluginError {
95    #[error("{name} is not installed: {reason}")]
96    NotInstalled { name: String, reason: String },
97    #[error("{name} could not be run: {source}")]
98    Spawn {
99        name: String,
100        #[source]
101        source: std::io::Error,
102    },
103    #[error("{name} produced output this version cannot read: {reason}")]
104    BadOutput { name: String, reason: String },
105}
106
107/// What runs a host adapter when its entry names no entrypoint.
108///
109/// One constant rather than two literals, because the availability check and
110/// the runner have to agree about which program they are talking about: a
111/// `doctor` line saying the tool is here, against an interpreter a run then
112/// fails to launch, is the exact drift this file's neighbours exist to stop.
113pub const DEFAULT_HOST_ENTRYPOINT: &str = "python3";
114
115/// The SHA256 of a file, which is how a binary plugin is pinned.
116///
117/// Stronger than a tag: a tag can be moved under you by whoever published it,
118/// and a hash of the bytes actually executed cannot.
119pub fn hash_file(path: &Path) -> std::io::Result<String> {
120    let bytes = std::fs::read(path)?;
121    let mut h = Sha256::new();
122    h.update(&bytes);
123    Ok(format!("sha256:{:x}", h.finalize()))
124}
125
126/// Finds an executable the way a shell would, so a registry entry can name
127/// `stegcore` rather than an absolute path that differs per machine.
128pub fn which(program: &str) -> Option<PathBuf> {
129    if program.contains('/') {
130        let p = PathBuf::from(program);
131        return p.is_file().then_some(p);
132    }
133    std::env::var_os("PATH").and_then(|paths| {
134        std::env::split_paths(&paths)
135            .map(|dir| dir.join(program))
136            .find(|p| p.is_file())
137    })
138}
139
140/// How long a tool gets to say what version it is.
141///
142/// A version probe is the cheapest question there is, so the ceiling is for
143/// the tool that does not answer it rather than for the one that is slow.
144const VERSION_TIMEOUT: Duration = Duration::from_secs(10);
145
146/// How many lines of a version probe's output are read before giving up.
147///
148/// Real tools print a banner and put the version on the second or third line.
149/// Nothing prints it on the fortieth, and an unbounded scan over a tool that
150/// answered with its whole help text is a parser with no resource cap.
151const VERSION_LINES_SCANNED: usize = 10;
152
153/// The longest a line may be and still be a version.
154///
155/// Generous on purpose: `zsteg 0.2.13 (c) 2019 ...` is a real version line and
156/// a cap tight enough to be elegant would throw it away. A line longer than
157/// this is prose, a traceback, or a usage message.
158const VERSION_LINE_MAX: usize = 200;
159
160/// What a version probe produced, and whether it can be believed.
161///
162/// Three answers rather than an `Option`, because "it refused to say" and "it
163/// said something that cannot be a version" send a reader to different
164/// problems, and the second used to be recorded as though it were the first.
165#[derive(Debug, Clone, PartialEq, Eq)]
166pub enum VersionProbe {
167    /// It answered, and the answer can be a version.
168    Reported(String),
169    /// It answered, and the answer cannot be recorded as a version. The string
170    /// is a short reason that names what the tool actually printed.
171    Refused(String),
172    /// It could not be asked at all: no binary, not on PATH, no answer inside
173    /// the deadline.
174    NotAsked,
175}
176
177impl VersionProbe {
178    /// The short string that goes in front of the hash in a presence pin.
179    ///
180    /// Kept under forty characters because `doctor` renders it in a column,
181    /// and it carries a digit wherever the tool's own answer did, so the
182    /// renderer prints it rather than collapsing it to "probe failed".
183    pub fn pin_label(&self) -> &str {
184        match self {
185            VersionProbe::Reported(v) => v,
186            VersionProbe::Refused(why) => why,
187            VersionProbe::NotAsked => "unknown version",
188        }
189    }
190}
191
192/// Asks a binary plugin what version it is, using the arguments its entry
193/// declares, and refuses an answer that cannot be a version.
194///
195/// WHY THE ANSWER IS CHECKED AND NOT JUST RECORDED
196/// -----------------------------------------------
197/// An adversarial researcher walked this build on 2026-10-02 with a shell
198/// script that printed a score for every input it was given. Asked `--version`
199/// it printed `0.010`, and the harness wrote `0.010` down as the version of the
200/// thing that produced a number. Nothing anywhere asked whether a version
201/// string looked like a version.
202///
203/// Where the line sits, and why it sits there. Real tools answer in wildly
204/// different shapes (`zsteg 0.2.13`, `v1.4`, `StegExpose 0.0.2`, a bare `1.0`,
205/// a banner with the version on the third line, sometimes on stderr), and a
206/// check strict enough to catch every cheat would throw honest versions away,
207/// which is the worse failure. So only two things are refused:
208///
209/// 1. Output with no version-shaped token anywhere in the lines scanned. A
210///    Python tool once answered `Traceback (most recent call last):` and that
211///    sat in the column every other row uses for a digest. A token is dotted
212///    digits, optionally `v` prefixed, which is what every shape above has and
213///    what a traceback, a usage line and a copyright year do not.
214/// 2. An answer with no letter in it that the tool's OWN output parser reads as
215///    a score. `0.010` is equally a version and this tool's answer about an
216///    image, nothing here can tell which was meant, and guessing is how a
217///    misread becomes a published figure. The cost is that a tool whose entire
218///    version output is a bare `1.0` is refused too; it keeps working, and its
219///    result is pinned by the hash of its bytes rather than by what it says
220///    about itself, which is the stronger pin anyway.
221///
222/// Bounded through [`exec::captured`] rather than `Command::output()`, which
223/// reads both pipes to end of file with no deadline. Measured 2026-09-29: a
224/// tool that prints its version, exits 0, and leaves one background child
225/// holding the write end of the pipe makes `output()` block forever, because
226/// end of file arrives when the LAST holder closes it rather than when the
227/// process we spawned exits. The tool had already answered. Forking a helper
228/// and returning is ordinary behaviour for a shell wrapper, which is what
229/// several registry entries point at.
230pub fn binary_version(entry: &Entry) -> VersionProbe {
231    let Some(bin) = entry.binary.as_ref() else {
232        return VersionProbe::NotAsked;
233    };
234    let Some(program) = bin.command.first() else {
235        return VersionProbe::NotAsked;
236    };
237    let Some(path) = which(program) else {
238        return VersionProbe::NotAsked;
239    };
240    let mut cmd = Command::new(&path);
241    cmd.args(&bin.version_args);
242    let Ok(out) = exec::captured(cmd, program, VERSION_TIMEOUT) else {
243        return VersionProbe::NotAsked;
244    };
245    let text = String::from_utf8_lossy(if out.stdout.is_empty() {
246        &out.stderr
247    } else {
248        &out.stdout
249    });
250    let parser = entry.invoke.as_ref().map(|i| i.parser.as_str());
251    read_version(&text, parser)
252}
253
254/// The version in a probe's output, or the reason there is not one.
255///
256/// Separated from the running so that every shape a real tool prints is a test
257/// rather than a claim. See [`binary_version`] for where the line sits.
258fn read_version(text: &str, parser: Option<&str>) -> VersionProbe {
259    let candidate = text
260        .lines()
261        .take(VERSION_LINES_SCANNED)
262        .map(str::trim)
263        .filter(|line| line.len() <= VERSION_LINE_MAX && !line.chars().any(char::is_control))
264        .find(|line| line.split_whitespace().any(is_version_token));
265
266    let Some(line) = candidate else {
267        // Named rather than summarised: the operator has to be able to see
268        // that the tool answered and what it said, or they go looking for an
269        // installation problem that is not there.
270        let first = text.lines().next().unwrap_or("").trim();
271        return VersionProbe::Refused(format!("no version in: {}", clip(first, 22)));
272    };
273
274    // The cheat's answer: a number and nothing else, which this tool's own
275    // parser reads as its answer about an image. Refused rather than resolved,
276    // and the message says which two readings it could not choose between.
277    let bare_number = !line.chars().any(|c| c.is_ascii_alphabetic());
278    if bare_number {
279        let reads_as_an_answer = parser.is_some_and(|p| {
280            matches!(
281                parsers::parse(p, line, ""),
282                Reading::Score(_) | Reading::Verdict(true)
283            )
284        });
285        if reads_as_an_answer {
286            return VersionProbe::Refused(format!("{} is a score, not a version", clip(line, 12)));
287        }
288    }
289    VersionProbe::Reported(line.to_string())
290}
291
292/// Is this token shaped like a version: `1.4`, `v2.0.0`, `0.2.13-rc1`?
293///
294/// Dotted digits, because that is the one thing every real shape has in common
295/// and the thing a traceback, a usage line and a copyright year all lack. A
296/// bare integer and a date are not accepted, and a tool versioned that way is
297/// refused with its answer named rather than having the answer guessed at.
298fn is_version_token(token: &str) -> bool {
299    let trimmed = token.trim_matches(|c: char| !c.is_ascii_alphanumeric());
300    let stripped = trimmed
301        .strip_prefix('v')
302        .or_else(|| trimmed.strip_prefix('V'))
303        .unwrap_or(trimmed);
304    let head: String = stripped
305        .chars()
306        .take_while(|c| c.is_ascii_digit() || *c == '.')
307        .collect();
308    let mut groups = head.split('.').peekable();
309    let mut count = 0usize;
310    while let Some(group) = groups.next() {
311        // An empty group means two dots in a row or a gap: `1..2` is not a
312        // number anybody versions with. A trailing dot never reaches here,
313        // because the punctuation at either end was trimmed above.
314        if group.is_empty() {
315            return groups.peek().is_none() && count >= 2;
316        }
317        if !group.chars().all(|c| c.is_ascii_digit()) {
318            return false;
319        }
320        count += 1;
321    }
322    count >= 2
323}
324
325/// A string cut to a length that fits a column, with the cut made visible.
326fn clip(text: &str, max: usize) -> String {
327    if text.chars().count() <= max {
328        return text.to_string();
329    }
330    let head: String = text.chars().take(max.saturating_sub(3)).collect();
331    format!("{head}...")
332}
333
334#[cfg(test)]
335mod tests {
336    use super::*;
337
338    /// The hang this whole module's timeout discipline exists to stop, driven
339    /// rather than described.
340    ///
341    /// A tool that prints its version, exits 0, and leaves one background
342    /// child alive is a shell wrapper doing something ordinary. The write end
343    /// of the pipe is inherited by that child, so end of file never arrives,
344    /// and `Command::output()` waits on it for as long as the machine stays
345    /// up. Measured 2026-09-29 with a five line script: `output()` had not
346    /// returned after 25 seconds although the tool answered immediately.
347    ///
348    /// Written to fail rather than to hang if the bound is ever removed: the
349    /// probe runs on its own thread and the assertion is on a channel with a
350    /// deadline, so a regression reports a failure in seconds instead of
351    /// wedging the suite, which is what a test for a hang has to do.
352    #[cfg(unix)]
353    #[test]
354    fn a_version_probe_returns_even_when_the_tool_leaves_a_child_holding_the_pipe() {
355        let dir = tempfile::tempdir().expect("tmp");
356        let script = dir.path().join("forker.sh");
357        std::fs::write(
358            &script,
359            "#!/bin/sh\nsleep 120 &\necho 'forker 9.9'\nexit 0\n",
360        )
361        .expect("written");
362        let mut perms = std::fs::metadata(&script).expect("stat").permissions();
363        std::os::unix::fs::PermissionsExt::set_mode(&mut perms, 0o755);
364        std::fs::set_permissions(&script, perms).expect("chmod");
365
366        let entry: Entry = toml::from_str(&format!(
367            "name = \"forker\"\nkind = \"detector\"\nlicence = \"MIT\"\n\
368             [binary]\ncommand = [\"{}\"]\nversion_args = [\"--version\"]\n\
369             [selftest]\nmust_detect = \"a.png\"\nmust_clear = \"b.png\"\n",
370            script.display()
371        ))
372        .expect("the entry parses");
373
374        let (tx, rx) = std::sync::mpsc::channel();
375        std::thread::spawn(move || {
376            let _ = tx.send(binary_version(&entry));
377        });
378
379        // Comfortably above VERSION_TIMEOUT and far below the 120s the
380        // grandchild lives for, so this can only pass because the probe is
381        // bounded.
382        match rx.recv_timeout(Duration::from_secs(VERSION_TIMEOUT.as_secs() + 20)) {
383            Ok(version) => assert_eq!(
384                version,
385                VersionProbe::Reported("forker 9.9".into()),
386                "the tool did answer, so its answer should survive the bound"
387            ),
388            Err(_) => panic!(
389                "the version probe never returned: a child holding the pipe \
390                 open is enough to hang it again"
391            ),
392        }
393    }
394
395    /// Every shape a real tool was observed to answer in, kept together so
396    /// that tightening the check has to face all of them at once.
397    #[test]
398    fn the_shapes_real_tools_answer_in_are_all_recorded() {
399        for answer in [
400            "zsteg 0.2.13",
401            "v1.4",
402            "StegExpose 0.0.2",
403            "stegcore 1.2.3-rc1",
404            "zsteg 0.2.13 (c) 2019 Andrey Zaikin, all rights reserved",
405            "tool version 1.4.",
406        ] {
407            assert_eq!(
408                read_version(answer, Some("number")),
409                VersionProbe::Reported(answer.trim().to_string()),
410                "an honest version was refused: {answer:?}"
411            );
412        }
413    }
414
415    #[test]
416    fn a_version_on_the_third_line_of_a_banner_is_found() {
417        // A banner is ordinary. The first line carries no version-shaped
418        // token, so the scan goes on rather than recording the product name.
419        assert_eq!(
420            read_version("MyTool\nCopyright (c) 2019 nobody\n1.4.2\n", Some("number")),
421            VersionProbe::Reported("1.4.2".into()),
422            "the copyright year must not be mistaken for the version either"
423        );
424    }
425
426    #[test]
427    fn a_traceback_is_refused_rather_than_recorded_as_a_version() {
428        // The failure this check was written beside: a Python tool answered
429        // with a traceback and it sat in the column every other row uses for
430        // a digest. The second line carries a digit, so a looser rule would
431        // have recorded `File "x.py", line 1` as the version.
432        match read_version(
433            "Traceback (most recent call last):\n  File \"x.py\", line 1, in <module>\n",
434            Some("number"),
435        ) {
436            VersionProbe::Refused(why) => assert!(why.contains("Traceback"), "got {why}"),
437            other => panic!("expected a refusal, got {other:?}"),
438        }
439    }
440
441    #[test]
442    fn a_tool_that_answers_the_version_probe_with_its_score_is_refused() {
443        // Rung 5, 2026-10-02: a shell script that printed a score for every
444        // input printed `0.010` for `--version` too, and the harness wrote
445        // that down as the version of the thing that produced a number.
446        match read_version("0.010\n", Some("number")) {
447            VersionProbe::Refused(why) => {
448                assert!(why.contains("0.010"), "the answer is named: {why}");
449                assert!(why.contains("score"), "and why it was refused: {why}");
450            }
451            other => panic!("expected a refusal, got {other:?}"),
452        }
453    }
454
455    #[test]
456    fn a_bare_number_is_refused_only_where_the_tools_own_parser_reads_it() {
457        // The line is drawn at ambiguity, not at shape. `1.0` is a perfectly
458        // ordinary version, and it is refused for an entry whose parser reads
459        // a bare number as an answer about an image, because nothing here can
460        // tell which of the two readings was meant. For a tool that answers in
461        // JSON there is no second reading, so it is recorded.
462        assert!(matches!(
463            read_version("1.0\n", Some("number")),
464            VersionProbe::Refused(_)
465        ));
466        assert_eq!(
467            read_version("1.0\n", Some("stegcore")),
468            VersionProbe::Reported("1.0".into())
469        );
470    }
471
472    #[test]
473    fn a_usage_message_and_an_empty_answer_are_both_refused() {
474        for answer in ["", "usage: tool [-h] [--version]\n", "   \n"] {
475            assert!(
476                matches!(
477                    read_version(answer, Some("number")),
478                    VersionProbe::Refused(_)
479                ),
480                "this is not a version: {answer:?}"
481            );
482        }
483    }
484
485    #[test]
486    fn a_probe_that_floods_the_output_is_bounded() {
487        // A tool that answers the version flag with its whole manual. The
488        // scan stops after a fixed number of lines, so the version buried
489        // below is not found and the answer is refused rather than the parser
490        // reading megabytes to find out.
491        let mut flood = "no version here\n".repeat(VERSION_LINES_SCANNED + 5);
492        flood.push_str("1.2.3\n");
493        assert!(matches!(
494            read_version(&flood, Some("number")),
495            VersionProbe::Refused(_)
496        ));
497    }
498
499    #[test]
500    fn an_overlong_line_is_not_a_version_even_with_a_number_in_it() {
501        let long = format!("{} 1.2.3", "x".repeat(VERSION_LINE_MAX));
502        assert!(matches!(
503            read_version(&long, Some("number")),
504            VersionProbe::Refused(_)
505        ));
506    }
507
508    #[test]
509    fn a_pin_label_is_short_enough_for_the_column_it_is_printed_in() {
510        // `doctor` renders this in a forty character column. A refusal that
511        // overflows it is a refusal nobody reads.
512        for probe in [
513            read_version("0.010\n", Some("number")),
514            read_version("Traceback (most recent call last):\n", Some("number")),
515            VersionProbe::NotAsked,
516        ] {
517            let label = probe.pin_label();
518            assert!(
519                label.chars().count() <= 40,
520                "{label:?} is {} characters",
521                label.chars().count()
522            );
523        }
524    }
525
526    #[test]
527    fn a_record_with_a_score_is_complete() {
528        let r = Record {
529            id: "1".into(),
530            score: Some(0.7),
531            verdict: None,
532            error: None,
533            elapsed_ms: None,
534        };
535        assert!(r.is_complete());
536    }
537
538    #[test]
539    fn a_record_with_an_error_is_also_complete() {
540        // An error is an answer. Re-running it would fail the same way, and
541        // treating it as outstanding makes a resume loop forever.
542        let r = Record {
543            id: "1".into(),
544            score: None,
545            verdict: None,
546            error: Some("decode failed".into()),
547            elapsed_ms: None,
548        };
549        assert!(r.is_complete());
550    }
551
552    #[test]
553    fn an_empty_placeholder_is_not_complete() {
554        // The 2026-09-17 bug, pinned. The panel writes a record before it
555        // scores, and counting that as done made a re-run print "resuming:
556        // 1480 files already scored" and then score nothing.
557        let r = Record {
558            id: "1".into(),
559            score: None,
560            verdict: None,
561            error: None,
562            elapsed_ms: None,
563        };
564        assert!(!r.is_complete());
565    }
566
567    #[test]
568    fn a_verdict_only_plugin_still_produces_complete_records() {
569        // zsteg answers yes or no and never a number. That is a property of
570        // the tool, not a failure, and resume must not treat it as unfinished.
571        let r = Record {
572            id: "1".into(),
573            score: None,
574            verdict: Some(true),
575            error: None,
576            elapsed_ms: None,
577        };
578        assert!(r.is_complete());
579    }
580
581    #[test]
582    fn hashing_a_file_is_stable_and_prefixed() {
583        let dir = tempfile::tempdir().unwrap();
584        let p = dir.path().join("x");
585        std::fs::write(&p, b"hello").unwrap();
586        let a = hash_file(&p).unwrap();
587        assert_eq!(a, hash_file(&p).unwrap());
588        assert!(a.starts_with("sha256:"));
589    }
590
591    #[test]
592    fn hashing_notices_a_changed_byte() {
593        let dir = tempfile::tempdir().unwrap();
594        let p = dir.path().join("x");
595        std::fs::write(&p, b"hello").unwrap();
596        let before = hash_file(&p).unwrap();
597        std::fs::write(&p, b"hellp").unwrap();
598        assert_ne!(before, hash_file(&p).unwrap());
599    }
600
601    #[cfg(unix)]
602    #[test]
603    fn which_finds_something_that_exists_and_not_something_that_does_not() {
604        assert!(which("sh").is_some(), "sh should be on PATH");
605        assert!(which("definitely-not-a-real-program-xyzzy").is_none());
606    }
607
608    #[cfg(unix)]
609    #[test]
610    fn an_absolute_path_is_used_directly() {
611        assert!(which("/bin/sh").is_some() || which("/usr/bin/sh").is_some());
612        assert!(which("/nope/nothing").is_none());
613    }
614
615    #[test]
616    fn records_round_trip_and_omit_what_is_absent() {
617        let r = Record {
618            id: "42".into(),
619            score: Some(0.5),
620            verdict: None,
621            error: None,
622            elapsed_ms: Some(10),
623        };
624        let text = serde_json::to_string(&r).unwrap();
625        assert!(
626            !text.contains("verdict"),
627            "absent fields should not be written"
628        );
629        assert_eq!(serde_json::from_str::<Record>(&text).unwrap(), r);
630    }
631}