use std::io::{BufRead, BufReader, Write};
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant};
use stegobench_core::registry::Entry;
use crate::{Record, WorkItem};
pub const HEARTBEAT: Duration = Duration::from_secs(30);
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub struct Tally {
pub scored: u64,
pub resumed: u64,
pub errored: u64,
pub raw_problem: Option<String>,
}
impl Tally {
pub fn seen(&self) -> u64 {
self.scored + self.resumed
}
}
#[derive(Debug, thiserror::Error)]
pub enum RunError {
#[error(
"this run was resumed, and at position {position} the record file \
names {recorded:?} while the corpus offers {offered:?}. The corpus \
has changed since the run that wrote those records, so resuming \
would score one image and file the answer under another. Start a \
fresh record file, or point at the corpus the records were made \
against."
)]
Diverged {
position: u64,
recorded: String,
offered: String,
},
#[error("could not read the existing records at {path}: {source}")]
ReadRecords {
path: String,
#[source]
source: std::io::Error,
},
#[error("could not write a record to {path}: {source}")]
WriteRecord {
path: String,
#[source]
source: std::io::Error,
},
#[error("a record already written to {path} could not be read back: {reason}")]
BadRecord { path: String, reason: String },
}
pub trait Sink {
fn write(&mut self, record: &Record) -> std::io::Result<()>;
}
struct RawSink {
path: Option<PathBuf>,
file: Option<std::fs::File>,
every: bool,
}
impl RawSink {
fn open(policy: RawPolicy<'_>) -> Self {
Self {
path: policy.path.map(Path::to_path_buf),
file: None,
every: policy.every,
}
}
#[must_use]
fn keep(&mut self, id: &str, record: &Record, raw: &crate::selftest::Raw) -> Option<String> {
if raw.is_empty() || (!self.every && record.error.is_none()) {
return None;
}
if self.file.is_none() {
let path = self.path.clone()?;
match std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(&path)
{
Ok(f) => self.file = Some(f),
Err(e) => {
self.path = None;
return Some(format!(
"could not open {} to keep what the tool printed: {e}. \
The measurement is unaffected",
path.display()
));
}
}
}
let file = self.file.as_mut()?;
let line = serde_json::json!({
"id": id,
"stdout": raw.stdout,
"stderr": raw.stderr,
});
match writeln!(file, "{line}").and_then(|()| file.flush()) {
Ok(()) => None,
Err(e) => {
self.file = None;
Some(format!(
"could not keep what the tool printed: {e}. The \
measurement is unaffected and nothing further will be \
kept for this run"
))
}
}
}
}
pub struct JsonLines<W: Write> {
out: W,
}
impl<W: Write> JsonLines<W> {
pub fn new(out: W) -> Self {
Self { out }
}
}
impl<W: Write> Sink for JsonLines<W> {
fn write(&mut self, record: &Record) -> std::io::Result<()> {
let line = serde_json::to_string(record)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
self.out.write_all(line.as_bytes())?;
self.out.write_all(b"\n")?;
self.out.flush()
}
}
struct Prior {
lines: Option<std::io::Lines<BufReader<std::fs::File>>>,
path: String,
}
impl Prior {
fn open(path: &Path) -> Result<Self, RunError> {
let lines = match std::fs::File::open(path) {
Ok(f) => Some(BufReader::new(f).lines()),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => None,
Err(e) => {
return Err(RunError::ReadRecords {
path: path.display().to_string(),
source: e,
})
}
};
Ok(Self {
lines,
path: path.display().to_string(),
})
}
fn next_id(&mut self) -> Result<Option<String>, RunError> {
let Some(lines) = self.lines.as_mut() else {
return Ok(None);
};
for line in lines.by_ref() {
let line = line.map_err(|e| RunError::ReadRecords {
path: self.path.clone(),
source: e,
})?;
if line.trim().is_empty() {
continue;
}
match serde_json::from_str::<Record>(&line) {
Ok(record) if record.is_complete() => return Ok(Some(record.id)),
Ok(_) => return Ok(None),
Err(_) => return Ok(None),
}
}
Ok(None)
}
}
pub struct Run<'a> {
pub entry: &'a Entry,
pub already: &'a Path,
pub timeout: Duration,
pub jobs: usize,
pub adapter_roots: &'a [PathBuf],
pub raw: RawPolicy<'a>,
}
#[derive(Debug, Default, Clone, Copy)]
pub struct RawPolicy<'a> {
pub path: Option<&'a Path>,
pub every: bool,
}
pub fn score<I, S, P>(
run: Run<'_>,
items: I,
sink: &mut S,
mut progress: P,
) -> Result<Tally, RunError>
where
I: IntoIterator<Item = WorkItem>,
S: Sink,
P: FnMut(&Tally),
{
let Run {
entry,
already,
timeout,
jobs,
adapter_roots,
raw: raw_policy,
} = run;
let mut raw_sink = RawSink::open(raw_policy);
let mut prior = Prior::open(already)?;
let mut recorded = prior.next_id()?;
let mut tally = Tally::default();
let mut last_beat = Instant::now();
let mut it = items.into_iter().enumerate();
while recorded.is_some() {
let Some((position, item)) = it.next() else {
break;
};
let done = recorded.as_deref().unwrap_or_default();
if done != item.id {
return Err(RunError::Diverged {
position: position as u64,
recorded: done.to_string(),
offered: item.id,
});
}
tally.resumed += 1;
recorded = prior.next_id()?;
}
#[allow(clippy::items_after_statements)]
fn score_group(
entry: &Entry,
group: &[WorkItem],
per_item: Duration,
adapter_roots: &[PathBuf],
) -> Vec<(Record, crate::selftest::Raw)> {
if group.len() == 1 {
let item = &group[0];
let mut raw = crate::selftest::Raw::default();
let record = crate::selftest::read_one_observed(
entry,
&item.path,
per_item,
adapter_roots,
&mut raw,
)
.into_record(item.id.as_str());
return vec![(record, raw)];
}
let timeout = per_item.saturating_mul(group.len() as u32);
let paths: Vec<PathBuf> = group.iter().map(|i| i.path.clone()).collect();
let mut raw = crate::selftest::Raw::default();
let readings =
crate::selftest::read_many_observed(entry, &paths, timeout, adapter_roots, &mut raw);
group
.iter()
.zip(readings)
.map(|(item, reading)| {
(reading.into_record(item.id.as_str()), raw.clone())
})
.collect()
}
let mut finish = |record: Record, tally: &mut Tally| -> Result<(), RunError> {
if record.error.is_some() {
tally.errored += 1;
}
sink.write(&record).map_err(|e| RunError::WriteRecord {
path: already.display().to_string(),
source: e,
})?;
tally.scored += 1;
Ok(())
};
let batch = entry
.invoke
.as_ref()
.and_then(|i| i.batch)
.unwrap_or(1)
.max(1) as usize;
if jobs <= 1 && batch == 1 {
for (_, item) in it {
let mut raw = crate::selftest::Raw::default();
let record = crate::selftest::read_one_observed(
entry,
&item.path,
timeout,
adapter_roots,
&mut raw,
)
.into_record(item.id.as_str());
if let Some(problem) = raw_sink.keep(&item.id, &record, &raw) {
tally.raw_problem.get_or_insert(problem);
}
finish(record, &mut tally)?;
if last_beat.elapsed() >= HEARTBEAT {
progress(&tally);
last_beat = Instant::now();
}
}
return Ok(tally);
}
if jobs <= 1 {
let mut group: Vec<WorkItem> = Vec::with_capacity(batch);
loop {
group.clear();
group.extend(it.by_ref().take(batch).map(|(_, item)| item));
if group.is_empty() {
break;
}
for ((record, raw), item) in score_group(entry, &group, timeout, adapter_roots)
.into_iter()
.zip(group.iter())
{
if let Some(problem) = raw_sink.keep(&item.id, &record, &raw) {
tally.raw_problem.get_or_insert(problem);
}
finish(record, &mut tally)?;
}
if last_beat.elapsed() >= HEARTBEAT {
progress(&tally);
last_beat = Instant::now();
}
}
return Ok(tally);
}
let window = jobs.saturating_mul(batch);
let mut chunk: Vec<WorkItem> = Vec::with_capacity(window);
loop {
chunk.clear();
chunk.extend(it.by_ref().take(window).map(|(_, item)| item));
if chunk.is_empty() {
break;
}
let groups: Vec<&[WorkItem]> = chunk.chunks(batch).collect();
let mut done: Vec<Option<Vec<(Record, crate::selftest::Raw)>>> = vec![None; groups.len()];
let mut outstanding = groups.len();
std::thread::scope(|scope| {
let (tx, rx) = std::sync::mpsc::channel();
for (i, group) in groups.iter().enumerate() {
let tx = tx.clone();
let group = *group;
scope.spawn(move || {
let _ = tx.send((i, score_group(entry, group, timeout, adapter_roots)));
});
}
drop(tx);
while outstanding > 0 {
match rx.recv_timeout(HEARTBEAT) {
Ok((i, scored)) => {
done[i] = Some(scored);
outstanding -= 1;
}
Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {
progress(&tally);
last_beat = Instant::now();
}
Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => break,
}
}
});
for (slot, group) in done.into_iter().zip(groups.iter()) {
let scored = slot.unwrap_or_else(|| {
group
.iter()
.map(|item| {
(
Record {
id: item.id.clone(),
score: None,
verdict: None,
error: Some(
"the worker scoring this item panicked; the run \
continued and this item was not measured"
.into(),
),
elapsed_ms: None,
},
crate::selftest::Raw::default(),
)
})
.collect()
});
for ((record, raw), item) in scored.into_iter().zip(group.iter()) {
if let Some(problem) = raw_sink.keep(&item.id, &record, &raw) {
tally.raw_problem.get_or_insert(problem);
}
finish(record, &mut tally)?;
}
}
if last_beat.elapsed() >= HEARTBEAT {
progress(&tally);
last_beat = Instant::now();
}
}
Ok(tally)
}
#[cfg(all(test, unix))]
mod tests {
use super::*;
use std::io::Cursor;
fn entry(toml: &str) -> Entry {
toml::from_str(toml).expect("parses")
}
fn scripted(script_path: &str) -> Entry {
entry(&format!(
"name = \"x\"\nkind = \"detector\"\nlicence = \"MIT\"\n\
[binary]\ncommand = [{script_path:?}]\n\
[emits]\noutput = \"score\"\n\
[invoke]\nargv = [\"{{file}}\"]\nparser = \"number\"\n\
[selftest]\nmust_detect = \"fixtures/a.png\"\nmust_clear = \"fixtures/b.png\"\n"
))
}
fn tool(dir: &Path, body: &str) -> String {
use std::os::unix::fs::PermissionsExt;
let path = dir.join("tool.sh");
std::fs::write(&path, format!("#!/bin/sh\n{body}\n")).expect("script");
let mut perms = std::fs::metadata(&path).expect("meta").permissions();
perms.set_mode(0o755);
std::fs::set_permissions(&path, perms).expect("chmod");
path.display().to_string()
}
fn items(dir: &Path, n: usize) -> Vec<WorkItem> {
(0..n)
.map(|i| {
let path = dir.join(format!("{i:05}.png"));
let mut bytes = b"\x89PNG\r\n\x1a\n".to_vec();
bytes.extend_from_slice(format!("{i:05}").as_bytes());
std::fs::write(&path, bytes).expect("image");
WorkItem {
id: format!("{i:05}"),
path,
}
})
.collect()
}
fn lines(buf: &[u8]) -> Vec<Record> {
String::from_utf8_lossy(buf)
.lines()
.filter(|l| !l.trim().is_empty())
.map(|l| serde_json::from_str(l).expect("a record"))
.collect()
}
#[test]
fn a_parallel_run_writes_what_a_serial_run_would_have() {
let dir = tempfile::tempdir().expect("tmp");
let e = scripted(&tool(
dir.path(),
"n=$(basename \"$1\" .png); sleep 0.$((9 - 10#$n % 10)); echo 0.5",
));
let mut serial_out = Cursor::new(Vec::new());
let mut sink = JsonLines::new(&mut serial_out);
let serial = score(
Run {
entry: &e,
already: &dir.path().join("none.jsonl"),
timeout: Duration::from_secs(30),
jobs: 1,
adapter_roots: &[],
raw: Default::default(),
},
items(dir.path(), 8),
&mut sink,
|_| {},
)
.expect("the serial run");
let mut parallel_out = Cursor::new(Vec::new());
let mut sink = JsonLines::new(&mut parallel_out);
let parallel = score(
Run {
entry: &e,
already: &dir.path().join("none.jsonl"),
timeout: Duration::from_secs(30),
jobs: 4,
adapter_roots: &[],
raw: Default::default(),
},
items(dir.path(), 8),
&mut sink,
|_| {},
)
.expect("the parallel run");
assert_eq!(serial_out.get_ref(), parallel_out.get_ref());
assert_eq!(parallel.scored, serial.scored);
assert_eq!(parallel.errored, serial.errored);
}
#[test]
fn a_parallel_run_resumes_from_what_a_parallel_run_left() {
let dir = tempfile::tempdir().expect("tmp");
let e = scripted(&tool(dir.path(), "echo 0.5"));
let whole = {
let mut out = Cursor::new(Vec::new());
let mut sink = JsonLines::new(&mut out);
score(
Run {
entry: &e,
already: &dir.path().join("none.jsonl"),
timeout: Duration::from_secs(30),
jobs: 4,
adapter_roots: &[],
raw: Default::default(),
},
items(dir.path(), 9),
&mut sink,
|_| {},
)
.expect("the uninterrupted run");
out.into_inner()
};
let part = dir.path().join("part.jsonl");
let five: Vec<u8> = whole
.split_inclusive(|b| *b == b'\n')
.take(5)
.flatten()
.copied()
.collect();
std::fs::write(&part, &five).expect("the partial records");
let mut rest = Cursor::new(Vec::new());
let mut sink = JsonLines::new(&mut rest);
let tally = score(
Run {
entry: &e,
already: &part,
timeout: Duration::from_secs(30),
jobs: 4,
adapter_roots: &[],
raw: Default::default(),
},
items(dir.path(), 9),
&mut sink,
|_| {},
)
.expect("the resumed run");
assert_eq!(tally.resumed, 5, "the prefix already on disk");
assert_eq!(tally.scored, 4, "only the tail is rescored");
let mut joined = five;
joined.extend_from_slice(rest.get_ref());
assert_eq!(joined, whole, "resuming produced a different file");
}
#[test]
fn every_item_is_scored_once_and_recorded_in_order() {
let dir = tempfile::tempdir().expect("tmp");
let e = scripted(&tool(dir.path(), "echo 0.5"));
let mut out = Cursor::new(Vec::new());
let mut sink = JsonLines::new(&mut out);
let tally = score(
Run {
entry: &e,
already: &dir.path().join("none.jsonl"),
timeout: Duration::from_secs(10),
jobs: 1,
adapter_roots: &[],
raw: Default::default(),
},
items(dir.path(), 4),
&mut sink,
|_| {},
)
.expect("ran");
assert_eq!(tally.scored, 4);
assert_eq!(tally.resumed, 0);
let written = lines(out.get_ref());
assert_eq!(written.len(), 4);
let ids: Vec<&str> = written.iter().map(|r| r.id.as_str()).collect();
assert_eq!(ids, ["00000", "00001", "00002", "00003"]);
}
#[test]
fn a_resumed_run_skips_what_is_already_recorded() {
let dir = tempfile::tempdir().expect("tmp");
let e = scripted(&tool(dir.path(), "echo 0.5"));
let record_path = dir.path().join("records.jsonl");
std::fs::write(
&record_path,
"{\"id\":\"00000\",\"score\":0.5}\n{\"id\":\"00001\",\"score\":0.5}\n",
)
.expect("records");
let mut out = Cursor::new(Vec::new());
let mut sink = JsonLines::new(&mut out);
let tally = score(
Run {
entry: &e,
already: &record_path,
timeout: Duration::from_secs(10),
jobs: 1,
adapter_roots: &[],
raw: Default::default(),
},
items(dir.path(), 4),
&mut sink,
|_| {},
)
.expect("ran");
assert_eq!(tally.resumed, 2);
assert_eq!(tally.scored, 2);
assert_eq!(tally.seen(), 4);
let ids: Vec<String> = lines(out.get_ref()).into_iter().map(|r| r.id).collect();
assert_eq!(ids, ["00002", "00003"]);
}
#[test]
fn a_corpus_that_changed_under_the_records_is_refused() {
let dir = tempfile::tempdir().expect("tmp");
let e = scripted(&tool(dir.path(), "echo 0.5"));
let record_path = dir.path().join("records.jsonl");
std::fs::write(&record_path, "{\"id\":\"something-else\",\"score\":0.5}\n")
.expect("records");
let mut out = Cursor::new(Vec::new());
let mut sink = JsonLines::new(&mut out);
let err = score(
Run {
entry: &e,
already: &record_path,
timeout: Duration::from_secs(10),
jobs: 1,
adapter_roots: &[],
raw: Default::default(),
},
items(dir.path(), 2),
&mut sink,
|_| {},
)
.expect_err("it should refuse");
let message = err.to_string();
assert!(message.contains("something-else"), "got {message}");
assert!(message.contains("00000"), "got {message}");
}
#[test]
fn an_incomplete_record_is_scored_again_rather_than_counted_as_done() {
let dir = tempfile::tempdir().expect("tmp");
let e = scripted(&tool(dir.path(), "echo 0.5"));
let record_path = dir.path().join("records.jsonl");
std::fs::write(&record_path, "{\"id\":\"00000\"}\n").expect("records");
let mut out = Cursor::new(Vec::new());
let mut sink = JsonLines::new(&mut out);
let tally = score(
Run {
entry: &e,
already: &record_path,
timeout: Duration::from_secs(10),
jobs: 1,
adapter_roots: &[],
raw: Default::default(),
},
items(dir.path(), 1),
&mut sink,
|_| {},
)
.expect("ran");
assert_eq!(tally.resumed, 0);
assert_eq!(tally.scored, 1);
}
#[test]
fn a_partial_last_line_does_not_stop_a_resume() {
let dir = tempfile::tempdir().expect("tmp");
let e = scripted(&tool(dir.path(), "echo 0.5"));
let record_path = dir.path().join("records.jsonl");
std::fs::write(
&record_path,
"{\"id\":\"00000\",\"score\":0.5}\n{\"id\":\"0000",
)
.expect("records");
let mut out = Cursor::new(Vec::new());
let mut sink = JsonLines::new(&mut out);
let tally = score(
Run {
entry: &e,
already: &record_path,
timeout: Duration::from_secs(10),
jobs: 1,
adapter_roots: &[],
raw: Default::default(),
},
items(dir.path(), 2),
&mut sink,
|_| {},
)
.expect("ran");
assert_eq!(tally.resumed, 1);
assert_eq!(tally.scored, 1);
}
#[test]
fn a_tool_that_fails_on_one_item_is_counted_rather_than_stopping_the_run() {
let dir = tempfile::tempdir().expect("tmp");
let e = scripted(&tool(
dir.path(),
"case \"$(cat \"$1\")\" in *00001) exit 1 ;; *) echo 0.5 ;; esac",
));
let mut out = Cursor::new(Vec::new());
let mut sink = JsonLines::new(&mut out);
let tally = score(
Run {
entry: &e,
already: &dir.path().join("none.jsonl"),
timeout: Duration::from_secs(10),
jobs: 1,
adapter_roots: &[],
raw: Default::default(),
},
items(dir.path(), 3),
&mut sink,
|_| {},
)
.expect("ran");
assert_eq!(tally.scored, 3);
assert_eq!(tally.errored, 1);
assert_eq!(lines(out.get_ref()).len(), 3);
}
#[test]
fn a_stuck_item_is_recorded_as_an_error_rather_than_holding_the_run() {
let dir = tempfile::tempdir().expect("tmp");
let e = scripted(&tool(dir.path(), "sleep 30"));
let mut out = Cursor::new(Vec::new());
let mut sink = JsonLines::new(&mut out);
let started = Instant::now();
let tally = score(
Run {
entry: &e,
already: &dir.path().join("none.jsonl"),
timeout: Duration::from_millis(200),
jobs: 1,
adapter_roots: &[],
raw: Default::default(),
},
items(dir.path(), 1),
&mut sink,
|_| {},
)
.expect("ran");
assert!(started.elapsed() < Duration::from_secs(10), "it waited");
assert_eq!(tally.errored, 1);
let written = lines(out.get_ref());
assert!(
written[0]
.error
.as_deref()
.unwrap_or("")
.contains("no answer"),
"got {:?}",
written[0].error
);
}
#[test]
fn a_record_is_on_disk_before_the_next_item_starts() {
let dir = tempfile::tempdir().expect("tmp");
let e = scripted(&tool(dir.path(), "echo 0.5"));
let record_path = dir.path().join("records.jsonl");
let file = std::fs::File::create(&record_path).expect("create");
let mut sink = JsonLines::new(file);
score(
Run {
entry: &e,
already: &dir.path().join("none.jsonl"),
timeout: Duration::from_secs(10),
jobs: 1,
adapter_roots: &[],
raw: Default::default(),
},
items(dir.path(), 3),
&mut sink,
|_| {},
)
.expect("ran");
let on_disk = std::fs::read(&record_path).expect("read back");
assert_eq!(lines(&on_disk).len(), 3);
}
}