use std::collections::BTreeMap;
use std::ffi::OsString;
use std::io::{Read, Write};
use std::path::{Path, PathBuf};
use std::process::{Child, Command, ExitStatus, Stdio};
use std::thread;
use std::time::Duration;
use serde::Deserialize;
use serde_json::{json, Value};
use wait_timeout::ChildExt;
use crate::domain::verdict::RuleVerdict;
use crate::errors::{io_err, Error, Result};
pub const DEFAULT_BIN: &str = "oneharness";
pub const MIN_VERSION: (u64, u64, u64) = (0, 3, 21);
fn format_version((major, minor, patch): (u64, u64, u64)) -> String {
format!("{major}.{minor}.{patch}")
}
fn parse_semver(version_line: &str) -> Option<(u64, u64, u64)> {
for token in version_line.split_whitespace() {
let token = token.strip_prefix('v').unwrap_or(token);
let core: String = token
.chars()
.take_while(|c| c.is_ascii_digit() || *c == '.')
.collect();
let parts: Vec<&str> = core.split('.').filter(|p| !p.is_empty()).collect();
if parts.len() < 2 {
continue;
}
let nums: Option<Vec<u64>> = parts.iter().map(|p| p.parse().ok()).collect();
if let Some(nums) = nums {
return Some((nums[0], nums[1], nums.get(2).copied().unwrap_or(0)));
}
}
None
}
pub struct Client {
pub bin: PathBuf,
}
fn found_in_paths(paths: &std::ffi::OsStr, name: &str) -> bool {
std::env::split_paths(paths).any(|dir| {
!dir.as_os_str().is_empty()
&& (dir.join(name).is_file()
|| (cfg!(windows) && dir.join(format!("{name}.exe")).is_file()))
})
}
fn sibling_in(dir: &Path) -> Option<PathBuf> {
let name = if cfg!(windows) {
"oneharness.exe"
} else {
"oneharness"
};
let candidate = dir.join(name);
candidate.is_file().then_some(candidate)
}
fn sibling_oneharness() -> Option<PathBuf> {
let exe = std::env::current_exe().ok()?;
let exe = exe.canonicalize().unwrap_or(exe);
sibling_in(exe.parent()?)
}
#[derive(Debug, Default, Clone)]
pub struct RunTrace {
pub command: String,
pub exit_code: Option<i32>,
pub stdout: String,
pub stderr: String,
}
pub struct RunRequest<'a> {
pub harness: Option<&'a str>,
pub model: Option<&'a str>,
pub system: &'a str,
pub prompt: &'a str,
pub schema: &'a Value,
pub schema_max_retries: Option<u32>,
pub cwd: &'a Path,
pub timeout_secs: u64,
pub oneharness_config: Option<&'a Path>,
pub no_config: bool,
}
#[derive(Deserialize)]
struct Report {
#[serde(default)]
results: Vec<RunResult>,
#[serde(default)]
fallback: Option<Fallback>,
}
#[derive(Deserialize)]
struct Fallback {
#[serde(default)]
ran: Option<String>,
}
#[derive(Deserialize)]
struct RunResult {
#[serde(default)]
harness: Option<String>,
#[serde(default)]
status: Option<String>,
#[serde(default)]
structured: Option<Value>,
#[serde(default)]
schema_valid: Option<bool>,
#[serde(default)]
schema_error: Option<String>,
#[serde(default)]
failure_kind: Option<String>,
#[serde(default)]
error: Option<String>,
}
impl RunResult {
fn produced_output(&self) -> bool {
self.structured.as_ref().is_some_and(|v| !v.is_null())
}
}
impl Client {
pub fn new(bin_override: Option<&str>) -> Client {
let bin = match bin_override {
Some(b) => PathBuf::from(b),
None => {
let on_path = std::env::var_os("PATH")
.is_some_and(|paths| found_in_paths(&paths, DEFAULT_BIN));
if on_path {
PathBuf::from(DEFAULT_BIN)
} else {
sibling_oneharness().unwrap_or_else(|| PathBuf::from(DEFAULT_BIN))
}
}
};
Client { bin }
}
pub fn version(&self) -> Result<String> {
let output = match Command::new(&self.bin).arg("--version").output() {
Ok(o) => o,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
return Err(Error::OneharnessNotFound(self.bin.display().to_string()))
}
Err(e) => return Err(io_err("running oneharness --version", e)),
};
if !output.status.success() {
return Err(Error::Oneharness(format!(
"`{} --version` failed: {}",
self.bin.display(),
String::from_utf8_lossy(&output.stderr).trim()
)));
}
Ok(String::from_utf8_lossy(&output.stdout).trim().to_string())
}
pub fn check_min_version(&self) -> Result<String> {
let raw = self.version()?;
match parse_semver(&raw) {
Some(v) if v >= MIN_VERSION => Ok(raw),
Some(_) => Err(Error::OneharnessTooOld {
found: raw,
required: format_version(MIN_VERSION),
}),
None => Err(Error::Oneharness(format!(
"could not determine the oneharness version from {raw:?}; llmlint \
requires oneharness >= {} for read-only mode",
format_version(MIN_VERSION)
))),
}
}
pub fn run(&self, req: &RunRequest) -> Result<BTreeMap<String, RuleVerdict>> {
self.run_with_trace(req).1
}
pub fn run_with_trace(
&self,
req: &RunRequest,
) -> (RunTrace, Result<BTreeMap<String, RuleVerdict>>) {
let harness = req.harness.unwrap_or("oneharness default");
let mut trace = RunTrace::default();
let mut schema_file = match tempfile::Builder::new()
.prefix("llmlint-schema-")
.suffix(".json")
.tempfile()
{
Ok(f) => f,
Err(e) => return (trace, Err(io_err("creating schema temp file", e))),
};
match serde_json::to_vec(req.schema)
.map_err(|e| Error::Io(e.to_string()))
.and_then(|bytes| {
schema_file
.write_all(&bytes)
.and_then(|_| schema_file.flush())
.map_err(|e| io_err("writing schema temp file", e))
}) {
Ok(()) => {}
Err(e) => return (trace, Err(e)),
}
let mut system_file = match tempfile::Builder::new()
.prefix("llmlint-system-")
.suffix(".txt")
.tempfile()
{
Ok(f) => f,
Err(e) => return (trace, Err(io_err("creating system temp file", e))),
};
match system_file
.write_all(req.system.as_bytes())
.and_then(|_| system_file.flush())
.map_err(|e| io_err("writing system temp file", e))
{
Ok(()) => {}
Err(e) => return (trace, Err(e)),
}
let mut args: Vec<OsString> = vec![
"run".into(),
"--system-file".into(),
system_file.path().as_os_str().to_os_string(),
"--prompt".into(),
req.prompt.into(),
"--schema".into(),
schema_file.path().as_os_str().to_os_string(),
"--cwd".into(),
req.cwd.as_os_str().to_os_string(),
"--timeout".into(),
req.timeout_secs.to_string().into(),
"--mode".into(),
"read-only".into(),
"--require-available".into(),
"--compact".into(),
];
if let Some(h) = req.harness {
args.push("--harness".into());
args.push(h.into());
}
if let Some(m) = req.model {
args.push("--model".into());
args.push(m.into());
}
if let Some(n) = req.schema_max_retries {
args.push("--schema-max-retries".into());
args.push(n.to_string().into());
}
if req.no_config {
args.push("--no-config".into());
} else if let Some(c) = req.oneharness_config {
args.push("--config".into());
args.push(c.as_os_str().to_os_string());
}
trace.command = render_command(&self.bin, &args);
let mut cmd = Command::new(&self.bin);
cmd.args(&args)
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped());
let child = match cmd.spawn() {
Ok(c) => c,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
return (
trace,
Err(Error::OneharnessNotFound(self.bin.display().to_string())),
)
}
Err(e) => return (trace, Err(io_err("spawning oneharness", e))),
};
let wall = Duration::from_secs(req.timeout_secs.saturating_add(30));
let capture = match wait_capture(child, wall) {
Ok(Some(c)) => c,
Ok(None) => {
return (
trace,
Err(Error::Oneharness(format!(
"oneharness did not exit within {}s (harness {})",
wall.as_secs(),
harness
))),
)
}
Err(e) => return (trace, Err(e)),
};
trace.exit_code = capture.status.code();
trace.stdout = String::from_utf8_lossy(&capture.stdout).into_owned();
trace.stderr = String::from_utf8_lossy(&capture.stderr).into_owned();
let verdicts = parse_verdicts(&capture, harness);
(trace, verdicts)
}
pub fn probe(
&self,
harness: Option<&str>,
model: Option<&str>,
timeout_secs: u64,
) -> Result<ProbeOutcome> {
const MARKER: &str = "LLMLINT_PROBE_OK";
let mut probe_file = tempfile::Builder::new()
.prefix("llmlint-probe-")
.suffix(".txt")
.tempfile()
.map_err(|e| io_err("creating probe temp file", e))?;
probe_file
.write_all(MARKER.as_bytes())
.and_then(|_| probe_file.flush())
.map_err(|e| io_err("writing probe temp file", e))?;
let path = probe_file.path();
let cwd = path
.parent()
.unwrap_or_else(|| Path::new("."))
.to_path_buf();
let prompt = format!(
"Use your file-reading tool to read the file at {} and report whether \
its entire contents are exactly `{MARKER}`.",
path.display()
);
let schema = json!({
"type": "object",
"additionalProperties": false,
"properties": { "read_ok": { "type": "boolean" } },
"required": ["read_ok"],
});
let req = RunRequest {
harness,
model,
system: "You are a probe. Use your tools to answer; never guess.",
prompt: &prompt,
schema: &schema,
schema_max_retries: None,
cwd: &cwd,
timeout_secs,
oneharness_config: None,
no_config: false,
};
match self.run(&req) {
Ok(_) => Ok(ProbeOutcome::Executed),
Err(Error::ToolDeferred { detail, .. }) => Ok(ProbeOutcome::Deferred(detail)),
Err(e) => Err(e),
}
}
}
#[derive(Debug)]
pub enum ProbeOutcome {
Executed,
Deferred(String),
}
fn parse_verdicts(capture: &Capture, harness: &str) -> Result<BTreeMap<String, RuleVerdict>> {
let report: Report = serde_json::from_slice(&capture.stdout).map_err(|e| {
Error::Oneharness(format!(
"could not parse oneharness output ({e}); exit {:?}; stderr: {}",
capture.status.code(),
String::from_utf8_lossy(&capture.stderr).trim()
))
})?;
if report.results.is_empty() {
return Err(Error::Oneharness(format!(
"oneharness returned no results for harness {harness}"
)));
}
let winner = select_winner_index(&report);
let result = match winner {
Some(i) => report.results.into_iter().nth(i).expect("index in range"),
None => return Err(fallback_chain_error(&report, harness)),
};
if result.failure_kind.as_deref() == Some("tool_deferred") {
return Err(Error::ToolDeferred {
harness: harness.to_string(),
detail: result
.error
.filter(|e| !e.trim().is_empty())
.unwrap_or_else(|| {
"The harness deferred a builtin tool call to a controller.".into()
}),
});
}
if result.schema_valid == Some(false) {
return Err(Error::Oneharness(format!(
"harness {} produced output that failed schema validation: {}",
harness,
result
.schema_error
.unwrap_or_else(|| "unknown error".into())
)));
}
let structured = match result.structured {
Some(v) if !v.is_null() => v,
_ => {
return Err(Error::Oneharness(format!(
"harness {} returned no structured output (status {:?}): {}",
harness,
result.status.as_deref().unwrap_or("?"),
result.error.unwrap_or_else(|| "no error reported".into())
)))
}
};
serde_json::from_value(structured).map_err(|e| {
Error::Oneharness(format!("invalid verdict shape from harness {harness}: {e}"))
})
}
fn select_winner_index(report: &Report) -> Option<usize> {
let Some(fallback) = &report.fallback else {
return (!report.results.is_empty()).then_some(0);
};
if let Some(ran) = fallback.ran.as_deref() {
if let Some(i) = report
.results
.iter()
.position(|r| r.harness.as_deref() == Some(ran))
{
return Some(i);
}
}
report.results.iter().position(RunResult::produced_output)
}
fn fallback_chain_error(report: &Report, harness: &str) -> Error {
let chain: Vec<String> = report
.results
.iter()
.map(|r| {
let name = r.harness.as_deref().unwrap_or("?");
let status = r.status.as_deref().unwrap_or("?");
match r.error.as_deref() {
Some(e) if !e.is_empty() => format!("{name} ({status}: {e})"),
_ => format!("{name} ({status})"),
}
})
.collect();
Error::Oneharness(format!(
"all harnesses in the fallback chain failed for {harness}: {}",
chain.join(", ")
))
}
fn render_command(bin: &Path, args: &[OsString]) -> String {
let mut parts = vec![shell_quote(&bin.to_string_lossy())];
parts.extend(args.iter().map(|a| shell_quote(&a.to_string_lossy())));
parts.join(" ")
}
fn shell_quote(s: &str) -> String {
let safe = !s.is_empty()
&& s.bytes()
.all(|b| b.is_ascii_alphanumeric() || b"-_./:=@,+".contains(&b));
if safe {
s.to_string()
} else {
format!("'{}'", s.replace('\'', "'\\''"))
}
}
struct Capture {
status: ExitStatus,
stdout: Vec<u8>,
stderr: Vec<u8>,
}
fn wait_capture(mut child: Child, wall: Duration) -> Result<Option<Capture>> {
let mut out = child.stdout.take().expect("piped stdout");
let mut err = child.stderr.take().expect("piped stderr");
let out_h = thread::spawn(move || {
let mut b = Vec::new();
let _ = out.read_to_end(&mut b);
b
});
let err_h = thread::spawn(move || {
let mut b = Vec::new();
let _ = err.read_to_end(&mut b);
b
});
let status = match child
.wait_timeout(wall)
.map_err(|e| io_err("waiting for subprocess", e))?
{
Some(s) => s,
None => {
let _ = child.kill();
let _ = child.wait();
return Ok(None);
}
};
Ok(Some(Capture {
status,
stdout: out_h.join().unwrap_or_default(),
stderr: err_h.join().unwrap_or_default(),
}))
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
use std::path::Path;
fn req<'a>(schema: &'a Value, cwd: &'a Path) -> RunRequest<'a> {
RunRequest {
harness: Some("claude-code"),
model: None,
system: "sys",
prompt: "go",
schema,
schema_max_retries: None,
cwd,
timeout_secs: 5,
oneharness_config: None,
no_config: true,
}
}
#[test]
fn missing_binary_is_not_found_error() {
let client = Client::new(Some("definitely-not-a-real-binary-xyz"));
assert!(matches!(
client.version(),
Err(Error::OneharnessNotFound(_))
));
let schema = json!({"type": "object"});
let cwd = std::env::temp_dir();
assert!(matches!(
client.run(&req(&schema, &cwd)),
Err(Error::OneharnessNotFound(_))
));
}
#[test]
fn trace_records_the_command_even_when_the_run_fails() {
let client = Client::new(Some("definitely-not-a-real-binary-xyz"));
let schema = json!({"type": "object"});
let cwd = std::env::temp_dir();
let (trace, result) = client.run_with_trace(&req(&schema, &cwd));
assert!(trace.command.contains("definitely-not-a-real-binary-xyz"));
assert!(trace.command.contains("run --system-file"));
assert!(trace.command.contains("--harness claude-code"));
assert!(trace.exit_code.is_none());
assert!(trace.stdout.is_empty());
assert!(matches!(result, Err(Error::OneharnessNotFound(_))));
}
#[test]
fn parse_semver_reads_major_minor_patch() {
assert_eq!(parse_semver("oneharness 0.3.0"), Some((0, 3, 0)));
assert_eq!(parse_semver("oneharness 0.3.1 (abc)"), Some((0, 3, 1)));
assert_eq!(parse_semver("oneharness 0.2.529 (mock)"), Some((0, 2, 529)));
assert_eq!(parse_semver("oneharness 1.2.3"), Some((1, 2, 3)));
assert_eq!(parse_semver("oneharness 0.4"), Some((0, 4, 0)));
assert_eq!(parse_semver("v0.5.0"), Some((0, 5, 0)));
assert_eq!(parse_semver("oneharness 0.3.0-rc1"), Some((0, 3, 0)));
}
#[test]
fn parse_semver_rejects_non_versions() {
assert_eq!(parse_semver("oneharness"), None);
assert_eq!(parse_semver(""), None);
assert_eq!(parse_semver("oneharness 7"), None);
}
#[test]
fn min_version_comparison_uses_tuple_order() {
assert!((0, 3, 21) >= MIN_VERSION);
assert!((0, 4, 0) >= MIN_VERSION);
assert!((1, 0, 0) >= MIN_VERSION);
assert!((0, 3, 20) < MIN_VERSION);
assert!((0, 3, 0) < MIN_VERSION);
}
#[test]
fn check_min_version_errors_when_binary_missing() {
let client = Client::new(Some("definitely-not-a-real-binary-xyz"));
assert!(matches!(
client.check_min_version(),
Err(Error::OneharnessNotFound(_))
));
}
fn verdicts_from(body: &Value) -> Result<BTreeMap<String, RuleVerdict>> {
let capture = Capture {
status: fake_status(0),
stdout: serde_json::to_vec(body).unwrap(),
stderr: Vec::new(),
};
parse_verdicts(&capture, "oneharness default")
}
fn fake_status(_code: i32) -> ExitStatus {
#[cfg(unix)]
{
Command::new("true").status().unwrap()
}
#[cfg(windows)]
{
Command::new("cmd").args(["/C", "exit 0"]).status().unwrap()
}
}
fn ok_result(harness: &str) -> Value {
json!({
"harness": harness,
"status": "ok",
"exit_code": 0,
"structured": { "some_rule": { "holds": true } },
"schema_valid": true,
})
}
fn skipped_result(harness: &str) -> Value {
json!({
"harness": harness,
"status": "skipped",
"available": false,
"exit_code": null,
"structured": null,
"error": format!("`{harness}` not found on PATH; harness skipped."),
})
}
#[test]
fn fallback_run_reads_the_ran_winner_not_results_zero() {
let body = json!({
"schema_version": "0.1",
"oneharness_version": "mock",
"fallback": { "ran": "claude-code",
"fell_through": [{ "harness": "codex", "reason": "not-installed" }] },
"results": [skipped_result("codex"), ok_result("claude-code")],
});
let verdicts = verdicts_from(&body).expect("winner's verdict is used");
assert!(verdicts.contains_key("some_rule"));
}
#[test]
fn fallback_run_all_failed_reports_the_whole_chain() {
let body = json!({
"schema_version": "0.1",
"oneharness_version": "mock",
"fallback": { "ran": null,
"fell_through": [{ "harness": "codex", "reason": "not-installed" }] },
"results": [skipped_result("codex"), skipped_result("claude-code")],
});
let err = verdicts_from(&body).unwrap_err();
let msg = err.to_string();
assert!(msg.contains("fallback chain"), "chain-aware message: {msg}");
assert!(
msg.contains("codex") && msg.contains("claude-code"),
"{msg}"
);
}
#[test]
fn non_fallback_run_still_uses_results_zero() {
let body = json!({
"schema_version": "0.1",
"oneharness_version": "mock",
"results": [ok_result("claude-code")],
});
assert!(verdicts_from(&body).unwrap().contains_key("some_rule"));
}
#[test]
fn deferred_tool_is_a_specific_error_not_a_schema_error() {
let body = json!({
"schema_version": "0.1",
"oneharness_version": "mock",
"results": [{
"harness": "claude-code",
"status": "ok",
"exit_code": 0,
"structured": null,
"schema_valid": null,
"failure_kind": "tool_deferred",
"error": "harness claude-code deferred a tool call (`Read`).",
}],
});
let err = verdicts_from(&body).unwrap_err();
assert!(
matches!(err, Error::ToolDeferred { .. }),
"expected ToolDeferred, got {err:?}"
);
let msg = err.to_string();
assert!(msg.contains("deferred a tool call"), "{msg}");
assert!(msg.contains("`Read`"), "{msg}");
}
#[test]
fn deferred_tool_without_detail_still_diagnoses() {
let body = json!({
"schema_version": "0.1",
"oneharness_version": "mock",
"results": [{
"harness": "claude-code",
"status": "ok",
"structured": null,
"failure_kind": "tool_deferred",
}],
});
let err = verdicts_from(&body).unwrap_err();
assert!(matches!(err, Error::ToolDeferred { .. }), "{err:?}");
}
#[test]
fn fallback_without_ran_name_picks_first_harness_that_answered() {
let body = json!({
"schema_version": "0.1",
"oneharness_version": "mock",
"fallback": { "fell_through": [{ "harness": "codex", "reason": "not-installed" }] },
"results": [skipped_result("codex"), ok_result("claude-code")],
});
assert!(verdicts_from(&body).unwrap().contains_key("some_rule"));
}
#[test]
fn shell_quote_is_bare_when_safe_and_quoted_otherwise() {
assert_eq!(shell_quote("run"), "run");
assert_eq!(shell_quote("--harness"), "--harness");
assert_eq!(shell_quote("/tmp/a.json"), "/tmp/a.json");
assert_eq!(shell_quote(""), "''");
assert_eq!(shell_quote("a b"), "'a b'");
assert_eq!(shell_quote("it's"), "'it'\\''s'");
}
#[test]
fn render_command_joins_program_and_args() {
let args: Vec<OsString> = vec!["run".into(), "--system".into(), "hi there".into()];
let rendered = render_command(Path::new("oneharness"), &args);
assert_eq!(rendered, "oneharness run --system 'hi there'");
}
#[cfg(unix)]
#[test]
fn wait_capture_collects_output() {
let child = Command::new("sh")
.arg("-c")
.arg("printf hello; printf oops 1>&2")
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.unwrap();
let cap = wait_capture(child, Duration::from_secs(5))
.unwrap()
.unwrap();
assert!(cap.status.success());
assert_eq!(cap.stdout, b"hello");
assert_eq!(cap.stderr, b"oops");
}
#[cfg(unix)]
#[test]
fn wait_capture_times_out_and_kills() {
let child = Command::new("sh")
.arg("-c")
.arg("sleep 30")
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.unwrap();
let result = wait_capture(child, Duration::from_millis(200)).unwrap();
assert!(result.is_none());
}
#[test]
fn found_in_paths_sees_the_binary_and_skips_empty_entries() {
let dir = tempfile::tempdir().unwrap();
let name = if cfg!(windows) {
"oneharness.exe"
} else {
"oneharness"
};
std::fs::write(dir.path().join(name), b"").unwrap();
let paths =
std::env::join_paths([Path::new(""), dir.path(), Path::new("/nonexistent-xyz")])
.unwrap();
assert!(found_in_paths(&paths, DEFAULT_BIN));
let empty = tempfile::tempdir().unwrap();
let paths = std::env::join_paths([empty.path()]).unwrap();
assert!(!found_in_paths(&paths, DEFAULT_BIN));
}
#[test]
fn sibling_in_finds_only_a_real_file() {
let dir = tempfile::tempdir().unwrap();
assert!(sibling_in(dir.path()).is_none());
let name = if cfg!(windows) {
"oneharness.exe"
} else {
"oneharness"
};
std::fs::write(dir.path().join(name), b"").unwrap();
assert_eq!(sibling_in(dir.path()), Some(dir.path().join(name)));
let dir2 = tempfile::tempdir().unwrap();
std::fs::create_dir(dir2.path().join(name)).unwrap();
assert!(sibling_in(dir2.path()).is_none());
}
}