#[cfg(feature = "eval")]
use std::path::PathBuf;
use std::process::Command;
#[cfg(feature = "eval")]
fn scratch_dir(tag: &str) -> PathBuf {
let dir = std::env::temp_dir().join(format!(
"oxibonsai_eval_cli_test_{tag}_{}",
std::process::id()
));
std::fs::create_dir_all(&dir).expect("create scratch dir");
dir
}
#[cfg(feature = "eval")]
#[test]
fn eval_help_advertises_the_real_flags() {
let output = Command::new(env!("CARGO_BIN_EXE_oxibonsai"))
.args(["eval", "--help"])
.output()
.expect("failed to spawn oxibonsai binary");
assert!(
output.status.success(),
"`eval --help` should succeed; stderr={}",
String::from_utf8_lossy(&output.stderr)
);
let stdout = String::from_utf8_lossy(&output.stdout);
for flag in ["--dataset", "--model", "--max-tokens", "--report-json"] {
assert!(
stdout.contains(flag),
"`eval --help` must advertise {flag}; got:\n{stdout}"
);
}
}
#[cfg(feature = "eval")]
#[test]
fn eval_rejects_missing_dataset_before_touching_the_model() {
let dataset = scratch_dir("missing_dataset").join("nope.jsonl");
let output = Command::new(env!("CARGO_BIN_EXE_oxibonsai"))
.args([
"eval",
"--model",
"/dev/null",
"--dataset",
dataset.to_str().unwrap(),
])
.output()
.expect("failed to spawn oxibonsai binary");
assert!(
!output.status.success(),
"eval must fail when --dataset does not exist"
);
let stderr = String::from_utf8_lossy(&output.stderr);
assert!(
stderr.contains(dataset.to_str().unwrap()),
"error must mention the missing dataset path; got:\n{stderr}"
);
}
#[cfg(feature = "eval")]
#[test]
fn eval_rejects_empty_dataset() {
let dir = scratch_dir("empty_dataset");
let dataset = dir.join("empty.jsonl");
std::fs::write(&dataset, "").expect("write empty dataset");
let output = Command::new(env!("CARGO_BIN_EXE_oxibonsai"))
.args([
"eval",
"--model",
"/dev/null",
"--dataset",
dataset.to_str().unwrap(),
])
.output()
.expect("failed to spawn oxibonsai binary");
assert!(
!output.status.success(),
"eval must reject a dataset with zero examples"
);
let stderr = String::from_utf8_lossy(&output.stderr);
assert!(
stderr.contains("no examples") || stderr.contains("empty"),
"error must explain the dataset is empty; got:\n{stderr}"
);
}
#[cfg(feature = "eval")]
#[test]
fn eval_rejects_malformed_dataset_line() {
let dir = scratch_dir("malformed_dataset");
let dataset = dir.join("bad.jsonl");
std::fs::write(&dataset, "not json at all\n").expect("write malformed dataset");
let output = Command::new(env!("CARGO_BIN_EXE_oxibonsai"))
.args([
"eval",
"--model",
"/dev/null",
"--dataset",
dataset.to_str().unwrap(),
])
.output()
.expect("failed to spawn oxibonsai binary");
assert!(
!output.status.success(),
"eval must reject a dataset line that is not valid JSON"
);
}
#[cfg(not(feature = "eval"))]
#[test]
fn eval_subcommand_absent_without_eval_feature() {
let output = Command::new(env!("CARGO_BIN_EXE_oxibonsai"))
.args(["eval", "--help"])
.output()
.expect("failed to spawn oxibonsai binary");
assert!(
!output.status.success(),
"`eval` must not be a recognized subcommand without the `eval` feature"
);
}