use std::path::PathBuf;
use clap::{ArgAction, Parser, Subcommand, ValueEnum};
use vernier_core::ParityMode;
use crate::error::CliError;
use crate::format::FormatName;
#[derive(Debug, Parser)]
#[command(
name = "vernier",
version,
about = "COCO-style evaluation CLI (vernier)",
propagate_version = true
)]
pub(crate) struct Cli {
#[command(subcommand)]
pub(crate) command: Command,
}
#[derive(Debug, Subcommand)]
pub(crate) enum Command {
Eval(EvalArgs),
Aggregate(AggregateArgs),
}
#[derive(Debug, Parser)]
pub(crate) struct AggregateArgs {
#[arg(long, value_name = "PATH")]
pub(crate) manifest: PathBuf,
#[arg(long, value_name = "GLOB")]
pub(crate) results: String,
#[arg(long, value_name = "VALUE")]
pub(crate) baseline: Option<String>,
#[arg(long, value_name = "NAME", action = ArgAction::Append)]
pub(crate) metric: Vec<String>,
#[arg(long = "emit", value_name = "FMT[=PATH]", num_args = 1, action = ArgAction::Append)]
pub(crate) emit: Vec<String>,
#[arg(long, action = ArgAction::SetTrue)]
pub(crate) quiet: bool,
}
impl AggregateArgs {
pub(crate) fn validate(&self) -> Result<Vec<EmitSpec>, CliError> {
resolve_emit_list(&self.emit)
}
}
pub(crate) fn resolve_emit_list(raw: &[String]) -> Result<Vec<EmitSpec>, CliError> {
let raw_emits: Vec<String> = if raw.is_empty() {
vec!["text".to_string()]
} else {
raw.to_vec()
};
let mut parsed: Vec<EmitSpec> = Vec::with_capacity(raw_emits.len());
let mut stdout_seen = false;
for entry in &raw_emits {
let spec = parse_emit(entry)?;
match &spec.destination {
EmitDestination::Stdout => {
if stdout_seen {
return Err(CliError::Validation(
"more than one --emit targets stdout; outputs would interleave".into(),
));
}
stdout_seen = true;
}
EmitDestination::File(path) => {
let collides = parsed.iter().any(|e| match &e.destination {
EmitDestination::File(p) => p == path,
EmitDestination::Stdout => false,
});
if collides {
return Err(CliError::Validation(format!(
"--emit path {} appears more than once",
path.display()
)));
}
}
}
parsed.push(spec);
}
Ok(parsed)
}
#[derive(Debug, Parser)]
pub(crate) struct EvalArgs {
#[arg(long, value_name = "PATH")]
pub(crate) gt: PathBuf,
#[arg(long, value_name = "PATH")]
pub(crate) dt: PathBuf,
#[arg(long = "iou-type", value_enum)]
pub(crate) iou_type: IouTypeArg,
#[arg(long = "parity-mode", value_enum, default_value_t = ParityModeArg::Strict)]
pub(crate) parity_mode: ParityModeArg,
#[arg(long = "max-dets", value_name = "a,b,c")]
pub(crate) max_dets: Option<String>,
#[arg(
long = "use-cats",
default_value_t = true,
action = ArgAction::Set,
value_name = "BOOL",
num_args = 0..=1,
default_missing_value = "true",
overrides_with = "no_use_cats"
)]
pub(crate) use_cats: bool,
#[arg(long = "no-use-cats", action = ArgAction::SetTrue, hide = true)]
pub(crate) no_use_cats: bool,
#[arg(long = "dilation-ratio", value_name = "FLOAT")]
pub(crate) dilation_ratio: Option<f64>,
#[arg(long, value_name = "FILE")]
pub(crate) sigmas: Option<PathBuf>,
#[arg(long = "emit", value_name = "FMT[=PATH]", num_args = 1, action = ArgAction::Append)]
pub(crate) emit: Vec<String>,
#[arg(long, action = ArgAction::SetTrue)]
pub(crate) quiet: bool,
#[arg(long = "metric", value_enum, default_value_t = MetricArg::Ap)]
pub(crate) metric: MetricArg,
#[arg(long, value_name = "PATH")]
pub(crate) manifest: Option<PathBuf>,
#[arg(long, value_name = "AXES", action = ArgAction::Append)]
pub(crate) cross: Vec<String>,
#[arg(long, value_name = "NAME")]
pub(crate) label: Option<String>,
#[arg(long = "threads", value_name = "N", default_value_t = 1)]
pub(crate) threads: usize,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
#[value(rename_all = "lower")]
pub(crate) enum MetricArg {
Ap,
Olrp,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
#[value(rename_all = "lower")]
pub(crate) enum IouTypeArg {
Bbox,
Segm,
Boundary,
Keypoints,
}
impl IouTypeArg {
pub(crate) fn as_str(self) -> &'static str {
match self {
Self::Bbox => "bbox",
Self::Segm => "segm",
Self::Boundary => "boundary",
Self::Keypoints => "keypoints",
}
}
pub(crate) fn kernel_kind(self) -> vernier_core::evaluate::KernelKind {
use vernier_core::evaluate::KernelKind;
match self {
Self::Bbox => KernelKind::Bbox,
Self::Segm => KernelKind::Segm,
Self::Boundary => KernelKind::Boundary,
Self::Keypoints => KernelKind::Keypoints,
}
}
pub(crate) fn default_area_ranges(self) -> Vec<vernier_core::evaluate::AreaRange> {
match self {
Self::Keypoints => vernier_core::evaluate::AreaRange::keypoints_default().to_vec(),
_ => vernier_core::evaluate::AreaRange::coco_default().to_vec(),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
#[value(rename_all = "lower")]
pub(crate) enum ParityModeArg {
Strict,
Aligned,
Corrected,
}
impl From<ParityModeArg> for ParityMode {
fn from(value: ParityModeArg) -> Self {
match value {
ParityModeArg::Strict | ParityModeArg::Aligned => Self::Strict,
ParityModeArg::Corrected => Self::Corrected,
}
}
}
#[derive(Debug, Clone)]
pub(crate) struct EmitSpec {
pub(crate) format: FormatName,
pub(crate) destination: EmitDestination,
}
#[derive(Debug, Clone)]
pub(crate) enum EmitDestination {
Stdout,
File(PathBuf),
}
impl EvalArgs {
pub(crate) fn parsed_max_dets(&self) -> Result<Option<Vec<usize>>, CliError> {
match &self.max_dets {
None => Ok(None),
Some(raw) => parse_max_dets(raw).map(Some).map_err(CliError::Validation),
}
}
pub(crate) fn parsed_cross_axes(&self) -> Result<Vec<Vec<String>>, CliError> {
let mut out: Vec<Vec<String>> = Vec::with_capacity(self.cross.len());
for (idx, raw) in self.cross.iter().enumerate() {
let parts: Vec<String> = raw.split(',').map(|s| s.trim().to_string()).collect();
if parts.iter().any(String::is_empty) {
return Err(CliError::Validation(format!(
"--cross tuple #{idx} ({raw:?}) contains an empty axis name"
)));
}
if parts.len() < 2 {
return Err(CliError::Validation(format!(
"--cross tuple #{idx} ({raw:?}) must list at least two axes \
(a single-axis cross is a marginal — already emitted by default)"
)));
}
out.push(parts);
}
Ok(out)
}
pub(crate) fn effective_use_cats(&self) -> bool {
if self.no_use_cats {
false
} else {
self.use_cats
}
}
pub(crate) fn validate(&self) -> Result<Vec<EmitSpec>, CliError> {
if self.dilation_ratio.is_some() && self.iou_type != IouTypeArg::Boundary {
return Err(CliError::Validation(format!(
"--dilation-ratio is only valid with --iou-type boundary; got --iou-type {}",
self.iou_type.as_str(),
)));
}
if self.sigmas.is_some() && self.iou_type != IouTypeArg::Keypoints {
return Err(CliError::Validation(format!(
"--sigmas is only valid with --iou-type keypoints; got --iou-type {}",
self.iou_type.as_str(),
)));
}
if let Some(d) = self.dilation_ratio {
if !d.is_finite() || d <= 0.0 {
return Err(CliError::Validation(format!(
"--dilation-ratio must be a positive finite float; got {d}"
)));
}
}
if let Some(d) = self.parsed_max_dets()? {
if d.is_empty() {
return Err(CliError::Validation(
"--max-dets must contain at least one entry".into(),
));
}
}
if !self.cross.is_empty() && self.manifest.is_none() {
return Err(CliError::Validation(
"--cross requires --manifest; cross-product cells are a partition-only concept"
.into(),
));
}
let _ = self.parsed_cross_axes()?;
resolve_emit_list(&self.emit)
}
}
fn parse_emit(raw: &str) -> Result<EmitSpec, CliError> {
let (name, dest) = match raw.split_once('=') {
Some((name, path)) => {
if path.is_empty() {
return Err(CliError::Validation(format!(
"--emit value {raw:?} has an empty path after '='"
)));
}
(name, EmitDestination::File(PathBuf::from(path)))
}
None => (raw, EmitDestination::Stdout),
};
let format = FormatName::lookup(name).ok_or_else(|| {
CliError::Validation(format!(
"unknown --emit format {name:?}; known formats: {}",
FormatName::known_names_joined()
))
})?;
Ok(EmitSpec {
format,
destination: dest,
})
}
fn parse_max_dets(raw: &str) -> Result<Vec<usize>, String> {
if raw.trim().is_empty() {
return Err("--max-dets must not be empty".into());
}
let mut out = Vec::new();
for part in raw.split(',') {
let trimmed = part.trim();
if trimmed.is_empty() {
return Err(format!("--max-dets contains an empty entry; got {raw:?}"));
}
let v: usize = trimmed.parse().map_err(|e| {
format!("--max-dets entry {trimmed:?} is not a non-negative integer: {e}")
})?;
out.push(v);
}
Ok(out)
}
#[cfg(test)]
mod tests {
use super::*;
use clap::Parser;
fn parse(args: &[&str]) -> Cli {
let mut full: Vec<&str> = vec!["vernier"];
full.extend_from_slice(args);
Cli::parse_from(full)
}
#[test]
fn happy_path_parses() {
let cli = parse(&[
"eval",
"--gt",
"gt.json",
"--dt",
"dt.json",
"--iou-type",
"bbox",
]);
let Command::Eval(args) = cli.command else {
panic!("expected Eval")
};
assert_eq!(args.iou_type, IouTypeArg::Bbox);
assert_eq!(args.parity_mode, ParityModeArg::Strict);
assert!(args.max_dets.is_none());
assert!(args.effective_use_cats());
}
#[test]
fn dilation_ratio_rejected_for_non_boundary() {
let cli = parse(&[
"eval",
"--gt",
"gt.json",
"--dt",
"dt.json",
"--iou-type",
"bbox",
"--dilation-ratio",
"0.02",
]);
let Command::Eval(args) = cli.command else {
panic!("expected Eval")
};
let err = args.validate().unwrap_err();
assert!(matches!(err, CliError::Validation(_)));
}
#[test]
fn sigmas_rejected_for_non_keypoints() {
let cli = parse(&[
"eval",
"--gt",
"gt.json",
"--dt",
"dt.json",
"--iou-type",
"segm",
"--sigmas",
"sigmas.json",
]);
let Command::Eval(args) = cli.command else {
panic!("expected Eval")
};
let err = args.validate().unwrap_err();
assert!(matches!(err, CliError::Validation(_)));
}
#[test]
fn double_stdout_emit_rejected() {
let cli = parse(&[
"eval",
"--gt",
"gt.json",
"--dt",
"dt.json",
"--iou-type",
"bbox",
"--emit",
"text",
"--emit",
"json",
]);
let Command::Eval(args) = cli.command else {
panic!("expected Eval")
};
let err = args.validate().unwrap_err();
assert!(matches!(err, CliError::Validation(_)));
}
#[test]
fn duplicate_path_emit_rejected() {
let cli = parse(&[
"eval",
"--gt",
"gt.json",
"--dt",
"dt.json",
"--iou-type",
"bbox",
"--emit",
"text=out.txt",
"--emit",
"json=out.txt",
]);
let Command::Eval(args) = cli.command else {
panic!("expected Eval")
};
let err = args.validate().unwrap_err();
assert!(matches!(err, CliError::Validation(_)));
}
#[test]
fn no_use_cats_overrides_use_cats() {
let cli = parse(&[
"eval",
"--gt",
"gt.json",
"--dt",
"dt.json",
"--iou-type",
"bbox",
"--no-use-cats",
]);
let Command::Eval(args) = cli.command else {
panic!("expected Eval")
};
assert!(!args.effective_use_cats());
}
#[test]
fn max_dets_parses_comma_list() {
let cli = parse(&[
"eval",
"--gt",
"gt.json",
"--dt",
"dt.json",
"--iou-type",
"bbox",
"--max-dets",
"1,10,100",
]);
let Command::Eval(args) = cli.command else {
panic!("expected Eval")
};
assert_eq!(args.max_dets.as_deref(), Some("1,10,100"));
assert_eq!(args.parsed_max_dets().unwrap(), Some(vec![1usize, 10, 100]));
}
#[test]
fn validate_default_emits_text_to_stdout() {
let cli = parse(&[
"eval",
"--gt",
"gt.json",
"--dt",
"dt.json",
"--iou-type",
"bbox",
]);
let Command::Eval(args) = cli.command else {
panic!("expected Eval")
};
let emits = args.validate().unwrap();
assert_eq!(emits.len(), 1);
assert!(matches!(emits[0].destination, EmitDestination::Stdout));
assert_eq!(emits[0].format, FormatName::Text);
}
#[test]
fn unknown_emit_format_rejected() {
let cli = parse(&[
"eval",
"--gt",
"gt.json",
"--dt",
"dt.json",
"--iou-type",
"bbox",
"--emit",
"yaml",
]);
let Command::Eval(args) = cli.command else {
panic!("expected Eval")
};
let err = args.validate().unwrap_err();
assert!(matches!(err, CliError::Validation(_)));
}
}