use anyhow::{Result, ensure};
use serde_json::{Value, json};
use std::collections::BTreeMap;
pub(in crate::console) fn properties() -> Value {
let mut properties = serde_json::Map::new();
for key in [
"candidate",
"baseline",
"baseline_harness",
"candidate_harness",
"baseline_model",
"candidate_model",
"baseline_effort",
"candidate_effort",
"baseline_process",
"candidate_process",
"baseline_runner",
"candidate_runner",
"name",
"prices",
"panel_vote",
"panel_arm_family_rule",
"panel",
"people",
] {
properties.insert(key.into(), json!({"type":"string"}));
}
for key in [
"margin",
"pass_level",
"fail_level",
"prior",
"cost_ceiling",
"time_ceiling",
"panel_agreement",
"panel_spending_limit",
] {
properties.insert(key.into(), json!({"type":"number"}));
}
for key in [
"seed",
"draws",
"min_discriminating_tasks",
"timeout",
"void_retries",
"trials_per_task",
"panel_families",
"people_sample",
"people_overlap",
] {
properties.insert(key.into(), json!({"type":"integer"}));
}
for key in ["release", "yes", "no_wait"] {
properties.insert(key.into(), json!({"type":"boolean"}));
}
for key in ["suite", "panel_seat", "people_reader"] {
properties.insert(
key.into(),
json!({"type":"array","items":{"type":"string"}}),
);
}
Value::Object(properties)
}
pub(super) fn arguments(options: Option<&BTreeMap<String, Value>>) -> Result<Vec<String>> {
let properties = properties();
let mut args = vec!["--json".into()];
for (key, value) in options.into_iter().flatten() {
let schema = properties
.get(key)
.ok_or_else(|| anyhow::anyhow!("unknown Experiment option: {key}"))?;
let valid = match schema["type"].as_str().unwrap() {
"string" => value.is_string(),
"number" => value.is_number(),
"integer" => value.is_i64() || value.is_u64(),
"boolean" => value.is_boolean(),
"array" => value
.as_array()
.is_some_and(|items| items.iter().all(Value::is_string)),
_ => unreachable!(),
};
ensure!(valid, "invalid Experiment option: {key}");
if matches!(key.as_str(), "baseline_runner" | "candidate_runner") {
crate::console::evals::named_runner(value.as_str().unwrap())?;
}
if key == "candidate" {
let candidate = value.as_str().unwrap();
ensure!(
!candidate.starts_with('-'),
"candidate must be a Git reference"
);
args.push(candidate.into());
} else if value.is_boolean() {
if value == true {
args.push(format!("--{}", key.replace('_', "-")));
}
} else if let Some(items) = value.as_array() {
let flag = format!("--{}", key.replace('_', "-"));
for item in items {
args.extend([flag.clone(), item.as_str().unwrap().into()]);
}
} else {
args.extend([
format!("--{}", key.replace('_', "-")),
value
.as_str()
.map(str::to_owned)
.unwrap_or_else(|| value.to_string()),
]);
}
}
crate::evals::experiment::parse(&args)?;
Ok(args)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn custom_runner_options_follow_cli_validation() {
for (options, cli) in [
(
json!({"candidate_runner":"outside", "suite":["process"]}),
vec!["--candidate-runner", "outside", "--suite", "process"],
),
(
json!({"baseline_runner":"outside", "candidate":"HEAD", "suite":["process"]}),
vec!["--baseline-runner", "outside", "HEAD", "--suite", "process"],
),
(
json!({"baseline_runner":"before", "candidate_runner":"after", "suite":["process"]}),
vec![
"--baseline-runner",
"before",
"--candidate-runner",
"after",
"--suite",
"process",
],
),
] {
let options = serde_json::from_value(options).unwrap();
let args = arguments(Some(&options)).unwrap();
assert_eq!(
args,
std::iter::once("--json").chain(cli).collect::<Vec<_>>()
);
}
for arm in ["baseline", "candidate"] {
for field in ["commit", "process", "harness", "model", "effort"] {
let runner = format!("{arm}_runner");
let key = match (arm, field) {
("baseline", "commit") => "baseline".into(),
("candidate", "commit") => "candidate".into(),
_ => format!("{arm}_{field}"),
};
let mut options = BTreeMap::from([
("baseline_runner".into(), json!("before")),
("candidate_runner".into(), json!("after")),
("suite".into(), json!(["process"])),
]);
options.insert(
key.clone(),
json!(if field == "harness" { "claude" } else { "HEAD" }),
);
let error = arguments(Some(&options)).unwrap_err().to_string();
assert!(error.contains("conflicts"), "{runner} + {key}: {error}");
}
}
for (options, expected) in [
(json!({"candidate_runner":"outside"}), "require --suite"),
(
json!({"candidate_runner":"outside", "suite":["process"], "release":true}),
"cannot use --release",
),
] {
let options = serde_json::from_value(options).unwrap();
assert!(
arguments(Some(&options))
.unwrap_err()
.to_string()
.contains(expected)
);
}
}
#[test]
fn custom_runner_file_paths_stay_cli_only() {
for options in [
json!({"candidate_runner":"/tmp/outside.toml", "suite":["process"]}),
json!({"baseline_runner":"/tmp/outside.toml", "candidate":"HEAD", "suite":["process"]}),
] {
let options = serde_json::from_value(options).unwrap();
let error = arguments(Some(&options)).unwrap_err().to_string();
assert!(error.contains("CLI-only"), "{error}");
}
}
#[test]
fn each_panel_seat_becomes_its_own_flag() {
let options = BTreeMap::from([
("candidate".to_owned(), json!("HEAD")),
(
"panel_seat".to_owned(),
json!([
"harness=claude,model=judge-one",
"harness=codex,model=judge-two"
]),
),
("panel_vote".to_owned(), json!("model")),
("panel_arm_family_rule".to_owned(), json!("off")),
("panel".to_owned(), json!("counted")),
("people".to_owned(), json!("counted")),
("people_reader".to_owned(), json!(["ann@example.com"])),
("people_sample".to_owned(), json!(3)),
]);
assert_eq!(
arguments(Some(&options)).unwrap(),
[
"--json",
"HEAD",
"--panel",
"counted",
"--panel-arm-family-rule",
"off",
"--panel-seat",
"harness=claude,model=judge-one",
"--panel-seat",
"harness=codex,model=judge-two",
"--panel-vote",
"model",
"--people",
"counted",
"--people-reader",
"ann@example.com",
"--people-sample",
"3",
]
);
}
}