thirdshift 0.35.1

Turn a GitHub issue into a ready-for-review pull request, unattended
//! Structured Experiment options, shared by admission and OpenAPI.
use anyhow::{Result, ensure};
use serde_json::{Value, json};
use std::collections::BTreeMap;

pub(in crate::console) fn properties() -> Value {
    let mut properties = serde_json::Map::new();
    for key in [
        "candidate",
        "baseline",
        "baseline_harness",
        "candidate_harness",
        "baseline_model",
        "candidate_model",
        "baseline_effort",
        "candidate_effort",
        "baseline_process",
        "candidate_process",
        "baseline_runner",
        "candidate_runner",
        "name",
        "prices",
        "panel_vote",
        "panel_arm_family_rule",
        "panel",
        "people",
    ] {
        properties.insert(key.into(), json!({"type":"string"}));
    }
    for key in [
        "margin",
        "pass_level",
        "fail_level",
        "prior",
        "cost_ceiling",
        "time_ceiling",
        "panel_agreement",
        "panel_spending_limit",
    ] {
        properties.insert(key.into(), json!({"type":"number"}));
    }
    for key in [
        "seed",
        "draws",
        "min_discriminating_tasks",
        "timeout",
        "void_retries",
        "trials_per_task",
        "panel_families",
        "people_sample",
        "people_overlap",
    ] {
        properties.insert(key.into(), json!({"type":"integer"}));
    }
    for key in ["release", "yes", "no_wait"] {
        properties.insert(key.into(), json!({"type":"boolean"}));
    }
    for key in ["suite", "panel_seat", "people_reader"] {
        properties.insert(
            key.into(),
            json!({"type":"array","items":{"type":"string"}}),
        );
    }
    Value::Object(properties)
}

pub(super) fn arguments(options: Option<&BTreeMap<String, Value>>) -> Result<Vec<String>> {
    let properties = properties();
    let mut args = vec!["--json".into()];
    for (key, value) in options.into_iter().flatten() {
        let schema = properties
            .get(key)
            .ok_or_else(|| anyhow::anyhow!("unknown Experiment option: {key}"))?;
        let valid = match schema["type"].as_str().unwrap() {
            "string" => value.is_string(),
            "number" => value.is_number(),
            "integer" => value.is_i64() || value.is_u64(),
            "boolean" => value.is_boolean(),
            "array" => value
                .as_array()
                .is_some_and(|items| items.iter().all(Value::is_string)),
            _ => unreachable!(),
        };
        ensure!(valid, "invalid Experiment option: {key}");
        if matches!(key.as_str(), "baseline_runner" | "candidate_runner") {
            crate::console::evals::named_runner(value.as_str().unwrap())?;
        }
        if key == "candidate" {
            let candidate = value.as_str().unwrap();
            ensure!(
                !candidate.starts_with('-'),
                "candidate must be a Git reference"
            );
            args.push(candidate.into());
        } else if value.is_boolean() {
            if value == true {
                args.push(format!("--{}", key.replace('_', "-")));
            }
        } else if let Some(items) = value.as_array() {
            let flag = format!("--{}", key.replace('_', "-"));
            for item in items {
                args.extend([flag.clone(), item.as_str().unwrap().into()]);
            }
        } else {
            args.extend([
                format!("--{}", key.replace('_', "-")),
                value
                    .as_str()
                    .map(str::to_owned)
                    .unwrap_or_else(|| value.to_string()),
            ]);
        }
    }
    crate::evals::experiment::parse(&args)?;
    Ok(args)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn custom_runner_options_follow_cli_validation() {
        for (options, cli) in [
            (
                json!({"candidate_runner":"outside", "suite":["process"]}),
                vec!["--candidate-runner", "outside", "--suite", "process"],
            ),
            (
                json!({"baseline_runner":"outside", "candidate":"HEAD", "suite":["process"]}),
                vec!["--baseline-runner", "outside", "HEAD", "--suite", "process"],
            ),
            (
                json!({"baseline_runner":"before", "candidate_runner":"after", "suite":["process"]}),
                vec![
                    "--baseline-runner",
                    "before",
                    "--candidate-runner",
                    "after",
                    "--suite",
                    "process",
                ],
            ),
        ] {
            let options = serde_json::from_value(options).unwrap();
            let args = arguments(Some(&options)).unwrap();
            assert_eq!(
                args,
                std::iter::once("--json").chain(cli).collect::<Vec<_>>()
            );
        }
        for arm in ["baseline", "candidate"] {
            for field in ["commit", "process", "harness", "model", "effort"] {
                let runner = format!("{arm}_runner");
                let key = match (arm, field) {
                    ("baseline", "commit") => "baseline".into(),
                    ("candidate", "commit") => "candidate".into(),
                    _ => format!("{arm}_{field}"),
                };
                let mut options = BTreeMap::from([
                    ("baseline_runner".into(), json!("before")),
                    ("candidate_runner".into(), json!("after")),
                    ("suite".into(), json!(["process"])),
                ]);
                options.insert(
                    key.clone(),
                    json!(if field == "harness" { "claude" } else { "HEAD" }),
                );
                let error = arguments(Some(&options)).unwrap_err().to_string();
                assert!(error.contains("conflicts"), "{runner} + {key}: {error}");
            }
        }
        for (options, expected) in [
            (json!({"candidate_runner":"outside"}), "require --suite"),
            (
                json!({"candidate_runner":"outside", "suite":["process"], "release":true}),
                "cannot use --release",
            ),
        ] {
            let options = serde_json::from_value(options).unwrap();
            assert!(
                arguments(Some(&options))
                    .unwrap_err()
                    .to_string()
                    .contains(expected)
            );
        }
    }

    #[test]
    fn custom_runner_file_paths_stay_cli_only() {
        for options in [
            json!({"candidate_runner":"/tmp/outside.toml", "suite":["process"]}),
            json!({"baseline_runner":"/tmp/outside.toml", "candidate":"HEAD", "suite":["process"]}),
        ] {
            let options = serde_json::from_value(options).unwrap();
            let error = arguments(Some(&options)).unwrap_err().to_string();
            assert!(error.contains("CLI-only"), "{error}");
        }
    }

    #[test]
    fn each_panel_seat_becomes_its_own_flag() {
        let options = BTreeMap::from([
            ("candidate".to_owned(), json!("HEAD")),
            (
                "panel_seat".to_owned(),
                json!([
                    "harness=claude,model=judge-one",
                    "harness=codex,model=judge-two"
                ]),
            ),
            ("panel_vote".to_owned(), json!("model")),
            ("panel_arm_family_rule".to_owned(), json!("off")),
            ("panel".to_owned(), json!("counted")),
            ("people".to_owned(), json!("counted")),
            ("people_reader".to_owned(), json!(["ann@example.com"])),
            ("people_sample".to_owned(), json!(3)),
        ]);
        assert_eq!(
            arguments(Some(&options)).unwrap(),
            [
                "--json",
                "HEAD",
                "--panel",
                "counted",
                "--panel-arm-family-rule",
                "off",
                "--panel-seat",
                "harness=claude,model=judge-one",
                "--panel-seat",
                "harness=codex,model=judge-two",
                "--panel-vote",
                "model",
                "--people",
                "counted",
                "--people-reader",
                "ann@example.com",
                "--people-sample",
                "3",
            ]
        );
    }
}