use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use serde::Deserialize;
use mira::protocol::RunResult;
use mira::report::{self, Format};
use mira::run::RunMeta;
pub const DEFAULT_RESULTS_DIR: &str = "./results";
#[derive(Debug, Default, Deserialize)]
pub struct Config {
#[serde(default)]
pub results: ResultsConfig,
#[serde(default)]
pub environment: EnvironmentConfig,
#[serde(default)]
pub presets: BTreeMap<String, Preset>,
#[serde(default)]
pub launchers: BTreeMap<String, LauncherConfig>,
#[serde(default)]
pub default_launcher: Option<String>,
#[serde(default)]
pub targets: BTreeMap<String, TargetConfig>,
#[serde(default, skip)]
base: Option<PathBuf>,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct Preset {
#[serde(default, deserialize_with = "string_or_seq")]
pub samples: Vec<String>,
#[serde(default)]
pub tag: Option<String>,
#[serde(default, deserialize_with = "string_or_seq")]
pub targets: Vec<String>,
#[serde(default, deserialize_with = "string_or_seq")]
pub evals: Vec<String>,
#[serde(default)]
pub axes: BTreeMap<String, Vec<String>>,
#[serde(default)]
pub timeout: Option<u64>,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct TargetConfig {
#[serde(default)]
pub timeout: Option<u64>,
}
fn string_or_seq<'de, D>(de: D) -> Result<Vec<String>, D::Error>
where
D: serde::Deserializer<'de>,
{
#[derive(Deserialize)]
#[serde(untagged)]
enum OneOrMany {
One(String),
Many(Vec<String>),
}
Ok(match OneOrMany::deserialize(de)? {
OneOrMany::One(s) => vec![s],
OneOrMany::Many(v) => v,
})
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct LauncherConfig {
#[serde(default)]
pub bin: Option<String>,
#[serde(default)]
pub example: Option<String>,
#[serde(default)]
pub cmd: Option<String>,
#[serde(default)]
pub script: Option<String>,
#[serde(default)]
pub uv: Option<String>,
#[serde(default)]
pub python: Option<String>,
#[serde(default)]
pub python3: Option<String>,
#[serde(default)]
pub package: Option<String>,
#[serde(default)]
pub manifest_path: Option<String>,
}
impl Config {
pub fn preset(&self, name: &str) -> Result<Preset, String> {
self.presets.get(name).cloned().ok_or_else(|| {
let mut names: Vec<&str> = self.presets.keys().map(String::as_str).collect();
names.sort_unstable();
let known = if names.is_empty() {
"none defined in mira.toml".to_string()
} else {
names.join(", ")
};
format!("no such preset {name:?} (known: {known})")
})
}
pub fn launcher(&self, name: &str) -> Result<LauncherConfig, String> {
self.launchers.get(name).cloned().ok_or_else(|| {
let mut names: Vec<&str> = self.launchers.keys().map(String::as_str).collect();
names.sort_unstable();
let known = if names.is_empty() {
"none defined in mira.toml".to_string()
} else {
names.join(", ")
};
format!("no such launcher {name:?} (known: {known})")
})
}
}
#[derive(Debug, Default, Deserialize)]
pub struct ResultsConfig {
pub dir: Option<String>,
}
#[derive(Debug, Deserialize)]
#[serde(default)]
pub struct EnvironmentConfig {
pub enabled: bool,
pub labels: BTreeMap<String, String>,
}
impl Default for EnvironmentConfig {
fn default() -> Self {
EnvironmentConfig {
enabled: true,
labels: BTreeMap::new(),
}
}
}
impl Config {
pub fn load() -> Config {
let Some(path) = find_config() else {
return Config::default();
};
match std::fs::read_to_string(&path) {
Ok(text) => match Config::parse(&text) {
Ok(mut cfg) => {
cfg.base = path.parent().map(Path::to_path_buf);
cfg
}
Err(e) => {
eprintln!("warning: ignoring {}: {e}", path.display());
Config::default()
}
},
Err(e) => {
eprintln!("warning: cannot read {}: {e}", path.display());
Config::default()
}
}
}
pub fn parse(text: &str) -> Result<Config, toml::de::Error> {
toml::from_str(text)
}
pub fn parse_at(text: &str, base: &Path) -> Result<Config, toml::de::Error> {
let mut cfg = Config::parse(text)?;
cfg.base = Some(base.to_path_buf());
Ok(cfg)
}
pub fn results_dir(&self) -> String {
let dir = self.results.dir.as_deref().unwrap_or(DEFAULT_RESULTS_DIR);
match &self.base {
Some(base) if !Path::new(dir).is_absolute() => {
let rel = dir.strip_prefix("./").unwrap_or(dir);
base.join(rel).to_string_lossy().into_owned()
}
_ => dir.to_string(),
}
}
}
pub fn find_config() -> Option<PathBuf> {
let mut dir = std::env::current_dir().ok()?;
loop {
let candidate = dir.join("mira.toml");
if candidate.is_file() {
return Some(candidate);
}
if !dir.pop() {
return None;
}
}
}
pub fn run_dir(base: &str, run_id: &str) -> PathBuf {
Path::new(base).join(run_id)
}
pub fn encode_key(key: &str) -> String {
let mut safe = String::with_capacity(key.len());
for b in key.bytes() {
if b.is_ascii_alphanumeric() {
safe.push(b as char);
} else {
safe.push('_');
safe.push_str(&format!("{b:02x}"));
}
}
safe
}
pub fn init_run(run_dir: &Path, meta: &RunMeta) -> std::io::Result<()> {
std::fs::create_dir_all(run_dir.join("cases"))?;
write_meta(run_dir, meta)
}
pub fn write_case_result(run_dir: &Path, key: &str, result: &RunResult) -> std::io::Result<()> {
let dir = run_dir.join("cases").join(encode_key(key));
std::fs::create_dir_all(&dir)?;
let json = serde_json::to_string_pretty(result)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
let tmp = dir.join("result.json.tmp");
std::fs::write(&tmp, json)?;
std::fs::rename(&tmp, dir.join("result.json"))
}
pub fn load_case_results(run_dir: &Path) -> Vec<RunResult> {
let mut out = Vec::new();
let entries = match std::fs::read_dir(run_dir.join("cases")) {
Ok(entries) => entries,
Err(_) => return out,
};
for entry in entries.flatten() {
let path = entry.path().join("result.json");
if !path.is_file() {
continue;
}
match std::fs::read_to_string(&path) {
Ok(text) => match serde_json::from_str::<RunResult>(&text) {
Ok(r) => out.push(r),
Err(e) => eprintln!(
"warning: skipping {}: invalid result JSON: {e}",
path.display()
),
},
Err(e) => eprintln!("warning: skipping {}: {e}", path.display()),
}
}
out.sort_by_key(|r| r.key());
out
}
pub fn write_meta(run_dir: &Path, meta: &RunMeta) -> std::io::Result<()> {
let json = serde_json::to_string_pretty(meta)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
std::fs::write(run_dir.join("meta.json"), json)
}
pub fn load_meta(run_dir: &Path) -> Option<RunMeta> {
let text = std::fs::read_to_string(run_dir.join("meta.json")).ok()?;
serde_json::from_str(&text).ok()
}
pub fn finalize_run(
run_dir: &Path,
meta: &RunMeta,
results: &[RunResult],
group: Option<report::Group<'_>>,
) -> std::io::Result<()> {
std::fs::create_dir_all(run_dir)?;
std::fs::write(
run_dir.join("report.json"),
report::render_with_group(results, Format::Json, group),
)?;
std::fs::write(
run_dir.join("report.html"),
report::render_with_group(results, Format::Html, group),
)?;
write_meta(run_dir, meta)
}
#[cfg(test)]
mod tests {
use super::*;
use mira::protocol::TranscriptSummary;
#[test]
fn parse_results_dir() {
let cfg = Config::parse("[results]\ndir = \"/tmp/evals\"\n").unwrap();
assert_eq!(cfg.results_dir(), "/tmp/evals");
}
#[test]
fn environment_capture_on_by_default() {
let cfg = Config::default();
assert!(cfg.environment.enabled);
assert!(cfg.environment.labels.is_empty());
assert!(Config::parse("").unwrap().environment.enabled);
assert!(
Config::parse("[results]\ndir = \"x\"\n")
.unwrap()
.environment
.enabled
);
}
#[test]
fn environment_config_parses_disable_and_labels() {
let cfg = Config::parse(
"[environment]\nenabled = false\n\n[environment.labels]\nteam = \"search\"\n",
)
.unwrap();
assert!(!cfg.environment.enabled);
assert_eq!(
cfg.environment.labels.get("team").map(String::as_str),
Some("search")
);
}
#[test]
fn empty_config_uses_default() {
let cfg = Config::default();
assert_eq!(cfg.results_dir(), DEFAULT_RESULTS_DIR);
assert_eq!(
Config::parse("").unwrap().results_dir(),
DEFAULT_RESULTS_DIR
);
}
#[test]
fn relative_dir_resolves_against_config_parent() {
let cfg = Config {
results: ResultsConfig {
dir: Some("./results".into()),
},
base: Some(PathBuf::from("/proj")),
..Default::default()
};
assert_eq!(cfg.results_dir(), "/proj/results");
let abs = Config {
results: ResultsConfig {
dir: Some("/var/evals".into()),
},
base: Some(PathBuf::from("/proj")),
..Default::default()
};
assert_eq!(abs.results_dir(), "/var/evals");
}
#[test]
fn parse_at_resolves_relative_paths_against_base() {
let cfg = Config::parse_at("[results]\ndir = \"./out\"\n", Path::new("/proj")).unwrap();
assert_eq!(cfg.results_dir(), "/proj/out");
assert!(Config::parse_at("not toml", Path::new("/proj")).is_err());
}
#[test]
fn launchers_parse_and_lookup() {
let cfg = Config::parse(
"default_launcher = \"greet\"\n\n\
[launchers.greet]\nbin = \"greet\"\npackage = \"myapp\"\n\n\
[launchers.py]\ncmd = \"python study.py\"\n",
)
.unwrap();
assert_eq!(cfg.default_launcher.as_deref(), Some("greet"));
let greet = cfg.launcher("greet").unwrap();
assert_eq!(greet.bin.as_deref(), Some("greet"));
assert_eq!(greet.package.as_deref(), Some("myapp"));
assert!(greet.cmd.is_none());
let py = cfg.launcher("py").unwrap();
assert_eq!(py.cmd.as_deref(), Some("python study.py"));
let err = cfg.launcher("nope").unwrap_err();
assert!(err.contains("greet") && err.contains("py"), "{err}");
}
#[test]
fn per_target_and_preset_timeouts_parse() {
let cfg = Config::parse(
"[targets.\"anthropic/opus\"]\ntimeout = 300\n\n\
[presets.slow]\ntimeout = 120\ntargets = [\"sim\"]\n",
)
.unwrap();
assert_eq!(
cfg.targets.get("anthropic/opus").and_then(|t| t.timeout),
Some(300)
);
assert_eq!(cfg.preset("slow").unwrap().timeout, Some(120));
assert!(!cfg.targets.contains_key("sim"));
}
#[test]
fn no_timeouts_by_default() {
let cfg = Config::parse("[presets.smoke]\ntargets = [\"sim\"]\n").unwrap();
assert!(cfg.targets.is_empty());
assert!(cfg.preset("smoke").unwrap().timeout.is_none());
}
#[test]
fn preset_accepts_string_or_list() {
let cfg = Config::parse(
"[presets.smoke]\n\
targets = \"anthropic/*\"\n\
samples = [\"france\", \"spain\"]\n\
evals = \"greet\"\n\
tag = \"quick\"\n",
)
.unwrap();
let p = cfg.preset("smoke").unwrap();
assert_eq!(p.targets, vec!["anthropic/*"]);
assert_eq!(p.samples, vec!["france", "spain"]);
assert_eq!(p.evals, vec!["greet"]);
assert_eq!(p.tag.as_deref(), Some("quick"));
let err = cfg.preset("nope").unwrap_err();
assert!(err.contains("smoke"), "{err}");
}
#[test]
fn no_launchers_is_default() {
let cfg = Config::default();
assert!(cfg.launchers.is_empty());
assert!(cfg.default_launcher.is_none());
assert!(cfg.launcher("x").unwrap_err().contains("none defined"));
}
fn run_result(sample: &str, passed: bool) -> RunResult {
RunResult {
eval: "greet".into(),
sample: sample.into(),
target: "sim".into(),
params: Default::default(),
trial: 0,
trials: 0,
seed: None,
input: Vec::new(),
expected: None,
passed,
aggregate: if passed { 1.0 } else { 0.0 },
scores: vec![],
transcript: TranscriptSummary::default(),
skipped: false,
}
}
#[test]
fn run_store_roundtrips_cases_and_meta() {
use mira::run::{RUN_META_FORMAT, RunSummary};
let tmp = tempfile::tempdir().unwrap();
let rd = run_dir(tmp.path().to_str().unwrap(), "20260621T090012Z-abcd");
let header = RunMeta {
format: RUN_META_FORMAT,
run_id: "20260621T090012Z-abcd".into(),
study: "greet".into(),
study_version: None,
started_unix: 100,
finished_unix: 0,
environment: None,
summary: RunSummary::default(),
};
init_run(&rd, &header).unwrap();
assert!(rd.join("cases").is_dir());
assert_eq!(load_meta(&rd).unwrap().started_unix, 100);
let hi = run_result("hi", true);
let bye = run_result("bye", false);
write_case_result(&rd, &hi.key(), &hi).unwrap();
write_case_result(&rd, &bye.key(), &bye).unwrap();
let loaded = load_case_results(&rd);
assert_eq!(loaded.len(), 2);
assert_eq!(loaded[0].sample, "bye", "sorted by case key");
assert_eq!(loaded[1].sample, "hi");
let final_meta = RunMeta {
finished_unix: 200,
summary: RunSummary::of(&loaded),
..header
};
finalize_run(&rd, &final_meta, &loaded, None).unwrap();
assert!(rd.join("report.json").is_file());
assert!(rd.join("report.html").is_file());
assert_eq!(load_meta(&rd).unwrap().finished_unix, 200);
}
}