mira-cli 0.1.0

//! `mira` — the host CLI. Compiles + spawns an eval **study** (a program that
//! calls `mira::Study::registered().serve()`), enumerates its evals, plans the
//! run (selection × matrix), executes each cell over the protocol, then
//! aggregates, saves, and checkpoints.
//!
//! ```bash
//! mira --bin greet list
//! mira --bin greet run                          # all cells (sim runs; keyed cells skip)
//! mira --bin greet run greet                    # substring filter
//! mira --bin greet run --tag smoke
//! mira --bin greet run --targets sim --format junit --out results.xml
//! mira --bin greet run --checkpoint ck.json     # resumable
//! mira --bin greet run --save                   # archive run under ./results/<run_id>/
//! mira --bin greet run --execute-only --artifacts art/  # capture transcripts
//! mira --bin greet score --artifacts art/        # score (or re-score) them
//! ```
//!
//! Execution and scoring can be split: `run --execute-only` captures one
//! full-transcript artifact per cell (for long-running subjects), and `score`
//! (re-)scores those artifacts without re-executing the subject.
//!
//! Each Rust example is a crate exposing a like-named binary, so `--bin <name>`
//! resolves it across the workspace. Point it at any study: `--bin NAME`,
//! `--example NAME`, an arbitrary `--cmd "..."` (e.g. a Python study), or
//! another package with `--package` / `--manifest-path`.

use std::collections::BTreeMap;
use std::io::IsTerminal;
use std::path::Path;
use std::str::FromStr;
use std::sync::Arc;

use clap::{Args, CommandFactory, Parser, Subcommand};
use indicatif::{ProgressBar, ProgressDrawTarget, ProgressStyle};
use tokio::process::Command;

mod config;
mod env;

use mira::Host;
use mira::Trial;
use mira::exec::{self, CellSpec, Concurrency};
use mira::protocol::{
    ExecuteResult, InitializeResult, ListResult, RunResult, TranscriptSummary, capabilities,
};
use mira::report::{self, Format};
use mira::run::{RUN_META_FORMAT, RunMeta, RunSummary, new_run_id_at};
use mira::session::{self, Session, now_unix};

/// Repository, issue tracker, and docs — surfaced in `mira help --full` and the
/// `--help` footer so an agent (or human) can always find their way home.
const REPO_URL: &str = "https://github.com/everruns/mira";
const ISSUES_URL: &str = "https://github.com/everruns/mira/issues";
const DOCS_URL: &str = "https://github.com/everruns/mira/tree/main/docs";
const API_DOCS_URL: &str = "https://docs.rs/mira-eval";

/// Short tagline for `-h`/`--help`. The CLI is the *host*: it plans the run,
/// drives subjects over the protocol, scores, and reports — not just a runner.
const ABOUT: &str = "Run code-first evals for agents and tools across a target matrix — \
the Mira host CLI.";

/// Footer on every `--help`/no-args screen. The one breadcrumb an agent needs to
/// discover the long-form guide.
const HELP_HINT: &str = "Tip: run `mira help --full` for an overview, every flag, examples, \
and links (repo, issues, docs).";

#[derive(Parser)]
#[command(
    name = "mira",
    version,
    about = ABOUT,
    after_help = HELP_HINT,
    disable_help_subcommand = true,
)]
struct Cli {
    #[command(flatten)]
    launcher: Launcher,
    #[command(subcommand)]
    cmd: Option<Cmd>,
}

/// How to launch the eval study process (not to be confused with a
/// [`mira::Target`] — the model/harness under evaluation).
#[derive(Args)]
struct Launcher {
    /// Run `cargo run -q --bin <NAME>` (defaults to `greet`).
    #[arg(long, global = true)]
    bin: Option<String>,
    /// Run `cargo run -q --example <NAME>`.
    #[arg(long, global = true)]
    example: Option<String>,
    /// Launch an arbitrary command (split on whitespace).
    #[arg(long, global = true)]
    cmd: Option<String>,
    /// Cargo package to run the bin/example from (`-p`).
    #[arg(long, global = true)]
    package: Option<String>,
    /// Passed through to cargo.
    #[arg(long, global = true)]
    manifest_path: Option<String>,
}

#[derive(Subcommand)]
enum Cmd {
    /// List the evals, samples, scorers, and targets the study advertises.
    List,
    /// Run selected cells and report.
    Run(RunArgs),
    /// Score (or re-score) previously captured execution artifacts.
    Score(ScoreArgs),
    /// Show help. Add `--full` for an overview, every flag, examples, and links.
    Help(HelpArgs),
}

#[derive(Args)]
struct HelpArgs {
    /// Render the full guide: high-level overview, all flags, examples, and
    /// contact links — written so an agent can self-orient in one read.
    #[arg(long)]
    full: bool,
}

#[derive(Args)]
struct RunArgs {
    /// Substring filter on the case key `eval/sample@target`.
    filter: Option<String>,
    /// Only run samples carrying this tag.
    #[arg(long)]
    tag: Option<String>,
    /// Restrict the primary (target) axis to these labels (comma-separated).
    /// Sugar for `--axis target=…`.
    #[arg(long)]
    targets: Option<String>,
    /// Restrict a matrix axis to a subset, e.g. `--axis effort=high,low`
    /// (repeatable). `NAME` is `target` (the primary axis) or any declared axis;
    /// values OR within one flag, multiple `--axis` flags AND (intersect).
    #[arg(long = "axis", value_name = "NAME=V1,V2")]
    axes: Vec<String>,
    /// Apply a named selection preset from `mira.toml` (`[presets.NAME]`): a saved
    /// bundle of targets / axes / tag / filter / evals. Explicit flags override
    /// the preset.
    #[arg(long)]
    preset: Option<String>,
    /// Run each cell this many times (trials/repetitions) for pass@k / variance.
    /// Overrides the eval's declared trials. The host groups the repetitions and
    /// reports pass-rate, pass@k, and score standard deviation per cell.
    #[arg(long)]
    trials: Option<usize>,
    /// Base seed for reproducible trials: trial `t` runs with seed `seed + t`.
    /// Overrides the eval's declared seed; the subject reads it via `cx.seed()`.
    #[arg(long)]
    seed: Option<u64>,
    /// Break resolve-rate down by a metadata key (e.g. `repo`, `difficulty`,
    /// `agent`). The value is resolved per cell from, in order: axis params,
    /// sample metadata, target metadata, then transcript metadata.
    #[arg(long)]
    group_by: Option<String>,
    /// Write a report file (see --format).
    #[arg(long)]
    out: Option<String>,
    /// Save a timestamped run folder (report.json/html + meta.json) under the
    /// results dir, so runs accumulate and can be compared later. With no value
    /// uses `[results].dir` from mira.toml, else `./results`; pass a dir to
    /// override.
    #[arg(long, num_args = 0..=1, default_missing_value = "", value_name = "DIR")]
    save: Option<String>,
    /// Report file format: json | junit | md | html.
    #[arg(long, default_value = "json")]
    format: String,
    /// Persist/resume results here; completed cells are skipped on re-run.
    #[arg(long)]
    checkpoint: Option<String>,
    /// Ignore an existing checkpoint/artifact and run everything fresh.
    #[arg(long)]
    fresh: bool,
    /// Max cells to run in parallel across all providers.
    #[arg(long, short = 'j', default_value_t = 8)]
    max_concurrent: usize,
    /// Per-provider concurrency ceilings, e.g. `anthropic=2,openai=4`. Caps a
    /// single provider below the global limit so it can't be flooded.
    #[arg(long)]
    provider_concurrency: Option<String>,
    /// Disable adaptive throttling (don't shrink a provider's concurrency or
    /// retry when it returns rate-limit / overload errors).
    #[arg(long)]
    no_adaptive: bool,
    /// Times a rate-limited cell is retried (after backoff) before it's failed.
    #[arg(long, default_value_t = 4)]
    max_retries: u32,
    /// Execute subjects only (no scoring), writing full transcripts to
    /// --artifacts for later `mira score`. For long-running subjects.
    /// --checkpoint/--out don't apply in this mode (no scores are produced).
    #[arg(long, requires = "artifacts", conflicts_with_all = ["checkpoint", "out", "save"])]
    execute_only: bool,
    /// Directory for full-transcript execution artifacts (one JSON per cell).
    /// Written when --execute-only; read by `mira score`.
    #[arg(long)]
    artifacts: Option<String>,
}

#[derive(Args)]
struct ScoreArgs {
    /// Substring filter on the case key `eval/sample@target`.
    filter: Option<String>,
    /// Directory of execution artifacts written by `run --execute-only`.
    #[arg(long)]
    artifacts: String,
    /// Break resolve-rate down by a metadata key (see `run --group-by`).
    #[arg(long)]
    group_by: Option<String>,
    /// Write a report file (see --format).
    #[arg(long)]
    out: Option<String>,
    /// Save a timestamped run folder under the results dir (see `run --save`).
    #[arg(long, num_args = 0..=1, default_missing_value = "", value_name = "DIR")]
    save: Option<String>,
    /// Report file format: json | junit | md | html.
    #[arg(long, default_value = "json")]
    format: String,
}

#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
    let cli = Cli::parse();

    // Help paths need no study process — handle them before spawning anything.
    // Bare `mira` (no subcommand) prints help too, so an agent landing here sees
    // the `mira help --full` breadcrumb instead of an opaque error.
    match &cli.cmd {
        None => {
            Cli::command().print_help()?;
            return Ok(());
        }
        Some(Cmd::Help(args)) => {
            if args.full {
                print_full_help()?;
            } else {
                Cli::command().print_help()?;
            }
            return Ok(());
        }
        _ => {}
    }

    // One progress bar, shared with the event handler so study `log` lines print
    // cleanly above the bar (via `suspend`) instead of corrupting it. It starts
    // hidden; `run` gives it a length and a draw target once the plan is known.
    let progress = Arc::new(ProgressBar::hidden());
    let progress_evt = progress.clone();
    let host = Host::spawn(build_command(&cli.launcher))
        .await?
        .on_event(move |n| {
            // Per-cell `event` notifications correlate to their `run` by
            // `request_id`; here only study `log`s are surfaced, so they no
            // longer spam stderr.
            if let Some(log) = n.as_log() {
                progress_evt.suspend(|| eprintln!("  study: {}", log.message));
            }
        });

    let info = host.initialize("mira-cli").await?;
    eprintln!(
        "study {} · protocol {} · {} evals",
        info.study, info.protocol_version, info.evals
    );
    // `list_complete` pages `list_samples` so the planner sees every sample even
    // when a study paginates a large/lazy dataset; small studies cost no extra
    // round-trips (no cursor ⇒ no follow-up calls).
    let listing = host.list_complete().await?;

    match cli.cmd {
        Some(Cmd::List) => {
            print_listing(&listing);
            host.shutdown().await?;
            Ok(())
        }
        Some(Cmd::Run(args)) => run(host, info, listing, args, progress).await,
        Some(Cmd::Score(args)) => score(host, info, listing, args).await,
        // Help/no-args returned earlier, before the host was spawned.
        None | Some(Cmd::Help(_)) => unreachable!("handled before host spawn"),
    }
}

/// `mira help --full`: the self-orienting guide. High-level description, the full
/// flag set (clap's own rendering, so it never drifts from the parser), worked
/// examples, and contact links — one read to get an agent productive.
fn print_full_help() -> std::io::Result<()> {
    use std::io::Write;

    let overview = "\
OVERVIEW
  Mira is a Rust-first, code-first evaluation framework for agents and tools —
  built for multi-turn, tool-using, long-running trajectories.

  You write evals as code (in Rust, or any language that speaks the protocol);
  this binary is the HOST. It owns the run end to end: it launches your eval
  program (the `study`), enumerates what it advertises, plans the grid
  (selection x target matrix x axes), executes each cell over the protocol,
  scores the results, then aggregates, reports, and checkpoints. Execution and
  scoring can be split for long runs (`run --execute-only` then `score`).

  Point it at any study: `--bin NAME`, `--example NAME`, an arbitrary
  `--cmd \"...\"` (e.g. a Python study), or `--package` / `--manifest-path`.";

    let examples = "\
EXAMPLES
  mira --bin greet list                 # what the study advertises
  mira --bin greet run                  # run the whole matrix
  mira --bin greet run greet            # selective (substring), like cargo test
  mira --bin greet run --tag smoke      # only samples carrying a tag
  mira --bin greet run --targets sim --format junit --out results.xml
  mira --bin greet run --format html --out report.html   # self-contained viewer
  mira --bin greet run --checkpoint ck.json              # resumable long runs
  mira --bin greet run --save                            # archive run under ./results/<run_id>/
  mira --bin greet run --execute-only --artifacts art/   # capture transcripts
  mira --bin greet score --artifacts art/                # score (or re-score) them
  mira --cmd \"python study.py\" run     # drive a non-Rust (polyglot) study";

    let links = format!(
        "\
LINKS
  Repository:  {REPO_URL}
  Issues:      {ISSUES_URL}
  Docs:        {DOCS_URL}
  API docs:    {API_DOCS_URL}"
    );

    // The full flag set, straight from the parser so it never drifts. Strip the
    // about header and after_help footer — we frame those ourselves here.
    let flags = Cli::command()
        .about(None)
        .after_help(None)
        .render_long_help();

    let mut out = std::io::stdout().lock();
    writeln!(out, "{ABOUT}\n")?;
    writeln!(out, "{overview}\n")?;
    write!(out, "{flags}")?;
    writeln!(out, "\n{examples}\n")?;
    writeln!(out, "{links}")?;
    Ok(())
}

/// Build the study launch command from the target flags.
fn build_command(launcher: &Launcher) -> Command {
    if let Some(raw) = &launcher.cmd {
        let mut parts = raw.split_whitespace();
        let program = parts.next().unwrap_or("false");
        let mut command = Command::new(program);
        command.args(parts);
        return command;
    }

    let mut command = Command::new("cargo");
    command.arg("run").arg("-q");
    if let Some(pkg) = &launcher.package {
        command.arg("-p").arg(pkg);
    }
    if let Some(bin) = &launcher.bin {
        command.arg("--bin").arg(bin);
    } else if let Some(example) = &launcher.example {
        command.arg("--example").arg(example);
    } else {
        // Default to the bundled `greet` example crate's binary.
        command.arg("--bin").arg("greet");
    }
    if let Some(manifest) = &launcher.manifest_path {
        command.arg("--manifest-path").arg(manifest);
    }
    command
}

async fn run(
    host: Host,
    info: InitializeResult,
    listing: ListResult,
    args: RunArgs,
    progress: Arc<ProgressBar>,
) -> Result<(), Box<dyn std::error::Error>> {
    let format = Format::from_str(&args.format)?;
    // Capture run identity at invocation start so the sortable run-id timestamp
    // matches `started_unix` (not the later finish time).
    let started_unix = now_unix();
    let run_id = new_run_id_at(started_unix);
    let save_dir = config::resolve_save_dir(&args.save);

    // Resolve selection (preset + flags), validate it against the advertised
    // axes/targets, then plan the full grid up front — so the host owns
    // selection/matrix without the study re-running anything.
    let selection = resolve_selection(&args).map_err(Box::<dyn std::error::Error>::from)?;
    validate_selection(&selection, &listing).map_err(Box::<dyn std::error::Error>::from)?;
    let plan = plan_grid(&listing, &args, &selection);
    if plan.is_empty() {
        eprintln!("no cells matched the selection");
    }

    // Execute-only: run subjects, persist full transcripts, defer scoring.
    if args.execute_only {
        require_capability(&info, capabilities::EXECUTE, "--execute-only")?;
        let dir = args.artifacts.as_ref().expect("clap requires artifacts");
        return execute_only(host, &plan, dir, args.fresh).await;
    }

    // Fingerprint the advertised definitions so a resume can detect stale caches.
    let fingerprints = session::fingerprints(&listing);

    // Resume from a session checkpoint unless --fresh. The session carries the
    // planned `total` and per-eval fingerprints, so we can report accurate
    // progress and warn when a cached cell's eval definition has changed.
    let mut done: BTreeMap<String, RunResult> = BTreeMap::new();
    let mut session = Session::new(
        info.study.clone(),
        info.study_version.clone(),
        plan.len(),
        fingerprints.clone(),
    );
    if let Some(path) = &args.checkpoint
        && !args.fresh
    {
        match Session::load(path) {
            Ok(Some(prev)) => {
                let stale = prev.stale_keys(&fingerprints);
                for r in prev.results {
                    done.insert(r.key(), r);
                }
                session.created_unix = prev.created_unix;
                let resumable = plan.iter().filter(|c| done.contains_key(&c.key())).count();
                eprintln!(
                    "resuming checkpoint: {resumable}/{} cells already done",
                    plan.len()
                );
                if !stale.is_empty() {
                    eprintln!(
                        "warning: {} cached cell(s) are stale — their eval definition changed \
                         since they were recorded. They'll be reused as-is; re-run with --fresh \
                         to recompute. e.g. {}",
                        stale.len(),
                        stale.iter().take(3).cloned().collect::<Vec<_>>().join(", "),
                    );
                }
            }
            // No checkpoint yet — a normal first run.
            Ok(None) => {}
            // The file exists but can't be used; don't silently discard it.
            Err(e) => eprintln!("warning: ignoring checkpoint {path}: {e}; starting fresh"),
        }
    }

    // Configure the progress bar now the plan is known. Drawn only when stderr is
    // a terminal, so it stays hidden when piped or redirected (e.g. under CI) and
    // never pollutes logs; the final report still prints.
    let resumable = plan.iter().filter(|c| done.contains_key(&c.key())).count();
    if !plan.is_empty() && std::io::stderr().is_terminal() {
        progress.set_draw_target(ProgressDrawTarget::stderr());
        progress.set_style(
            ProgressStyle::with_template(
                "{spinner:.green} [{elapsed_precise}] [{bar:30.cyan/blue}] \
                 {pos}/{len} (eta {eta}) {msg}",
            )
            .unwrap()
            .progress_chars("=>-"),
        );
    }
    progress.set_length(plan.len() as u64);
    progress.set_position(resumable as u64);

    // Only run cells not already checkpointed.
    let todo: Vec<CellSpec> = plan
        .iter()
        .filter(|cell| !done.contains_key(&cell.key()))
        .cloned()
        .collect();

    let cfg = concurrency(&args);

    // Run cells concurrently under the bounded, provider-aware policy. Each
    // finished cell advances the bar and is persisted to the session checkpoint
    // as it lands, so a long run stays resumable.
    {
        let handle = host.handle();
        exec::run_cells(
            todo,
            &cfg,
            |cell| {
                let handle = handle.clone();
                async move {
                    handle
                        .run(
                            &cell.eval,
                            &cell.sample,
                            &cell.target,
                            &cell.params,
                            cell.trial,
                        )
                        .await
                }
            },
            |cell, result| {
                progress.set_message(cell.key());
                done.insert(cell.key(), result);
                progress.inc(1);
                if let Some(path) = &args.checkpoint {
                    // Persist only the planned cells, in plan order — so the file
                    // stays deterministic and doesn't accumulate cells dropped by a
                    // narrower selection on resume.
                    let results = plan
                        .iter()
                        .filter_map(|c| done.get(&c.key()).cloned())
                        .collect();
                    session.update(plan.len(), fingerprints.clone(), results);
                    if let Err(e) = session.save(path) {
                        progress.suspend(|| eprintln!("warning: failed to write checkpoint: {e}"));
                    }
                }
            },
        )
        .await;
    }
    progress.finish_and_clear();
    host.shutdown().await?;

    // Report only the planned cells, in plan order.
    let results: Vec<RunResult> = plan
        .iter()
        .filter_map(|cell| done.get(&cell.key()).cloned())
        .collect();

    report::print_results(&results);

    // Resolve the --group-by breakdown once; reused by the terminal, file, and
    // saved reports so they all agree.
    let group_vals = args
        .group_by
        .as_deref()
        .map(|key| group_values(&results, key, &listing));
    let group = match (args.group_by.as_deref(), group_vals.as_deref()) {
        (Some(key), Some(values)) => {
            report::print_group_breakdown(&results, key, values);
            Some(report::Group { key, values })
        }
        _ => None,
    };

    if let Some(path) = &args.out {
        std::fs::write(path, report::render_with_group(&results, format, group))?;
        eprintln!("\nwrote {path} ({:?})", format);
    }

    if let Some(base) = &save_dir {
        save_results(base, &run_id, &info, started_unix, &results, group)?;
    }

    // A cell that's N/A (all scores N/A — e.g. an infra failure) is neither
    // passed nor failed, so it doesn't make CI red.
    let failed = results
        .iter()
        .any(|r| !r.skipped && !report::is_na(r) && !r.passed);
    std::process::exit(if failed { 1 } else { 0 });
}

/// Write a timestamped run folder under `base` and report where it landed. The
/// `run_id` is captured at invocation start so it sorts by start time.
fn save_results(
    base: &str,
    run_id: &str,
    info: &InitializeResult,
    started_unix: u64,
    results: &[RunResult],
    group: Option<report::Group<'_>>,
) -> Result<(), Box<dyn std::error::Error>> {
    // Capture environment context (commit, box, host version, labels) unless
    // disabled in mira.toml. Best-effort: never fails the save.
    let cfg = config::Config::load();
    let environment = cfg
        .environment
        .enabled
        .then(|| env::collect(&cfg.environment.labels))
        .flatten();
    let meta = RunMeta {
        format: RUN_META_FORMAT,
        run_id: run_id.to_string(),
        study: info.study.clone(),
        study_version: info.study_version.clone(),
        started_unix,
        finished_unix: now_unix(),
        environment,
        summary: RunSummary::of(results),
    };
    let dir = config::save_run(base, &meta, results, group)?;
    eprintln!("\nsaved run {} to {}", meta.run_id, dir.display());
    Ok(())
}

/// Build the concurrency policy from the run flags.
fn concurrency(args: &RunArgs) -> Concurrency {
    let mut cfg = Concurrency::new(args.max_concurrent);
    cfg.adaptive = !args.no_adaptive;
    cfg.max_retries = args.max_retries;
    if let Some(spec) = &args.provider_concurrency {
        for entry in spec.split(',') {
            let entry = entry.trim();
            if entry.is_empty() {
                continue;
            }
            if let Some((provider, n)) = entry.split_once('=')
                && let Ok(limit) = n.trim().parse::<usize>()
            {
                cfg = cfg.provider(provider.trim(), limit);
            } else {
                eprintln!("ignoring malformed --provider-concurrency entry: {entry:?}");
            }
        }
    }
    cfg
}

/// Error out if the study didn't advertise `cap`, naming the `feature` that needs
/// it — so a study lacking the optional capability fails fast with a clear
/// message rather than a generic RPC error mid-run.
fn require_capability(
    info: &InitializeResult,
    cap: &str,
    feature: &str,
) -> Result<(), Box<dyn std::error::Error>> {
    if info.capabilities.iter().any(|c| c == cap) {
        return Ok(());
    }
    Err(format!(
        "study {} doesn't support {feature}: it doesn't advertise the `{cap}` capability",
        info.study
    )
    .into())
}

/// `run --execute-only`: run each cell's subject, persist the full transcript as
/// an artifact, and skip scoring. Resumable — a cell whose artifact already
/// exists is skipped unless `--fresh`.
async fn execute_only(
    host: Host,
    plan: &[CellSpec],
    dir: &str,
    fresh: bool,
) -> Result<(), Box<dyn std::error::Error>> {
    std::fs::create_dir_all(dir)?;
    let mut wrote = 0usize;
    for cell in plan {
        let path = artifact_path(dir, &cell.key());
        if !fresh && path.exists() {
            continue;
        }
        let result = host
            .execute(
                &cell.eval,
                &cell.sample,
                &cell.target,
                &cell.params,
                cell.trial,
            )
            .await?;
        std::fs::write(&path, serde_json::to_string_pretty(&result)?)?;
        wrote += 1;
    }
    host.shutdown().await?;
    eprintln!("executed {wrote} cell(s); artifacts in {dir}");
    eprintln!("score them with: mira score --artifacts {dir}");
    Ok(())
}

/// `score`: load execution artifacts, (re-)score each via the study, and report.
/// Re-running this over the same artifacts is a re-score (e.g. after a scorer
/// change) — no subject is re-executed.
async fn score(
    host: Host,
    info: InitializeResult,
    listing: ListResult,
    args: ScoreArgs,
) -> Result<(), Box<dyn std::error::Error>> {
    require_capability(&info, capabilities::SCORE, "mira score")?;
    let format = Format::from_str(&args.format)?;
    let started_unix = now_unix();
    let run_id = new_run_id_at(started_unix);
    let save_dir = config::resolve_save_dir(&args.save);
    let mut artifacts = load_artifacts(&args.artifacts);
    if let Some(f) = &args.filter {
        artifacts.retain(|a| a.key().contains(f.as_str()));
    }
    if artifacts.is_empty() {
        eprintln!("no artifacts in {}", args.artifacts);
    }

    let mut results = Vec::with_capacity(artifacts.len());
    for artifact in &artifacts {
        // A skipped (unexecuted) cell has no transcript to score; pass it through.
        if artifact.skipped {
            results.push(skipped_result(artifact));
        } else {
            results.push(host.score(artifact).await?);
        }
    }
    host.shutdown().await?;

    report::print_results(&results);

    // Resolve the --group-by breakdown once; reused by the terminal, file, and
    // saved reports so they all agree.
    let group_vals = args
        .group_by
        .as_deref()
        .map(|key| group_values(&results, key, &listing));
    let group = match (args.group_by.as_deref(), group_vals.as_deref()) {
        (Some(key), Some(values)) => {
            report::print_group_breakdown(&results, key, values);
            Some(report::Group { key, values })
        }
        _ => None,
    };

    if let Some(path) = &args.out {
        std::fs::write(path, report::render_with_group(&results, format, group))?;
        eprintln!("\nwrote {path} ({:?})", format);
    }

    if let Some(base) = &save_dir {
        save_results(base, &run_id, &info, started_unix, &results, group)?;
    }

    // A cell that's N/A (all scores N/A — e.g. an infra failure) is neither
    // passed nor failed, so it doesn't make CI red.
    let failed = results
        .iter()
        .any(|r| !r.skipped && !report::is_na(r) && !r.passed);
    std::process::exit(if failed { 1 } else { 0 });
}

/// A skipped (unexecuted) artifact carried straight to a `RunResult`.
fn skipped_result(a: &ExecuteResult) -> RunResult {
    RunResult {
        eval: a.eval.clone(),
        sample: a.sample.clone(),
        target: a.target.clone(),
        params: a.params.clone(),
        trial: a.trial,
        trials: a.trials,
        seed: a.seed,
        passed: false,
        aggregate: 0.0,
        scores: Vec::new(),
        transcript: TranscriptSummary::of(&a.transcript),
        skipped: true,
    }
}

/// Filesystem path for a cell's artifact under `dir`. The cell key is encoded
/// reversibly — `[A-Za-z0-9]` kept verbatim, every other byte escaped as `_HH`
/// (hex) — so distinct keys can never collide onto the same filename (which would
/// overwrite an artifact or wrongly skip execution on resume).
fn artifact_path(dir: &str, key: &str) -> std::path::PathBuf {
    let mut safe = String::with_capacity(key.len());
    for b in key.bytes() {
        if b.is_ascii_alphanumeric() {
            safe.push(b as char);
        } else {
            safe.push('_');
            safe.push_str(&format!("{b:02x}"));
        }
    }
    Path::new(dir).join(format!("{safe}.json"))
}

/// Load every execution artifact in `dir`, sorted by cell key for stable order.
/// Unreadable or invalid files are skipped with a warning, so a corrupted or
/// partially-written artifact is visible rather than silently dropped.
fn load_artifacts(dir: &str) -> Vec<ExecuteResult> {
    let mut out = Vec::new();
    let entries = match std::fs::read_dir(dir) {
        Ok(entries) => entries,
        Err(e) => {
            eprintln!("warning: cannot read artifacts dir {dir}: {e}");
            return out;
        }
    };
    for entry in entries.flatten() {
        let path = entry.path();
        if path.extension().and_then(|e| e.to_str()) != Some("json") {
            continue;
        }
        match std::fs::read_to_string(&path) {
            Ok(text) => match serde_json::from_str::<ExecuteResult>(&text) {
                Ok(result) => out.push(result),
                Err(e) => {
                    eprintln!(
                        "warning: skipping {}: invalid artifact JSON: {e}",
                        path.display()
                    )
                }
            },
            Err(e) => eprintln!("warning: skipping {}: {e}", path.display()),
        }
    }
    out.sort_by_key(|a| a.key());
    out
}

/// Every combination of axis values for an eval, as `params` maps (cross
/// product), always including at least one empty map.
fn axis_combinations(eval: &mira::protocol::EvalInfo) -> Vec<BTreeMap<String, String>> {
    let mut combos = vec![BTreeMap::new()];
    for axis in &eval.axes {
        let mut next = Vec::new();
        for combo in &combos {
            for value in &axis.values {
                let mut c = combo.clone();
                c.insert(axis.name.clone(), value.clone());
                next.push(c);
            }
        }
        if !next.is_empty() {
            combos = next;
        }
    }
    combos
}

/// The resolved selection for a run: a host-side *subset* of the declared grid.
/// Built from `--preset` (if any) and explicit flags (flags win). `axes` keys an
/// axis name (`target` for the primary axis, or any declared axis) to the values
/// kept; `evals` restricts which evals (hence subjects) run.
struct Selection {
    filter: Option<String>,
    tag: Option<String>,
    /// `None` = every eval; `Some` = only these.
    evals: Option<Vec<String>>,
    /// Axis name → allowed values. `target` is the primary axis. An axis absent
    /// here is unconstrained.
    axes: BTreeMap<String, Vec<String>>,
}

/// Split a comma-separated value list, trimming and dropping empties.
fn split_csv(s: &str) -> Vec<String> {
    s.split(',')
        .map(|v| v.trim().to_string())
        .filter(|v| !v.is_empty())
        .collect()
}

/// Resolve the selection from a `--preset` (loaded from `mira.toml`) overlaid by
/// explicit flags. `--targets` folds into the `target` axis; `--axis NAME=v,v`
/// (repeatable) sets/overrides any axis.
fn resolve_selection(args: &RunArgs) -> Result<Selection, String> {
    let preset = match &args.preset {
        Some(name) => config::Config::load().preset(name)?,
        None => config::Preset::default(),
    };

    let filter = args.filter.clone().or(preset.filter);
    let tag = args.tag.clone().or(preset.tag);
    let evals = (!preset.evals.is_empty()).then_some(preset.evals.clone());

    // Start from the preset's axis constraints, then layer flags on top.
    let mut axes: BTreeMap<String, Vec<String>> = preset.axes.clone();

    // Primary axis: --targets wins over the preset's targets.
    let targets = match &args.targets {
        Some(s) => split_csv(s),
        None => preset.targets.clone(),
    };
    if !targets.is_empty() {
        axes.insert("target".to_string(), targets);
    }

    // --axis NAME=V1,V2 (repeatable) overrides the same-named axis.
    for spec in &args.axes {
        let (name, vals) = spec
            .split_once('=')
            .ok_or_else(|| format!("--axis expects NAME=V1,V2, got {spec:?}"))?;
        let values = split_csv(vals);
        if values.is_empty() {
            return Err(format!("--axis {name}= lists no values"));
        }
        axes.insert(name.trim().to_string(), values);
    }

    Ok(Selection {
        filter,
        tag,
        evals,
        axes,
    })
}

/// Reject a selection that names an axis/value/eval the study didn't advertise,
/// so a typo fails loudly instead of silently matching nothing.
fn validate_selection(sel: &Selection, listing: &ListResult) -> Result<(), String> {
    use std::collections::BTreeSet;
    // Declared axis values across all evals: `target` → all target labels; each
    // declared axis → the union of its values.
    let mut declared: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
    for eval in &listing.evals {
        let t = declared.entry("target".to_string()).or_default();
        for m in &eval.targets {
            t.insert(m.label.clone());
        }
        for a in &eval.axes {
            let e = declared.entry(a.name.clone()).or_default();
            for v in &a.values {
                e.insert(v.clone());
            }
        }
    }
    let listed = |set: &BTreeSet<String>| {
        let mut v: Vec<&str> = set.iter().map(String::as_str).collect();
        v.sort_unstable();
        v.join(", ")
    };
    for (name, allowed) in &sel.axes {
        let Some(valid) = declared.get(name) else {
            let names: BTreeSet<String> = declared.keys().cloned().collect();
            return Err(format!(
                "unknown axis {name:?} (declared: {})",
                listed(&names)
            ));
        };
        for v in allowed {
            if !valid.contains(v) {
                return Err(format!(
                    "axis {name:?} has no value {v:?} (declared: {})",
                    listed(valid)
                ));
            }
        }
    }
    if let Some(evals) = &sel.evals {
        let known: BTreeSet<String> = listing.evals.iter().map(|e| e.name.clone()).collect();
        for e in evals {
            if !known.contains(e) {
                return Err(format!("unknown eval {e:?} (declared: {})", listed(&known)));
            }
        }
    }
    Ok(())
}

/// True when a cell's axis params satisfy the (non-target) axis constraints.
fn axes_allowed(sel: &Selection, params: &mira::Params) -> bool {
    params.iter().all(|(name, value)| {
        sel.axes
            .get(name)
            .is_none_or(|allowed| allowed.contains(value))
    })
}

/// Expand the advertised listing into an ordered, selected list of cells. Each
/// cell carries its target's provider so the executor can bucket concurrency.
fn plan_grid(listing: &ListResult, args: &RunArgs, sel: &Selection) -> Vec<CellSpec> {
    let mut plan = Vec::new();
    for eval in &listing.evals {
        if let Some(evals) = &sel.evals
            && !evals.iter().any(|e| e == &eval.name)
        {
            continue;
        }
        let combos = axis_combinations(eval);
        // Trials: --trials overrides the eval's declared count (0/1 → single).
        // Seed base: --seed overrides the eval's declared seed; trial t uses
        // `base + t` so the repetition set replays deterministically.
        let trials = args.trials.unwrap_or(eval.trials).max(1);
        let seed_base = args.seed.or(eval.seed);
        for sample in &eval.samples {
            if let Some(tag) = &sel.tag
                && !sample.tags.contains(tag)
            {
                continue;
            }
            for target in &eval.targets {
                if let Some(allow) = sel.axes.get("target")
                    && !allow.contains(&target.label)
                {
                    continue;
                }
                for params in &combos {
                    if !axes_allowed(sel, params) {
                        continue;
                    }
                    let key = mira::cell_key(&eval.name, &sample.id, &target.label, params);
                    // Filter on the logical key, so `--filter` keeps or drops all
                    // trials of a cell together (a stable group to aggregate).
                    if let Some(f) = &sel.filter
                        && !key.contains(f.as_str())
                    {
                        continue;
                    }
                    for index in 0..trials {
                        plan.push(CellSpec {
                            eval: eval.name.clone(),
                            sample: sample.id.clone(),
                            target: target.label.clone(),
                            provider: target.provider.clone(),
                            params: params.clone(),
                            trial: Trial {
                                index,
                                count: trials,
                                // wrapping_add so a huge --seed can't panic
                                // (debug) or differ by build mode (release).
                                seed: seed_base.map(|s| s.wrapping_add(index as u64)),
                            },
                        });
                    }
                }
            }
        }
    }
    plan
}

/// Per-(eval,sample) and per-(eval,target) metadata, indexed from the listing so
/// `--group-by` can resolve a key against the sample and target columns.
type MetaIndex = BTreeMap<(String, String), mira::Metadata>;

fn meta_indexes(listing: &ListResult) -> (MetaIndex, MetaIndex) {
    let mut sample_meta = MetaIndex::new();
    let mut model_meta = MetaIndex::new();
    for eval in &listing.evals {
        for s in &eval.samples {
            if !s.metadata.is_empty() {
                sample_meta.insert((eval.name.clone(), s.id.clone()), s.metadata.clone());
            }
        }
        for m in &eval.targets {
            if !m.metadata.is_empty() {
                model_meta.insert((eval.name.clone(), m.label.clone()), m.metadata.clone());
            }
        }
    }
    (sample_meta, model_meta)
}

/// Resolve a `--group-by` key for one cell, in priority order: axis params,
/// sample metadata, target metadata, then transcript metadata. `None` ⇒ the cell
/// carried no value for the key.
fn group_value(
    r: &RunResult,
    key: &str,
    sample_meta: &MetaIndex,
    model_meta: &MetaIndex,
) -> Option<String> {
    if let Some(v) = r.params.get(key) {
        return Some(v.clone());
    }
    if let Some(v) = sample_meta
        .get(&(r.eval.clone(), r.sample.clone()))
        .and_then(|m| m.get(key))
    {
        return Some(mira::metadata_display(v));
    }
    if let Some(v) = model_meta
        .get(&(r.eval.clone(), r.target.clone()))
        .and_then(|m| m.get(key))
    {
        return Some(mira::metadata_display(v));
    }
    r.transcript.metadata.get(key).map(mira::metadata_display)
}

/// Resolve `--group-by` values for every result (parallel to `results`).
fn group_values(results: &[RunResult], key: &str, listing: &ListResult) -> Vec<Option<String>> {
    let (sample_meta, model_meta) = meta_indexes(listing);
    results
        .iter()
        .map(|r| group_value(r, key, &sample_meta, &model_meta))
        .collect()
}

fn print_listing(listing: &ListResult) {
    for eval in &listing.evals {
        let desc = if eval.description.is_empty() {
            String::new()
        } else {
            format!(" — {}", eval.description)
        };
        let trials = if eval.trials > 1 {
            let seed = eval.seed.map(|s| format!(", seed={s}")).unwrap_or_default();
            format!(", trials={}{seed}", eval.trials)
        } else {
            String::new()
        };
        println!(
            "{}{desc}  (max_turns={}{trials})",
            eval.name, eval.max_turns
        );
        println!(
            "  samples: {}",
            eval.samples
                .iter()
                .map(|s| {
                    let mut label = s.id.clone();
                    if !s.tags.is_empty() {
                        label.push_str(&format!(" [{}]", s.tags.join(",")));
                    }
                    if !s.metadata.is_empty() {
                        label.push_str(&format!(" {{{}}}", fmt_meta(&s.metadata)));
                    }
                    label
                })
                .collect::<Vec<_>>()
                .join(", ")
        );
        println!("  scorers: {}", eval.scorers.join(", "));
        println!(
            "  targets:  {}",
            eval.targets
                .iter()
                .map(|m| {
                    let mut label = m.label.clone();
                    if !m.available {
                        label.push_str(" (unavailable)");
                    }
                    if !m.metadata.is_empty() {
                        label.push_str(&format!(" {{{}}}", fmt_meta(&m.metadata)));
                    }
                    label
                })
                .collect::<Vec<_>>()
                .join(", ")
        );
        if !eval.axes.is_empty() {
            let axes: Vec<String> = eval
                .axes
                .iter()
                .map(|a| format!("{}=[{}]", a.name, a.values.join(",")))
                .collect();
            println!("  axes:    {}", axes.join(", "));
        }
        if !eval.metadata.is_empty() {
            println!("  meta:    {}", fmt_meta(&eval.metadata));
        }
    }
}

/// Render a metadata map as `k=v, …` for the listing (values via the shared
/// `metadata_display`, so a JSON string shows raw and anything else compact JSON).
fn fmt_meta(meta: &mira::Metadata) -> String {
    meta.iter()
        .map(|(k, v)| format!("{k}={}", mira::metadata_display(v)))
        .collect::<Vec<_>>()
        .join(", ")
}