jev-repl 0.8.0

//! `jev` — an interactive playground for learning TypeSafe AI System One questions.
//!
//! Run it with a `TYPESAFE_API_KEY` for live answers, or without one to explore offline with
//! simulated answers. `:help` inside lists every command; `:lesson` starts the guided track.
//!
//! With a subcommand it does not open a terminal at all: `jev run page.jev`, `jev json`, `jev cost`
//! and friends read a page (or stdin) and print one answer, so a session shaped in the REPL can be
//! saved with `:save` and then run from a script, a Makefile or CI.

use std::future::Future;
use std::io::{IsTerminal, Read, Write};
use std::path::Path;
use std::process::ExitCode;
use std::sync::Arc;
use std::time::Duration;

use ratatui::crossterm::event::{
    self, KeyboardEnhancementFlags, PopKeyboardEnhancementFlags, PushKeyboardEnhancementFlags,
};
use ratatui::crossterm::{execute, terminal};
use serde_json::Value;
use tokio::sync::mpsc;

use jev_repl::app::{App, Msg};
use jev_repl::cost::Rates;
use jev_repl::evaluate::{self, Outcome};
use jev_repl::headless::Answered;
use jev_repl::mcp::{self, Host, Sent};
use jev_repl::session::Session;
use jev_repl::{cost, headless, installer, serve, sketch, ui};
use typesafe::{Client, Usage};

/// 0 when it worked, 1 when the call or the file did not, 2 when the command line did not parse.
const OK: u8 = 0;
const FAILED: u8 = 1;
const BAD_USAGE: u8 = 2;

#[tokio::main]
async fn main() -> ExitCode {
    let args: Vec<String> = std::env::args().skip(1).collect();

    if let Some(command) = args.first().filter(|a| headless::is_command(a)) {
        let command = command.clone();
        return ExitCode::from(one_shot(&command, &args[1..]).await);
    }
    if args.first().is_some_and(|a| a == "mcp") {
        return ExitCode::from(serve_mcp(&args[1..]).await);
    }
    if args.first().is_some_and(|a| a == "install") {
        return ExitCode::from(installer::run(&args[1..]));
    }
    if args.iter().any(|a| a == "--help" || a == "-h") {
        print!("{}", help());
        return ExitCode::from(OK);
    }
    if args.iter().any(|a| a == "--version" || a == "-v") {
        println!("{}", env!("CARGO_PKG_VERSION"));
        return ExitCode::from(OK);
    }
    if let Some(unknown) = args.first() {
        eprintln!("jev: unknown command {unknown:?}. jev --help lists them.");
        return ExitCode::from(BAD_USAGE);
    }
    if !std::io::stdin().is_terminal() || !std::io::stdout().is_terminal() {
        eprintln!(
            "jev needs an interactive terminal (stdin and stdout must be a TTY).\n\
             Without one, `jev run <file>` sends a saved page and prints the answers; \
             jev --help lists the rest."
        );
        return ExitCode::from(FAILED);
    }
    match repl().await {
        Ok(()) => ExitCode::from(OK),
        Err(e) => {
            eprintln!("jev: {e}");
            ExitCode::from(FAILED)
        }
    }
}

fn help() -> String {
    let commands: String = headless::COMMANDS
        .iter()
        .map(|(name, about)| format!("  {name:<22} {about}\n"))
        .collect();
    format!(
        "jev — a REPL for TypeSafe AI System One questions\n\n\
         Set TYPESAFE_API_KEY for live answers; without one, answers are simulated locally.\n\n\
         \x20 jev                    the REPL: :help for commands, :lesson for the guided track,\n\
         \x20                        :sketch to write a request as one page, :quit to leave\n\
         \x20 jev <command> [file]   one shot, no terminal needed\n\
         \x20 jev mcp                serve the same commands to an agent over MCP, on stdin/stdout\n\
         \x20 jev install            register the MCP server and the jev skill with an agent\n\
         \x20                        (Claude Code, Codex, OpenCode, pi) — jev install --help\n\n\
         Commands\n{commands}\n\
         The file is a .jev sketch page or a request body; `-`, or no file at all, reads stdin.\n\n\
         Options\n\
         \x20 --state <text>         set the state, or replace the one on the page\n\
         \x20 --turn <who>: <text>   append a turn, making the state a conversation; repeatable\n\
         \x20 --model <name>         the model to ask\n\
         \x20 --threshold <0-1>      what counts as a yes for a noul (default 0.5)\n\
         \x20 --price <in>/<out>     dollars per million tokens, input then output\n\
         \x20 --timeout <seconds>    per-attempt timeout for a live call\n\
         \x20 --mock                 simulated answers, even when a key is set\n\
         \x20 --json                 print the raw response body instead of the answer page\n\
         \x20 --help, -h             this message\n\
         \x20 --version, -v          the version of this package\n\n\
         Options for eval\n\
         \x20 --cases <file>         the JSON Lines file of labelled states to score, or `-`\n\
         \x20 --concurrency <n>      how many cases are in the air at once (default 4)\n\
         \x20 --cache <dir>          keep the responses here, so running it again sends nothing\n\
         \x20 --max-cost <dollars>   refuse to send when the estimate is above this\n\
         \x20 --min-accuracy <0-1>   exit 1 when a scored question falls below this\n\
         \x20 --compare <page>       run a second page over the same cases and report the difference\n\
         \x20 --fail-on-regression   with --compare, exit 1 when the second page is significantly worse\n\
         \x20 --calibrate            write the thresholds and confidence bars the run supports into the page\n\
         \x20 --target-accuracy <0-1>  the accuracy a confidence bar has to reach (default 0.9)\n\n\
         Exit status is 0 when it worked, 1 when the call or the file did not, 2 when the\n\
         command line did not parse.\n"
    )
}

/// Everything the one-shot commands read off the command line.
#[derive(Debug, Default)]
struct Options {
    file: String,
    state: Option<String>,
    /// Turns appended to the state, in the order they were given.
    turns: Vec<String>,
    model: Option<String>,
    threshold: f64,
    rates: Option<Rates>,
    timeout: Option<Duration>,
    mock: bool,
    json: bool,
    /// `jev eval`: the file of labelled cases, and what to do with them.
    cases: Option<String>,
    concurrency: usize,
    cache: Option<String>,
    max_cost: Option<f64>,
    min_accuracy: Option<f64>,
    /// `jev eval --compare`: the second page, and whether a regression fails the run.
    compare: Option<String>,
    fail_on_regression: bool,
    /// `jev eval --calibrate`: write the bars back into the page, aiming at this accuracy.
    calibrate: bool,
    target_accuracy: Option<f64>,
}

/// Read the flags after a subcommand. `--flag value` and `--flag=value` both work, and the first
/// bare word is the file — a page is a path, not a flag, so there is only ever one.
fn parse_options(args: &[String]) -> Result<Options, String> {
    let mut options = Options {
        file: "-".to_owned(),
        threshold: 0.5,
        rates: cost::rates_from_env(),
        concurrency: 4,
        ..Options::default()
    };
    let mut file: Option<String> = None;
    let mut i = 0;
    while i < args.len() {
        let arg = &args[i];
        let (name, inline) = match arg.strip_prefix("--").and_then(|_| arg.split_once('=')) {
            Some((name, value)) => (name.to_owned(), Some(value.to_owned())),
            None => (arg.clone(), None),
        };
        let mut value = || -> Result<String, String> {
            match &inline {
                Some(v) => Ok(v.clone()),
                None => {
                    i += 1;
                    args.get(i)
                        .cloned()
                        .ok_or_else(|| format!("{name} needs a value."))
                }
            }
        };
        match name.as_str() {
            "--state" => options.state = Some(value()?),
            "--turn" => {
                let turn = value()?;
                options.turns.push(turn);
            }
            "--model" => options.model = Some(value()?),
            "--threshold" => {
                let text = value()?;
                let n: f64 = text
                    .parse()
                    .map_err(|_| "--threshold takes a number from 0 to 1.".to_owned())?;
                if !(0.0..=1.0).contains(&n) {
                    return Err("--threshold takes a number from 0 to 1.".to_owned());
                }
                options.threshold = n;
            }
            "--price" => options.rates = Some(cost::parse_rates(&value()?)?),
            "--timeout" => {
                let text = value()?;
                let seconds: f64 = text.parse().unwrap_or(0.0);
                if !(seconds.is_finite() && seconds > 0.0) {
                    return Err("--timeout takes a number of seconds greater than 0.".to_owned());
                }
                options.timeout = Some(Duration::from_secs_f64(seconds));
            }
            "--cases" => options.cases = Some(value()?),
            "--concurrency" => {
                let text = value()?;
                let workers: i64 = text
                    .parse()
                    .map_err(|_| "--concurrency takes a whole number of 1 or more.".to_owned())?;
                if workers < 1 {
                    return Err("--concurrency takes a whole number of 1 or more.".to_owned());
                }
                options.concurrency = workers as usize;
            }
            "--cache" => options.cache = Some(value()?),
            "--max-cost" => {
                let text = value()?;
                let dollars: f64 = text.parse().unwrap_or(f64::NAN);
                if !(dollars.is_finite() && dollars > 0.0) {
                    return Err("--max-cost takes a number of dollars greater than 0.".to_owned());
                }
                options.max_cost = Some(dollars);
            }
            "--min-accuracy" => {
                let text = value()?;
                let bar: f64 = text
                    .parse()
                    .map_err(|_| "--min-accuracy takes a number from 0 to 1.".to_owned())?;
                if !(0.0..=1.0).contains(&bar) {
                    return Err("--min-accuracy takes a number from 0 to 1.".to_owned());
                }
                options.min_accuracy = Some(bar);
            }
            "--compare" => options.compare = Some(value()?),
            "--fail-on-regression" => options.fail_on_regression = true,
            "--calibrate" => options.calibrate = true,
            "--target-accuracy" => {
                let target: f64 = value()?.parse().unwrap_or(f64::NAN);
                if !(0.0..=1.0).contains(&target) {
                    return Err("--target-accuracy takes a number from 0 to 1.".to_owned());
                }
                options.target_accuracy = Some(target);
            }
            "--mock" => options.mock = true,
            "--json" => options.json = true,
            other => {
                if other.starts_with('-') && other != "-" {
                    return Err(format!("unknown option {other}. jev --help lists them."));
                }
                if let Some(first) = &file {
                    return Err(format!("expected one file, got {first:?} and {arg:?}."));
                }
                file = Some(arg.clone());
            }
        }
        i += 1;
    }
    if let Some(path) = file {
        options.file = path;
    }
    Ok(options)
}

/// The flags `eval` reads differently from the other commands.
///
/// `--state` is the interesting one: a page's state is what the cases replace, so passing one
/// would quietly judge the same text forty times. `--turn` is the same thing said a turn at a
/// time — a case that is a conversation carries its turns in its own `state`.
fn check_eval_options(options: &Options) -> Result<(), String> {
    if options.state.is_some() {
        return Err("--state does not apply to eval: the cases carry the states.".to_owned());
    }
    if !options.turns.is_empty() {
        return Err(
            "--turn does not apply to eval: a case's own state carries its turns.".to_owned(),
        );
    }
    let Some(cases) = &options.cases else {
        return Err("--cases <file> is required: jev eval page.jev --cases cases.jsonl".to_owned());
    };
    if cases == "-" && options.file == "-" {
        return Err("the page and the cases cannot both come from stdin.".to_owned());
    }
    if options.max_cost.is_some() && options.rates.is_none() {
        return Err("--max-cost needs rates: pass --price <in>/<out> or set JEV_PRICE.".to_owned());
    }
    if options.compare.as_deref() == Some("-") && (options.file == "-" || cases == "-") {
        return Err("only one of the page, --compare and --cases can come from stdin.".to_owned());
    }
    if options.fail_on_regression && options.compare.is_none() {
        return Err(
            "--fail-on-regression needs --compare: there is nothing to regress from.".to_owned(),
        );
    }
    if options.calibrate && options.compare.is_some() {
        return Err(
            "--calibrate and --compare do not mix: calibrate one page at a time.".to_owned(),
        );
    }
    if options.calibrate && options.file == "-" {
        return Err(
            "--calibrate writes the page back, so the page has to be a file, not stdin.".to_owned(),
        );
    }
    if options.target_accuracy.is_some() && !options.calibrate {
        return Err("--target-accuracy only applies with --calibrate.".to_owned());
    }
    Ok(())
}

/// The page: a file, or everything on stdin when the path is `-`.
fn read_input(path: &str) -> Result<String, String> {
    if path == "-" {
        let mut text = String::new();
        return std::io::stdin()
            .read_to_string(&mut text)
            .map(|_| text)
            .map_err(|e| format!("could not read stdin: {e}"));
    }
    std::fs::read_to_string(path).map_err(|e| format!("could not read {path}: {e}"))
}

/// One shot: read a page, print one thing, say whether it worked.
async fn one_shot(command: &str, args: &[String]) -> u8 {
    let options = match parse_options(args) {
        Ok(options) => options,
        Err(e) => {
            eprintln!("jev {command}: {e}");
            return BAD_USAGE;
        }
    };
    if command == "eval"
        && let Err(e) = check_eval_options(&options)
    {
        eprintln!("jev {command}: {e}");
        return BAD_USAGE;
    }

    let text = match read_input(&options.file) {
        Ok(text) => text,
        Err(e) => {
            eprintln!("jev {command}: {e}");
            return FAILED;
        }
    };

    if command == "check" {
        return match headless::check_text(&text) {
            Ok(summary) => {
                println!("{summary}");
                OK
            }
            Err(problems) => {
                eprintln!("{problems}");
                FAILED
            }
        };
    }

    let mut session: Session = match headless::load(&text) {
        Ok(session) => session,
        Err(e) => {
            eprintln!("jev {command}: {e}");
            return FAILED;
        }
    };
    if let Some(state) = &options.state {
        session.state = serde_json::Value::String(state.clone());
    }
    for text in &options.turns {
        let added = jev_repl::session::parse_turn(text).and_then(|turn| session.add_turn(turn));
        if let Err(e) = added {
            eprintln!("jev {command}: --turn {e}");
            return BAD_USAGE;
        }
    }
    if let Some(model) = &options.model {
        session.model = Some(model.clone());
    }

    // A key makes the model name the client's default; without one the published default stands.
    let client = if options.mock {
        None
    } else {
        Client::from_env().ok()
    };
    let model = session
        .model
        .clone()
        .or_else(|| client.as_ref().map(|c| c.default_model().to_owned()))
        .unwrap_or_else(|| "jev-latest".to_owned());

    if command == "eval" {
        if options.calibrate && text.trim_start().starts_with('{') {
            eprintln!(
                "jev eval: --calibrate needs a .jev page: a request body has nowhere to keep a bar."
            );
            return FAILED;
        }
        return run_eval(session, &text, &options, &model, client).await;
    }

    match command {
        "json" => {
            print!("{}", headless::request_text(&session, &model));
            return OK;
        }
        "cost" => {
            print!("{}", headless::cost_text(&session, &model, options.rates));
            return OK;
        }
        "rust" => {
            print!(
                "{}",
                headless::code_text(&session, &model, options.threshold)
            );
            return OK;
        }
        _ => {}
    }

    if let Some(why) = headless::sendable(&session) {
        eprintln!("jev run: {why}");
        return FAILED;
    }

    let Some(client) = client else {
        let answers = headless::mock_answers(&session);
        if options.json {
            print!("{}", headless::answers_json(&answers, &model, None));
        } else {
            print!(
                "{}",
                headless::session_answers_text(&answers, options.threshold, &session)
            );
            print!(
                "{}",
                headless::usage_text(&session, &model, options.rates, None)
            );
            eprintln!(
                "Simulated answers: deterministic noise, not judgement. Set TYPESAFE_API_KEY for real ones."
            );
        }
        return OK;
    };

    let mut request = client
        .system_one(session.state.clone(), session.to_questions())
        .model(model.clone());
    if let Some(timeout) = options.timeout {
        request = request.timeout(timeout);
    }
    match request.await {
        Ok(response) => {
            let answers = headless::live_answers(&session, &response);
            if options.json {
                print!(
                    "{}",
                    headless::answers_json(&answers, &model, Some(&response.raw))
                );
            } else {
                print!(
                    "{}",
                    headless::session_answers_text(&answers, options.threshold, &session)
                );
                print!(
                    "{}",
                    headless::usage_text(&session, &model, options.rates, Some(&response.usage))
                );
            }
            OK
        }
        Err(e) => {
            let mut stderr = std::io::stderr();
            for line in jev_repl::format::error_lines(&e) {
                let text: String = line.spans.iter().map(|s| s.content.as_ref()).collect();
                let _ = writeln!(stderr, "{text}");
            }
            FAILED
        }
    }
}

/// `jev mcp`: the one-shot commands again, this time as tools an agent can call.
async fn serve_mcp(args: &[String]) -> u8 {
    let options = match parse_options(args) {
        Ok(options) => options,
        Err(e) => {
            eprintln!("jev mcp: {e}");
            return BAD_USAGE;
        }
    };
    if options.file != "-" {
        eprintln!("jev mcp: takes no file: the pages arrive in the tool calls.");
        return BAD_USAGE;
    }
    if options.state.is_some() || !options.turns.is_empty() || options.cases.is_some() {
        eprintln!("jev mcp: --state, --turn and --cases belong to a tool call, not to the server.");
        return BAD_USAGE;
    }

    let client = if options.mock {
        None
    } else {
        Client::from_env().ok()
    };
    let model = options
        .model
        .clone()
        .or_else(|| client.as_ref().map(|c| c.default_model().to_owned()))
        .unwrap_or_else(|| "jev-latest".to_owned());
    let live = client.is_some();

    let timeout = options.timeout;
    let fallback = model.clone();
    let ask: mcp::Ask = match client {
        None => Arc::new(move |session: Session| {
            Box::pin(async move {
                Sent {
                    outcome: Outcome::Ok {
                        answers: headless::mock_answers(&session),
                        usage: None,
                    },
                    raw: None,
                }
            })
        }),
        Some(client) => {
            let client = Arc::new(client);
            Arc::new(move |session: Session| {
                let client = Arc::clone(&client);
                let fallback = fallback.clone();
                Box::pin(async move { mcp_ask(&client, &fallback, timeout, session).await })
            })
        }
    };

    // stdout carries the protocol and nothing else, so the greeting goes to stderr.
    let state = if live {
        format!("live, model {model}")
    } else {
        "no API key — every answer is simulated".to_owned()
    };
    eprintln!("jev mcp {}: {state}", env!("CARGO_PKG_VERSION"));
    serve::serve(Host {
        version: env!("CARGO_PKG_VERSION").to_owned(),
        model,
        live,
        rates: options.rates,
        ask,
    })
    .await;
    OK
}

/// One live call for an MCP tool, with the page's own model when it pins one.
///
/// `jev run` resolves the model once, on the command line; a server answers pages it has never
/// seen, so each one gets to say what it should be asked with.
async fn mcp_ask(
    client: &Client,
    fallback: &str,
    timeout: Option<Duration>,
    session: Session,
) -> Sent {
    let model = session.model.clone().unwrap_or_else(|| fallback.to_owned());
    let mut request = client
        .system_one(session.state.clone(), session.to_questions())
        .model(model);
    if let Some(timeout) = timeout {
        request = request.timeout(timeout);
    }
    match request.await {
        Ok(response) => Sent {
            outcome: Outcome::Ok {
                answers: headless::live_answers(&session, &response),
                usage: Some(response.usage.clone()),
            },
            raw: Some(response.raw.clone()),
        },
        Err(e) => Sent::failed(error_text(&e)),
    }
}

/// Simulated answers for a case, the same deterministic ones `jev run --mock` prints.
async fn mock_ask(session: Session) -> Outcome {
    Outcome::Ok {
        answers: headless::mock_answers(&session),
        usage: None,
    }
}

/// The cache key: the request body this case would POST, hashed. It is the SDK's cassette key,
/// so a cache directory can be replayed with `TYPESAFE_REPLAY` and a recording used as a cache.
fn cache_key(session: &Session, model: &str) -> String {
    typesafe::cassette::key(&session.state, model, &session.to_questions())
}

/// A cached response, or `None` when there is none this run can use.
fn cached(file: &Path, session: &Session) -> Option<(Vec<Answered>, Option<Usage>)> {
    // A file this version cannot read is not worth failing a case over; send the request instead.
    let body: Value = serde_json::from_str(&std::fs::read_to_string(file).ok()?).ok()?;
    headless::cached_answers(session, &body)
}

/// One live call for a case, through the cache when there is one.
async fn live_ask(
    client: &Client,
    model: &str,
    timeout: Option<Duration>,
    cache: Option<&str>,
    session: Session,
) -> Outcome {
    let file = cache.map(|dir| Path::new(dir).join(format!("{}.json", cache_key(&session, model))));
    if let Some(file) = &file
        && let Some((answers, usage)) = cached(file, &session)
    {
        return Outcome::Ok { answers, usage };
    }
    let mut request = client
        .system_one(session.state.clone(), session.to_questions())
        .model(model.to_owned());
    if let Some(timeout) = timeout {
        request = request.timeout(timeout);
    }
    match request.await {
        Ok(response) => {
            if let Some(file) = &file {
                let body = serde_json::to_string(&response.raw).unwrap_or_default();
                let _ = std::fs::write(file, format!("{body}\n"));
            }
            Outcome::Ok {
                answers: headless::live_answers(&session, &response),
                usage: Some(response.usage.clone()),
            }
        }
        Err(e) => Outcome::Failed {
            error: error_text(&e),
        },
    }
}

/// The same explanation `jev run` prints when a call fails, as one string.
fn error_text(err: &typesafe::Error) -> String {
    jev_repl::format::error_lines(err)
        .iter()
        .map(|line| {
            line.spans
                .iter()
                .map(|span| span.content.as_ref())
                .collect::<String>()
        })
        .collect::<Vec<_>>()
        .join("\n")
        .trim()
        .to_owned()
}

/// A worker's `ask` for a live run: one page's model, through the cache when there is one.
fn live_asker(
    client: Client,
    model: &str,
    options: &Options,
) -> impl Fn(Session) -> std::pin::Pin<Box<dyn Future<Output = Outcome> + Send>>
+ Send
+ Sync
+ Clone
+ 'static {
    let model = model.to_owned();
    let timeout = options.timeout;
    let cache = options.cache.clone();
    move |one: Session| {
        let (client, model, cache) = (client.clone(), model.clone(), cache.clone());
        Box::pin(async move { live_ask(&client, &model, timeout, cache.as_deref(), one).await })
    }
}

/// The live preflight: say what the run is about to cost, refuse it above `--max-cost`, and make
/// the cache directory. `Err` carries the exit status to stop with.
fn before_sending(
    what: &str,
    estimates: &[evaluate::Preflight],
    options: &Options,
) -> Result<(), u8> {
    let input_tokens: usize = estimates.iter().map(|one| one.input_tokens).sum();
    let output_tokens: usize = estimates.iter().map(|one| one.output_tokens).sum();
    let total = options
        .rates
        .map(|rates| cost::price(input_tokens as u64, output_tokens as u64, rates).total);
    let money = match (total, options.rates) {
        (Some(total), Some(rates)) => {
            format!(", ≈ {} at {}", cost::usd(total), cost::format_rates(rates))
        }
        _ => String::new(),
    };
    eprintln!("jev eval: {what}, ≈ {input_tokens} in / {output_tokens} out tokens{money}");
    if let (Some(max), Some(total)) = (options.max_cost, total)
        && total > max
    {
        eprintln!(
            "jev eval: refusing to send: ≈ {} is above --max-cost {}.",
            cost::usd(total),
            cost::usd(max)
        );
        return Err(FAILED);
    }
    if let Some(dir) = &options.cache
        && let Err(e) = std::fs::create_dir_all(dir)
    {
        eprintln!("jev eval: could not use {dir}: {e}");
        return Err(FAILED);
    }
    Ok(())
}

/// What a page is called in a comparison: the path it came from, or stdin.
fn label_of(path: &str) -> String {
    if path == "-" {
        "stdin".to_owned()
    } else {
        path.to_owned()
    }
}

/// `jev eval --compare`: two pages over the same cases, and whether the difference is real.
///
/// Everything that costs money is shared — one preflight over both pages, one pool of workers, one
/// cache — so comparing two pages costs what running them both costs and nothing more.
async fn run_eval_compare(
    a: Session,
    text: &str,
    options: &Options,
    model_a: &str,
    client: Option<Client>,
) -> u8 {
    let second = options.compare.as_deref().unwrap_or("-");
    let (label_a, label_b) = (label_of(&options.file), label_of(second));
    let page = match read_input(second) {
        Ok(page) => page,
        Err(e) => {
            eprintln!("jev eval: {e}");
            return FAILED;
        }
    };
    let mut b = match headless::load(&page) {
        Ok(b) => b,
        Err(e) => {
            eprintln!("jev eval: {label_b}: {e}");
            return FAILED;
        }
    };
    if let Some(model) = &options.model {
        b.model = Some(model.clone());
    }
    let model_b = b
        .model
        .clone()
        .or_else(|| client.as_ref().map(|c| c.default_model().to_owned()))
        .unwrap_or_else(|| "jev-latest".to_owned());

    let labels = evaluate::Labels {
        a: &label_a,
        b: &label_b,
    };
    let (cases_a, cases_b) = match evaluate::parse_compare_cases(text, &a, &b, labels) {
        Ok(both) => both,
        Err(e) => {
            eprintln!("jev eval: {e}");
            return FAILED;
        }
    };

    let live = client.is_some();
    if live {
        let estimates = [
            evaluate::preflight(&a, &cases_a, model_a, None),
            evaluate::preflight(&b, &cases_b, &model_b, None),
        ];
        let what = format!("{} + {} cases over two pages", cases_a.len(), cases_b.len());
        if let Err(code) = before_sending(&what, &estimates, options) {
            return code;
        }
    }

    let (outcomes_a, outcomes_b) = match client {
        None => {
            evaluate::run_compare(
                evaluate::Leg {
                    session: &a,
                    cases: &cases_a,
                    ask: mock_ask,
                },
                evaluate::Leg {
                    session: &b,
                    cases: &cases_b,
                    ask: mock_ask,
                },
                options.concurrency,
            )
            .await
        }
        Some(client) => {
            let ask_a = live_asker(client.clone(), model_a, options);
            let ask_b = live_asker(client, &model_b, options);
            evaluate::run_compare(
                evaluate::Leg {
                    session: &a,
                    cases: &cases_a,
                    ask: ask_a,
                },
                evaluate::Leg {
                    session: &b,
                    cases: &cases_b,
                    ask: ask_b,
                },
                options.concurrency,
            )
            .await
        }
    };
    let comparison = evaluate::compare(
        evaluate::Side {
            label: &label_a,
            session: &a,
            cases: &cases_a,
            outcomes: &outcomes_a,
            model: model_a,
        },
        evaluate::Side {
            label: &label_b,
            session: &b,
            cases: &cases_b,
            outcomes: &outcomes_b,
            model: &model_b,
        },
        evaluate::CompareOptions {
            threshold: options.threshold,
            rates: options.rates,
        },
    );
    if options.json {
        println!(
            "{}",
            serde_json::to_string_pretty(&evaluate::compare_json(&comparison)).unwrap_or_default()
        );
    } else {
        print!("{}", evaluate::compare_text(&comparison));
    }
    if !live {
        eprintln!(
            "Simulated answers: deterministic noise, not judgement. Set TYPESAFE_API_KEY for real ones."
        );
    }

    let sides = [&comparison.a, &comparison.b];
    let mut code = if sides.iter().any(|side| !side.report.errors.is_empty()) {
        FAILED
    } else {
        OK
    };
    if let Some(bar) = options.min_accuracy {
        for side in sides {
            for (name, accuracy) in evaluate::below_bar(&side.report, bar) {
                eprintln!(
                    "jev eval: {}: {name} accuracy {} is below {}.",
                    side.label,
                    evaluate::two(accuracy),
                    evaluate::two(bar)
                );
                code = FAILED;
            }
        }
    }
    if options.fail_on_regression {
        for question in evaluate::regressions(&comparison) {
            eprintln!(
                "jev eval: {} is significantly worse in {label_b} (McNemar p {}).",
                question.name,
                evaluate::three(question.mcnemar.p)
            );
            code = FAILED;
        }
    }
    code
}

/// `jev eval`: the page over a file of labelled states, scored.
///
/// The order matters. Nothing is sent until the cases have parsed and the estimate has been shown,
/// because a cases file is the one input that turns a typo into a bill.
async fn run_eval(
    session: Session,
    page: &str,
    options: &Options,
    model: &str,
    client: Option<Client>,
) -> u8 {
    let text = match read_input(options.cases.as_deref().unwrap_or("-")) {
        Ok(text) => text,
        Err(e) => {
            eprintln!("jev eval: {e}");
            return FAILED;
        }
    };
    if options.compare.is_some() {
        return run_eval_compare(session, &text, options, model, client).await;
    }
    let cases = match evaluate::parse_cases(&text, &session) {
        Ok(cases) => cases,
        Err(e) => {
            eprintln!("jev eval: {e}");
            return FAILED;
        }
    };

    let live = client.is_some();
    if live {
        let estimate = evaluate::preflight(&session, &cases, model, None);
        let n = estimate.cases;
        let what = format!("{n} case{}", if n == 1 { "" } else { "s" });
        if let Err(code) = before_sending(&what, &[estimate], options) {
            return code;
        }
    }

    let outcomes = match client {
        None => evaluate::run(&session, &cases, mock_ask, options.concurrency).await,
        Some(client) => {
            let ask = live_asker(client, model, options);
            evaluate::run(&session, &cases, ask, options.concurrency).await
        }
    };

    let report = evaluate::report(
        &session,
        &cases,
        &outcomes,
        evaluate::ReportOptions {
            model,
            threshold: options.threshold,
            rates: options.rates,
        },
    );
    let mut code = if report.errors.is_empty() { OK } else { FAILED };

    // Calibration writes before it prints, so the report never claims a file it failed to write.
    let calibration = (options.calibrate && report.errors.is_empty()).then(|| {
        evaluate::calibrate(
            &session,
            &cases,
            &outcomes,
            &report,
            options.target_accuracy.unwrap_or(evaluate::DEFAULT_TARGET),
        )
    });
    let mut written = true;
    if let Some(calibration) = &calibration
        && !calibration.changed.is_empty()
        && let Err(e) = std::fs::write(&options.file, sketch::set_bars(page, &calibration.changed))
    {
        eprintln!("jev eval: could not write {}: {e}", options.file);
        written = false;
        code = FAILED;
    }
    let shown = calibration.filter(|_| written);

    if options.json {
        let mut json = evaluate::report_json(&report);
        if let (Some(shown), Value::Object(object)) = (&shown, &mut json) {
            object.insert(
                "calibration".to_owned(),
                Value::Object(evaluate::calibration_json(shown, &options.file)),
            );
        }
        println!(
            "{}",
            serde_json::to_string_pretty(&json).unwrap_or_default()
        );
    } else {
        print!("{}", evaluate::report_text(&report));
        if let Some(shown) = &shown {
            print!("\n{}", evaluate::calibration_text(shown, &options.file));
        }
    }
    if !live {
        eprintln!(
            "Simulated answers: deterministic noise, not judgement. Set TYPESAFE_API_KEY for real ones."
        );
    }
    if options.calibrate && !report.errors.is_empty() {
        eprintln!(
            "jev eval: {}",
            evaluate::not_calibrating(report.errors.len())
        );
    }

    if let Some(bar) = options.min_accuracy {
        for (name, accuracy) in evaluate::below_bar(&report, bar) {
            eprintln!(
                "jev eval: {name} accuracy {} is below {}.",
                evaluate::two(accuracy),
                evaluate::two(bar)
            );
            code = FAILED;
        }
    }
    code
}

/// The REPL itself: the terminal, the event loop, the draw.
async fn repl() -> std::io::Result<()> {
    let mut terminal = ratatui::init();
    // Asked for before the reader thread starts: the query is answered on stdin, and the thread
    // would eat the answer.
    let enhanced = push_key_disambiguation();

    let (tx, mut rx) = mpsc::unbounded_channel();
    spawn_input(tx.clone());
    spawn_ticker(tx.clone());

    let mut app = App::new(tx);
    let mut redraw = true;
    let result = loop {
        if redraw && let Err(e) = terminal.draw(|frame| ui::render(frame, &mut app)) {
            break Err(e);
        }
        let Some(msg) = rx.recv().await else {
            break Ok(());
        };
        // Ticks only matter while the spinner is turning; otherwise an idle REPL redraws nothing.
        redraw = !matches!(msg, Msg::Tick) || app.pending;
        app.handle(msg);
        if app.quit {
            break Ok(());
        }
    };
    if enhanced {
        let _ = execute!(std::io::stdout(), PopKeyboardEnhancementFlags);
    }
    ratatui::restore();
    result
}

/// Ask the terminal to report modified keys unambiguously, and say whether it agreed.
///
/// A terminal that spells Alt as "prefix with Esc" sends `Esc Esc [ A` for Alt-Up, which arrives
/// as an Esc followed by the literal characters `[` and `A` — the arrow is lost and the page gets
/// typed into. Terminals that know the keyboard enhancement protocol report the modifier instead.
/// The ones that do not are left as they were; `Alt-b` / `Alt-f` and Ctrl-←/→ cover them.
fn push_key_disambiguation() -> bool {
    if !matches!(terminal::supports_keyboard_enhancement(), Ok(true)) {
        return false;
    }
    execute!(
        std::io::stdout(),
        PushKeyboardEnhancementFlags(KeyboardEnhancementFlags::DISAMBIGUATE_ESCAPE_CODES)
    )
    .is_ok()
}

/// Terminal events come from a blocking thread so the async side stays free for API calls.
fn spawn_input(tx: mpsc::UnboundedSender<Msg>) {
    std::thread::spawn(move || {
        while let Ok(ev) = event::read() {
            if tx.send(Msg::Term(ev)).is_err() {
                break;
            }
        }
    });
}

/// Drives the spinner while a request is in flight.
fn spawn_ticker(tx: mpsc::UnboundedSender<Msg>) {
    tokio::spawn(async move {
        let mut interval = tokio::time::interval(Duration::from_millis(120));
        loop {
            interval.tick().await;
            if tx.send(Msg::Tick).is_err() {
                break;
            }
        }
    });
}