use std::io::{Read, Write};
use std::path::PathBuf;
use std::str::FromStr;
use anyhow::{Context, Result};
use clap::{Parser, Subcommand};
use llmtrim::bench::{self, BenchCase};
use llmtrim::monitor;
use llmtrim::tracking::{Period, Record, Tracker};
use llmtrim::transport::Endpoint;
use llmtrim::ui::{self, Tone};
use llmtrim_core::config::DenseConfig;
use llmtrim_core::ir::ProviderKind;
const HELP_STYLES: clap::builder::Styles = clap::builder::Styles::styled()
.header(
clap::builder::styling::AnsiColor::BrightBlue
.on_default()
.bold(),
)
.usage(
clap::builder::styling::AnsiColor::BrightBlue
.on_default()
.bold(),
)
.literal(
clap::builder::styling::AnsiColor::BrightCyan
.on_default()
.bold(),
)
.placeholder(clap::builder::styling::AnsiColor::Cyan.on_default());
const HELP_TEMPLATE: &str = "\
{about-with-newline}
{usage-heading} {usage}
Get started:
setup Set everything up and start saving (CA, env, autostart, daemon)
status Show the savings dashboard + interceptor health [aliases: monitor, gain]
Daemon:
start Start the background interceptor (no-op if already running)
stop Stop the background interceptor
autostart Run the interceptor at login (--off to disable)
When something's wrong:
doctor Check the install end-to-end and explain anything broken
update Update llmtrim to the latest release
uninstall Undo everything `setup` did
Pipes & one-shots:
compress Compress a request from stdin to stdout
send Compress a request, send it to the provider, print the response
serve Run the HTTPS interceptor in the foreground
ca Print the local CA certificate path and how to trust it
Measurement (dev):
eval Measure retrieval recall + token savings on a corpus
bench A/B benchmark: tokens saved vs quality retained, on a real model
Options:
{options}
Run `llmtrim help <command>` for details on any command.
";
#[derive(Parser)]
#[command(
name = "llmtrim",
version,
about = "Static, deterministic LLM prompt/payload compressor",
styles = HELP_STYLES,
help_template = HELP_TEMPLATE
)]
struct Cli {
#[command(subcommand)]
command: Commands,
}
#[derive(Subcommand)]
enum Commands {
Compress {
#[arg(long)]
provider: Option<String>,
},
Send {
#[arg(long)]
provider: Option<String>,
},
Serve {
#[arg(long, default_value_t = llmtrim::setup::DEFAULT_PORT)]
port: u16,
#[arg(long, hide = true)]
supervised: bool,
},
Setup {
#[arg(long)]
port: Option<u16>,
},
Uninstall {
#[arg(long)]
purge: bool,
#[arg(long)]
keep_binary: bool,
},
Start {
#[arg(long)]
port: Option<u16>,
},
Stop,
Update,
#[command(name = "status", visible_aliases = ["monitor", "gain"])]
Monitor {
#[arg(long)]
watch: bool,
#[arg(long, default_value_t = 2)]
interval: u64,
#[arg(long)]
daily: bool,
#[arg(long)]
weekly: bool,
#[arg(long)]
monthly: bool,
#[arg(long)]
json: bool,
#[arg(long)]
csv: bool,
#[arg(long, short, conflicts_with_all = ["watch", "daily", "weekly", "monthly", "json", "csv"])]
quiet: bool,
},
Doctor,
Autostart {
#[arg(long)]
off: bool,
#[arg(long, default_value_t = llmtrim::setup::DEFAULT_PORT)]
port: u16,
},
Ca {
#[arg(long)]
pem: bool,
},
Eval {
#[arg(long)]
corpus: PathBuf,
#[arg(long, default_value = "openai")]
provider: String,
#[arg(long, default_value_t = 0.5)]
keep_ratio: f64,
},
Bench(Box<BenchArgs>),
BenchAgent(Box<BenchAgentArgs>),
}
#[derive(clap::Args)]
struct BenchAgentArgs {
#[arg(long = "tasks", default_value = "bench/agent")]
tasks: Vec<PathBuf>,
#[arg(long, default_value = "baseline,agent")]
conditions: String,
#[arg(long, default_value_t = 1)]
repeats: usize,
#[arg(long)]
model: Option<String>,
#[arg(long, default_value = "bench/pricing.json")]
pricing: PathBuf,
#[arg(long)]
live: bool,
#[arg(long, default_value_t = 2)]
tool_turns: usize,
#[arg(long)]
json_out: Option<PathBuf>,
}
#[derive(clap::Args)]
struct BenchArgs {
#[arg(long)]
corpus: PathBuf,
#[arg(long, default_value = "openai")]
provider: String,
#[arg(long, default_value = "auto")]
preset: String,
#[arg(long, default_value = "openai/gpt-oss-20b")]
model: String,
#[arg(long, default_value = "openai/gpt-4o-mini")]
judge_model: String,
#[arg(long, default_value = "groq")]
route: String,
#[arg(long)]
n: Option<usize>,
#[arg(long)]
config: Option<PathBuf>,
#[arg(long, default_value = "bench/pricing.json")]
pricing: PathBuf,
#[arg(long)]
offline: bool,
#[arg(long)]
ablate: bool,
#[arg(long)]
json_out: Option<PathBuf>,
}
fn read_stdin() -> Result<String> {
let mut buf = String::new();
std::io::stdin()
.read_to_string(&mut buf)
.context("failed to read stdin")?;
Ok(buf)
}
fn main() {
if let Err(e) = run() {
eprint!("{}", ui::render_error(ui::color_stderr(), &e));
std::process::exit(1);
}
}
fn run() -> Result<()> {
let cli = Cli::parse();
match cli.command {
Commands::Compress { provider } => {
let input = read_stdin()?;
let kind = provider
.as_deref()
.map(ProviderKind::from_str)
.transpose()?;
let result = llmtrim_core::compress(&input, kind)?;
if let Ok(tracker) = Tracker::open() {
let _ = tracker.record(&Record {
provider: result.provider.as_str().to_string(),
model: result.model.clone(),
tokenizer: result.tokenizer_label.clone(),
exact: result.tokenizer_exact,
input_before: result.input_tokens_before.0 as i64,
input_after: result.input_tokens_after.0 as i64,
output_before: None,
output_after: None,
compress_micros: None,
cache_read_tokens: None,
fresh_input_tokens: None,
cache_write_tokens: None,
output_shaped: Some(result.output_shaped),
frozen_input_tokens: Some(result.frozen_input_tokens.0 as i64),
});
}
let mut stdout = std::io::stdout().lock();
stdout
.write_all(result.request_json.as_bytes())
.context("failed to write stdout")?;
let _ = stdout.write_all(b"\n");
}
Commands::Send { provider } => {
let input = read_stdin()?;
let kind = provider
.as_deref()
.map(ProviderKind::from_str)
.transpose()?;
let result = llmtrim_core::compress(&input, kind)?;
let endpoint = Endpoint::from_env(result.provider)?;
let response = endpoint.send(&result.request_json)?;
if let Ok(counter) =
llmtrim_core::tokenizer::counter_for(result.provider, result.model.as_deref())
{
let output_after = serde_json::from_str::<serde_json::Value>(&response)
.ok()
.and_then(|v| llmtrim_core::provider::for_kind(result.provider).answer_text(&v))
.map(|text| counter.count(&text) as i64);
if let Ok(tracker) = Tracker::open() {
let _ = tracker.record(&Record {
provider: result.provider.as_str().to_string(),
model: result.model.clone(),
tokenizer: counter.label().to_string(),
exact: counter.is_exact(),
input_before: result.input_tokens_before.0 as i64,
input_after: result.input_tokens_after.0 as i64,
output_before: None,
output_after,
compress_micros: None,
cache_read_tokens: None,
fresh_input_tokens: None,
cache_write_tokens: None,
output_shaped: Some(result.output_shaped),
frozen_input_tokens: Some(result.frozen_input_tokens.0 as i64),
});
}
}
println!("{response}");
}
Commands::Serve { port, supervised } => {
if supervised {
llmtrim::serve::run_supervised(port)?;
} else {
llmtrim::serve::run(port)?;
}
}
Commands::Setup { port } => llmtrim::setup::run(port)?,
Commands::Uninstall { purge, keep_binary } => {
llmtrim::setup::uninstall(purge, keep_binary)?
}
Commands::Start { port } => {
let color = ui::color_stdout();
if let Some(state) = llmtrim::daemon::running() {
println!(
"{}",
ui::ok(
color,
&format!(
"Interceptor already running · pid {} · port {}",
state.pid, state.port
)
)
);
} else {
let port = llmtrim::setup::resolve_port(port, None)?;
let pid = llmtrim::daemon::spawn_detached(port)?;
println!(
"{}",
ui::ok(
color,
&format!("Interceptor running · pid {pid} · port {port}")
)
);
for (label, value) in [
("watch", "llmtrim status".to_string()),
("stop", "llmtrim stop".to_string()),
] {
println!(
" {} {value}",
ui::paint(color, Tone::Dim, &format!("{label:<5}"))
);
}
if !llmtrim::setup::profile_has_block() {
eprintln!(
"{}",
ui::note(
ui::color_stderr(),
"HTTPS_PROXY isn't set for your user yet — run `llmtrim setup` so \
tools actually route through the interceptor."
)
);
}
}
}
Commands::Stop => match llmtrim::daemon::stop()? {
Some(pid) => {
println!(
"{}",
ui::ok(
ui::color_stdout(),
&format!("Stopped interceptor (pid {pid}).")
)
);
if llmtrim::setup::profile_has_block() {
eprintln!(
"{}",
ui::warn(
ui::color_stderr(),
"HTTPS_PROXY still points at llmtrim in your environment — \
new HTTPS to LLM hosts will fail until you start it again \
(llmtrim start) or run llmtrim uninstall."
)
);
}
}
None => println!(
"{}",
ui::note(ui::color_stdout(), "No interceptor daemon was running.")
),
},
Commands::Update => llmtrim::update::run()?,
Commands::Monitor {
watch,
interval,
daily,
weekly,
monthly,
json,
csv,
quiet,
} => run_monitor(watch, interval, daily, weekly, monthly, json, csv, quiet)?,
Commands::Doctor => {
let report = llmtrim::doctor::gather();
print!("{}", llmtrim::doctor::render(ui::color_stdout(), &report));
if report.problems > 0 {
std::process::exit(1);
}
}
Commands::Autostart { off, port } => {
llmtrim::autostart::configure(!off, port)?;
let color = ui::color_stdout();
if off {
println!("{}", ui::ok(color, "Autostart disabled."));
} else {
println!(
"{}",
ui::ok(
color,
&format!("Autostart enabled — llmtrim serve --port {port} runs at login.")
)
);
println!(
" {} llmtrim autostart --off",
ui::paint(color, Tone::Dim, "undo ")
);
}
}
Commands::Ca { pem } => {
let path = llmtrim::serve::ca_cert_path()?;
llmtrim::serve::ensure_ca()?; if pem {
print!("{}", std::fs::read_to_string(&path)?);
return Ok(());
}
let color = ui::color_stdout();
print!(
"{}",
ui::panel(color, "llmtrim local CA", &[path.display().to_string()])
);
println!();
println!("Trust it for your tool, then route its traffic through llmtrim:");
#[cfg(windows)]
{
println!(" $env:NODE_EXTRA_CA_CERTS = \"{}\"", path.display());
println!(
" $env:HTTPS_PROXY = \"http://127.0.0.1:{}\"",
llmtrim::setup::DEFAULT_PORT
);
println!(" llmtrim serve");
println!();
println!("System-wide trust (non-PowerShell / GUI apps):");
println!(" certutil -addstore -user Root \"{}\"", path.display());
}
#[cfg(not(windows))]
{
println!(" export NODE_EXTRA_CA_CERTS={}", path.display());
println!(
" export HTTPS_PROXY=http://127.0.0.1:{}",
llmtrim::setup::DEFAULT_PORT
);
println!(" llmtrim serve");
}
println!();
println!(
"{}",
ui::note(
ui::color_stdout(),
"The CA is name-constrained to LLM API domains only."
)
);
}
Commands::Eval {
corpus,
provider,
keep_ratio,
} => {
let kind = ProviderKind::from_str(&provider)?;
let jsonl = std::fs::read_to_string(&corpus)
.with_context(|| format!("failed to read corpus {}", corpus.display()))?;
let cases = llmtrim::quality::load_corpus(&jsonl, kind)?;
let keep_ratio = keep_ratio.clamp(0.0, 1.0);
let config = llmtrim_core::config::DenseConfig {
retrieve: true,
retrieve_keep_ratio: keep_ratio,
retrieve_min_segment_chars: 120,
output_control: false,
..Default::default()
};
let results = llmtrim::quality::run_recall(&cases, &config)?;
let color = ui::color_stdout();
let saved_cell = |pct: f64| {
if pct > 0.0 {
ui::paint(color, Tone::Accent, &format!("-{pct:.1}%"))
} else {
ui::paint(color, Tone::Dim, &format!("{:.1}%", pct.abs()))
}
};
let mut t = ui::table(color, &["case", "recall", "tokens", "saved"]);
for r in &results {
t.add_row(vec![
comfy_table::Cell::new(&r.name),
ui::right(format!("{:.2}", r.recall)),
ui::right(format!("{} → {}", r.tokens_before, r.tokens_after)),
ui::right(saved_cell(r.savings_pct())),
]);
}
for line in t.to_string().lines() {
println!(" {line}");
}
let recall = llmtrim::quality::mean_recall(&results);
let savings = if results.is_empty() {
0.0
} else {
results.iter().map(|r| r.savings_pct()).sum::<f64>() / results.len() as f64
};
print!(
"\n{}",
ui::panel(
color,
"eval",
&[format!(
"{} cases mean recall {} mean savings {} keep_ratio {keep_ratio}",
results.len(),
ui::paint(color, Tone::Bold, &format!("{recall:.2}")),
saved_cell(savings),
)]
)
);
}
Commands::Bench(args) => run_bench(*args)?,
Commands::BenchAgent(args) => run_bench_agent(*args)?,
}
Ok(())
}
#[cfg(feature = "live")]
fn dotenv_get(key: &str) -> Option<String> {
if let Ok(v) = std::env::var(key) {
return Some(v);
}
dotenvy::from_path_iter(".env")
.ok()?
.flatten()
.find(|(k, _)| k == key)
.map(|(_, v)| v)
}
fn prepare_request(request_json: &str, model: &str, route: &str) -> String {
match serde_json::from_str::<serde_json::Value>(request_json) {
Ok(mut v) => {
if let Some(obj) = v.as_object_mut() {
obj.insert("model".to_string(), serde_json::json!(model));
if !route.is_empty() {
let (provider, quant) = match route.split_once('/') {
Some((p, q)) => (p, Some(q)),
None => (route, None),
};
let mut routing =
serde_json::json!({"order": [provider], "allow_fallbacks": false});
if let Some(q) = quant {
routing["quantizations"] = serde_json::json!([q]);
}
obj.insert("provider".to_string(), routing);
}
obj.insert("usage".to_string(), serde_json::json!({"include": true}));
}
v.to_string()
}
Err(_) => request_json.to_string(),
}
}
fn run_bench(args: BenchArgs) -> Result<()> {
let kind = ProviderKind::from_str(&args.provider)?;
let jsonl = std::fs::read_to_string(&args.corpus)
.with_context(|| format!("failed to read corpus {}", args.corpus.display()))?;
let mut cases = bench::load_bench_corpus(&jsonl, kind, &args.model)?;
if let Some(limit) = args.n {
cases.truncate(limit);
}
for c in &mut cases {
c.request = prepare_request(&c.request, &args.model, &args.route);
}
let mut config = match &args.config {
Some(path) => {
let t = std::fs::read_to_string(path)
.with_context(|| format!("failed to read config {}", path.display()))?;
toml::from_str::<DenseConfig>(&t)
.with_context(|| format!("failed to parse config {}", path.display()))?
}
None => DenseConfig::preset(&args.preset).with_context(|| {
format!(
"unknown preset '{}' (auto|safe|lossless|rag|agent|code|aggressive|cache|reasoning)",
args.preset
)
})?,
};
config.retrieve_keep_ratio = config.retrieve_keep_ratio.clamp(0.0, 1.0);
config.retrieve_mmr_lambda = config.retrieve_mmr_lambda.clamp(0.0, 1.0);
if args.ablate {
run_ablation(&cases, &config, &args.preset)
} else if args.offline {
run_offline(&cases, kind, &config, &args.preset)
} else {
run_live(&cases, kind, &config, &args)
}
}
fn run_bench_agent(args: BenchAgentArgs) -> Result<()> {
use llmtrim::bench::agent::Condition;
let mut tasks = load_agent_tasks(&args.tasks)?;
if tasks.is_empty() {
anyhow::bail!("no agent tasks found under {:?}", args.tasks);
}
if let Some(m) = &args.model {
for t in &mut tasks {
t.model = m.clone();
}
}
let conditions: Vec<Condition> = args
.conditions
.split(',')
.map(|s| Condition::parse(s.trim()))
.collect();
let table = std::fs::read_to_string(&args.pricing)
.map(|s| bench::load_pricing(&s))
.unwrap_or_default();
if args.live {
run_agent_live(&tasks, &conditions, &args, &table)
} else {
run_agent_dry(&tasks, &conditions, &args, &table)
}
}
fn load_agent_tasks(paths: &[PathBuf]) -> Result<Vec<llmtrim::bench::agent::AgentTask>> {
use llmtrim::bench::agent::AgentTask;
let mut files = Vec::new();
for p in paths {
if p.is_dir() {
for entry in
std::fs::read_dir(p).with_context(|| format!("read dir {}", p.display()))?
{
let path = entry?.path();
if path.extension().and_then(|e| e.to_str()) == Some("json") {
files.push(path);
}
}
} else {
files.push(p.clone());
}
}
files.sort();
let mut tasks = Vec::new();
for f in files {
let s = std::fs::read_to_string(&f).with_context(|| format!("read {}", f.display()))?;
tasks.push(AgentTask::from_json(&s).with_context(|| format!("parse {}", f.display()))?);
}
Ok(tasks)
}
fn run_agent_dry(
tasks: &[llmtrim::bench::agent::AgentTask],
conditions: &[llmtrim::bench::agent::Condition],
args: &BenchAgentArgs,
table: &bench::PriceTable,
) -> Result<()> {
use llmtrim::bench::agent::{OpenAiAgent, dry_run_transport, run_agent_loop};
let provider = OpenAiAgent;
let mut results = Vec::new();
for task in tasks {
let price = bench::resolve_pricing(table, &task.model);
for cond in conditions {
for _ in 0..args.repeats.max(1) {
let mut send = dry_run_transport(args.tool_turns);
results.push(run_agent_loop(&provider, task, cond, &price, &mut send)?);
}
}
}
print_agent_results(&results, true);
write_agent_json(&results, &args.json_out)
}
#[cfg(feature = "live")]
fn run_agent_live(
tasks: &[llmtrim::bench::agent::AgentTask],
conditions: &[llmtrim::bench::agent::Condition],
args: &BenchAgentArgs,
table: &bench::PriceTable,
) -> Result<()> {
use llmtrim::bench::agent::{OpenAiAgent, run_agent_loop};
let api_key =
dotenv_get("OPENROUTER_API_KEY").context("OPENROUTER_API_KEY not set (env or a .env)")?;
let model = llmtrim::quality::OpenRouterModel::new(api_key, ProviderKind::OpenAi)?;
let provider = OpenAiAgent;
let mut results = Vec::new();
for task in tasks {
let price = bench::resolve_pricing(table, &task.model);
for cond in conditions {
for _ in 0..args.repeats.max(1) {
let mut send = |req: &str| model.send_raw(req);
results.push(run_agent_loop(&provider, task, cond, &price, &mut send)?);
}
}
}
print_agent_results(&results, false);
write_agent_json(&results, &args.json_out)
}
#[cfg(not(feature = "live"))]
fn run_agent_live(
_tasks: &[llmtrim::bench::agent::AgentTask],
_conditions: &[llmtrim::bench::agent::Condition],
_args: &BenchAgentArgs,
_table: &bench::PriceTable,
) -> Result<()> {
anyhow::bail!(
"--live needs the `live` feature: rebuild with `cargo run --features live -- bench-agent --live …`"
)
}
fn print_agent_results(results: &[llmtrim::bench::agent::AgentRunResult], dry: bool) {
let color = ui::color_stdout();
let mut t = ui::table(
color,
&[
"task",
"condition",
"iters",
"input",
"cached",
"output",
"cost$",
"done",
],
);
for r in results {
t.add_row(vec![
r.task_id.clone(),
r.condition.clone(),
r.iterations.to_string(),
r.input_tokens.to_string(),
r.cached_tokens.to_string(),
r.output_tokens.to_string(),
format!("{:.4}", r.cost_usd),
if r.completed { "yes" } else { "no" }.to_string(),
]);
}
println!("{t}");
if dry {
println!(
"(dry-run: synthetic usage, cached estimated by prefix overlap — no API calls; use --live for real numbers)"
);
}
}
fn write_agent_json(
results: &[llmtrim::bench::agent::AgentRunResult],
out: &Option<PathBuf>,
) -> Result<()> {
if let Some(path) = out {
let json = serde_json::to_string_pretty(results).context("serialize agent results")?;
std::fs::write(path, json).with_context(|| format!("write {}", path.display()))?;
}
Ok(())
}
fn run_ablation(cases: &[BenchCase], config: &DenseConfig, preset: &str) -> Result<()> {
let configs = bench::ablation_configs(config);
let rows = bench::run_token_ablation(cases, &configs)?;
let full_after = rows.first().map(|(_, _, a)| *a).unwrap_or(0) as f64;
let color = ui::color_stdout();
println!(
"{}",
ui::paint(
color,
Tone::Bold,
&format!(
"ablation — input tokens, offline, preset={preset}, {} cases",
cases.len()
)
)
);
let mut t = ui::table(
color,
&["config", "before", "after", "saved", "stage saves"],
);
for (label, before, after) in &rows {
let saved = ui::saved_pct(*before as f64, *after as f64);
let contribution = if label == "full" {
String::new()
} else {
format!("{:+.0} tok", *after as f64 - full_after)
};
t.add_row(vec![
comfy_table::Cell::new(label),
ui::right(before.to_string()),
ui::right(after.to_string()),
ui::right(ui::paint(color, Tone::Accent, &format!("{saved:.1}%"))),
ui::right(contribution),
]);
}
for line in t.to_string().lines() {
println!(" {line}");
}
Ok(())
}
fn run_offline(
cases: &[BenchCase],
kind: ProviderKind,
config: &DenseConfig,
preset: &str,
) -> Result<()> {
let (mut before, mut after) = (0usize, 0usize);
for c in cases {
let r = llmtrim_core::compress_with_config(&c.request, Some(kind), config)?;
before += r.input_tokens_before.0;
after += r.input_tokens_after.0;
}
println!(
"offline: {} cases input {before} -> {after} tok ({:.1}% saved) preset={preset}",
cases.len(),
ui::saved_pct(before as f64, after as f64),
);
Ok(())
}
#[cfg(feature = "live")]
fn run_live(
cases: &[BenchCase],
kind: ProviderKind,
config: &DenseConfig,
args: &BenchArgs,
) -> Result<()> {
if kind != ProviderKind::OpenAi {
anyhow::bail!(
"live A/B bench only supports --provider openai (the OpenRouter endpoint is \
OpenAI-shaped; a {} request would be sent OpenAI-style and mis-parsed, scoring \
nothing). Use --offline or --ablate to measure {}-shaped input-token savings, \
or re-run with --provider openai.",
args.provider,
args.provider
);
}
if args.judge_model == args.model
&& cases
.iter()
.any(|c| matches!(c.scorer, bench::Scorer::LlmJudge))
{
eprintln!(
"{}",
ui::warn(
ui::color_stderr(),
&format!(
"--judge-model == --model ({}) — the model is grading its own answers; \
pass a different --judge-model for an unbiased judge.",
args.model
)
)
);
}
let table = std::fs::read_to_string(&args.pricing)
.map(|s| bench::load_pricing(&s))
.unwrap_or_default();
let price = bench::resolve_pricing(&table, &args.model);
let counter = llmtrim_core::tokenizer::counter_for(kind, Some(&args.model))?;
let api_key = dotenv_get("OPENROUTER_API_KEY")
.context("OPENROUTER_API_KEY not set (in env or a local .env)")?;
let llm = llmtrim::quality::OpenRouterModel::new(api_key, kind)?;
let scorer = bench::BenchScorer {
exec_timeout: 10,
judge: Some(&llm),
judge_model: args.judge_model.clone(),
};
let run = bench::run_ab(cases, config, &llm, counter.as_ref(), &scorer, price)?;
let color = ui::color_stdout();
let mut t = ui::table(color, &["case", "quality", "input", "output"]);
for o in &run.outcomes {
let q_tone = if o.quality_comp < o.quality_orig {
Tone::Warn
} else {
Tone::Accent
};
t.add_row(vec![
comfy_table::Cell::new(ui::truncate(&o.name, 24)),
ui::right(ui::paint(
color,
q_tone,
&format!("{:.2} → {:.2}", o.quality_orig, o.quality_comp),
)),
ui::right(format!("{} → {}", o.tokens_in_before, o.tokens_in_after)),
ui::right(format!("{} → {}", o.tokens_out_orig, o.tokens_out_comp)),
]);
}
for line in t.to_string().lines() {
println!(" {line}");
}
let f = bench::summarize(&run);
let cache_note = if f.cache_busted {
"cache busted (per-arm nonce; cache stage off)"
} else {
"cache stage under test"
};
let metric = |v: f64| ui::paint(color, Tone::Accent, &format!("{v:.1}%"));
let lines = vec![
ui::paint(
color,
Tone::Dim,
&format!(
"{} · model {} · judge {}",
args.corpus.display(),
args.model,
args.judge_model
),
),
String::new(),
format!("input saved {}", metric(f.tokens_in_saved_pct)),
format!("output saved {}", metric(f.tokens_out_saved_pct)),
format!("cost saved {}", metric(f.cost_saved_pct)),
format!("cache used {:.1}% ({cache_note})", f.cache_used_pct),
format!(
"quality {} → {} (retention {:+.1}pp, paired 95%CI ±{:.1})",
ui::paint(
color,
Tone::Bold,
&format!("{:.1}%", f.quality_orig.mean * 100.0)
),
ui::paint(
color,
Tone::Bold,
&format!("{:.1}%", f.quality_comp.mean * 100.0)
),
f.retention_pp,
f.retention_ci95_pp,
),
format!(
"cases {} scored, {} failed (compressed 4xx → 0), {} skipped (transient)",
f.n, f.failed, f.skipped
),
];
print!(
"\n{}",
ui::panel(
color,
&format!("bench · {} · n={}", args.preset, f.n),
&lines
)
);
if let Some(path) = &args.json_out {
let rows: Vec<_> = run
.outcomes
.iter()
.map(|o| {
serde_json::json!({
"name": o.name,
"quality_orig": o.quality_orig, "quality_comp": o.quality_comp,
"tokens_in_before": o.tokens_in_before, "tokens_in_after": o.tokens_in_after,
"tokens_out_orig": o.tokens_out_orig, "tokens_out_comp": o.tokens_out_comp,
"cached_in_orig": o.cached_in_orig, "cached_in_comp": o.cached_in_comp,
"cost_orig": o.cost_orig, "cost_comp": o.cost_comp,
})
})
.collect();
let doc = serde_json::json!({
"preset": args.preset,
"config_path": args.config.as_ref().map(|p| p.display().to_string()),
"config_resolved": config,
"model": args.model,
"judge_model": args.judge_model,
"route": args.route,
"corpus": args.corpus.display().to_string(),
"n": f.n,
"failed": f.failed,
"skipped": f.skipped,
"cache_busted": f.cache_busted,
"tokens_in_saved_pct": f.tokens_in_saved_pct,
"tokens_out_saved_pct": f.tokens_out_saved_pct,
"cost_saved_pct": f.cost_saved_pct,
"cache_used_pct": f.cache_used_pct,
"quality_orig": f.quality_orig.mean, "quality_comp": f.quality_comp.mean,
"retention_pp": f.retention_pp,
"retention_ci95_pp": f.retention_ci95_pp,
"cases": rows,
});
std::fs::write(path, serde_json::to_string_pretty(&doc)?)
.with_context(|| format!("failed to write {}", path.display()))?;
eprintln!(
"{}",
ui::note(ui::color_stderr(), &format!("wrote {}", path.display()))
);
}
Ok(())
}
#[cfg(not(feature = "live"))]
fn run_live(
_cases: &[BenchCase],
_kind: ProviderKind,
_config: &DenseConfig,
_args: &BenchArgs,
) -> Result<()> {
anyhow::bail!(
"live A/B bench requires the `live` feature: rebuild with \
`cargo run --features live -- bench …`. Use `--offline` (or `--ablate`) for \
token-only measurement, which needs no extra feature."
)
}
fn period_flag(daily: bool, weekly: bool, monthly: bool) -> Option<Period> {
if daily {
Some(Period::Day)
} else if weekly {
Some(Period::Week)
} else if monthly {
Some(Period::Month)
} else {
None
}
}
fn daemon_view(summary: &llmtrim::tracking::Summary) -> monitor::DaemonView {
use llmtrim::daemon;
let ca_present = matches!(llmtrim::serve::ca_cert_path(), Ok(p) if p.exists());
let env_port = llmtrim::setup::configured_port();
let autostart = llmtrim::autostart::is_enabled();
let log_path = daemon::logfile().ok().map(|p| p.display().to_string());
let last_request = summary.last_ts.as_deref().and_then(daemon::human_age);
let binary_version = env!("CARGO_PKG_VERSION").to_string();
match daemon::running() {
Some(s) => monitor::DaemonView {
running: true,
pid: s.pid,
port: s.port,
uptime: daemon::human_uptime(daemon::uptime_secs(s.started_at)),
uptime_secs: daemon::uptime_secs(s.started_at),
ca_present,
port_accepting: daemon::probe_port(s.port),
env_port,
autostart,
restarts: s.restarts,
version: s.version,
binary_version,
log_path,
last_request,
},
None => monitor::DaemonView {
running: false,
pid: 0,
port: 0,
uptime: String::new(),
uptime_secs: 0,
ca_present,
port_accepting: false,
env_port,
autostart,
restarts: 0,
version: None,
binary_version,
log_path,
last_request,
},
}
}
fn monitor_cost(tracker: &Tracker) -> Option<monitor::Cost> {
let models = tracker.by_model().ok()?;
cost_estimate(&models)
}
fn model_views(tracker: &Tracker) -> Result<Vec<monitor::ModelView>> {
let mut models: Vec<monitor::ModelView> = tracker
.by_model()?
.into_iter()
.filter(|m| m.events > 0)
.map(|m| {
let priced = m.model.as_deref().and_then(llm_prices);
let new_before = m.metered_input_before - m.frozen_input_tokens;
let new_pct = (m.frozen_input_tokens > 0 && new_before > 0).then(|| {
ui::saved_pct(
new_before as f64,
(m.metered_input_after - m.frozen_input_tokens) as f64,
)
});
let cost_saved = priced
.map(|(inp, _)| (m.input_before - m.input_after).max(0) as f64 / 1_000_000.0 * inp);
let out_spend = priced.map(|(_, outp)| m.output_after as f64 / 1_000_000.0 * outp);
let spend = priced.map(|(inp, outp)| {
m.input_after as f64 / 1_000_000.0 * inp
+ m.output_after as f64 / 1_000_000.0 * outp
});
monitor::ModelView {
name: m
.model
.unwrap_or_else(|| format!("{} · unknown model", m.provider)),
events: m.events,
saved_pct: ui::saved_pct(m.input_before as f64, m.input_after as f64),
cost_saved,
spend,
out_spend,
cached: m.cache_read > 0,
new_pct,
}
})
.collect();
models.sort_unstable_by_key(|b| std::cmp::Reverse(b.events));
models.truncate(8);
Ok(models)
}
fn today_saved_usd(tracker: &Tracker) -> Option<f64> {
let models = tracker.by_model_today().ok()?;
cost_estimate(&models).map(|c| c.saved)
}
fn render_snapshot(tracker: &Tracker, color: bool) -> Result<(String, monitor::Health)> {
let summary = tracker.summary()?;
let models = model_views(tracker)?;
let cost = monitor_cost(tracker);
let daemon = daemon_view(&summary);
let health = monitor::health(&daemon);
let mut out = monitor::snapshot(
color,
Some(&daemon),
&summary,
&models,
cost.as_ref(),
today_saved_usd(tracker),
);
if let Some(v) = llmtrim::update::check(false) {
out.push_str(&format!(
"\n {} llmtrim v{v} available — run llmtrim update\n",
ui::paint(color, Tone::Accent, "↑")
));
}
Ok((out, health))
}
struct AltScreenGuard;
impl Drop for AltScreenGuard {
fn drop(&mut self) {
let mut out = std::io::stdout();
let _ = out.write_all(b"\x1b[?1049l");
let _ = out.flush();
}
}
fn run_watch(tracker: &Tracker, interval: u64) -> Result<()> {
let color = ui::color_stdout();
let tty = ui::stdout_is_tty();
let _alt = if tty {
let mut out = std::io::stdout();
let _ = out.write_all(b"\x1b[?1049h\x1b[H");
let _ = out.flush();
let _ = ctrlc::set_handler(|| {
let mut out = std::io::stdout();
let _ = out.write_all(b"\x1b[?1049l");
let _ = out.flush();
std::process::exit(0);
});
Some(AltScreenGuard)
} else {
None
};
let mut prev: Option<i64> = None;
loop {
let summary = tracker.summary()?;
let mut body = render_snapshot(tracker, color)?.0;
if let Some(p) = prev {
let rate = (summary.saved() - p) as f64 / interval as f64;
if rate.abs() >= 0.5 {
body.push_str(&format!("\n {rate:+.0} input tokens/s saved\n"));
}
}
prev = Some(summary.saved());
body.push_str(&ui::paint(
color,
Tone::Dim,
&format!(" refreshing every {interval}s · Ctrl-C to exit\n"),
));
let frame = if tty {
format!(
"\x1b[?2026h\x1b[H{}\x1b[0J\x1b[?2026l",
body.replace('\n', "\x1b[K\n")
)
} else {
body
};
let mut stdout = std::io::stdout();
if stdout.write_all(frame.as_bytes()).is_err() {
return Ok(());
}
stdout.flush().ok();
std::thread::sleep(std::time::Duration::from_secs(interval));
}
}
#[allow(clippy::too_many_arguments)]
fn run_monitor(
watch: bool,
interval: u64,
daily: bool,
weekly: bool,
monthly: bool,
json: bool,
csv: bool,
quiet: bool,
) -> Result<()> {
let tracker = Tracker::open().context("failed to open savings ledger")?;
if quiet {
let summary = tracker.summary()?;
let health = monitor::health(&daemon_view(&summary));
println!("{}", health.label());
std::process::exit(health.exit_code());
}
if let Some(period) = period_flag(daily, weekly, monthly) {
let rows = tracker.by_period(period)?;
if csv {
print!("{}", monitor::export_csv(&rows));
} else if json {
let s = tracker.summary()?;
let models = model_views(&tracker)?;
let daemon = daemon_view(&s);
println!(
"{}",
monitor::export_json(
&s,
&models,
monitor_cost(&tracker).as_ref(),
&rows,
Some(&daemon)
)
);
} else {
print!(
"{}",
monitor::period_report(ui::color_stdout(), period.label(), &rows)
);
}
return Ok(());
}
if json || csv {
let rows = tracker.by_period(Period::Day)?;
if csv {
print!("{}", monitor::export_csv(&rows));
} else {
let s = tracker.summary()?;
let models = model_views(&tracker)?;
let daemon = daemon_view(&s);
println!(
"{}",
monitor::export_json(
&s,
&models,
monitor_cost(&tracker).as_ref(),
&rows,
Some(&daemon)
)
);
}
return Ok(());
}
if watch {
run_watch(&tracker, interval.max(1))
} else {
let (out, health) = render_snapshot(&tracker, ui::color_stdout())?;
print!("{out}");
if health != monitor::Health::Healthy {
use std::io::Write as _;
let _ = std::io::stdout().flush();
std::process::exit(health.exit_code());
}
Ok(())
}
}
fn cache_multipliers(provider: &str) -> (f64, f64) {
match provider {
"anthropic" => (0.10, 1.25),
"openai" => (0.50, 0.0),
"google" => (0.25, 0.0),
_ => (1.0, 0.0),
}
}
fn cost_estimate(models: &[llmtrim::tracking::ModelRow]) -> Option<monitor::Cost> {
let mut cost = monitor::Cost {
saved: 0.0,
spend: 0.0,
out_spend: 0.0,
net_saved: 0.0,
out_spend_shaped: 0.0,
live_saved: 0.0,
};
let mut matched = false;
for m in models {
let Some(model_id) = m.model.as_deref() else {
continue;
};
if let Some((input_price, output_price)) = llm_prices(model_id) {
let delta = (m.input_before - m.input_after).max(0) as f64;
cost.saved += delta / 1_000_000.0 * input_price;
let out = m.output_after as f64 / 1_000_000.0 * output_price;
cost.spend += m.input_after as f64 / 1_000_000.0 * input_price + out;
cost.out_spend += out;
cost.out_spend_shaped += m.output_after_shaped as f64 / 1_000_000.0 * output_price;
let (read_mult, write_mult) = cache_multipliers(&m.provider);
let net_bill = (m.fresh_input_est as f64
+ m.cache_write as f64 * write_mult
+ m.cache_read as f64 * read_mult)
/ 1_000_000.0
* input_price;
let pct = if m.input_before > 0 {
(delta / m.input_before as f64).min(0.95)
} else {
0.0
};
cost.net_saved += net_bill * pct / (1.0 - pct);
let live_used = m.fresh_input_est + m.cache_write;
let live_rate = if live_used > 0 {
(m.fresh_input_est as f64 + m.cache_write as f64 * write_mult.max(1.0))
/ live_used as f64
} else {
1.0
};
cost.live_saved += delta / 1_000_000.0 * input_price * live_rate;
matched = true;
}
}
matched.then_some(cost)
}
fn llm_prices(model_id: &str) -> Option<(f64, f64)> {
#[cfg(feature = "intercept")]
for &provider_id in llm_providers::get_providers_data().keys() {
if let Some(model) = llm_providers::get_model_ref(provider_id, model_id) {
return Some((model.input_price, model.output_price));
}
}
snapshot_prices(model_id)
}
fn snapshot_prices(model_id: &str) -> Option<(f64, f64)> {
static TABLE: once_cell::sync::Lazy<bench::PriceTable> =
once_cell::sync::Lazy::new(|| bench::load_pricing(include_str!("../bench/pricing.json")));
let p = TABLE.get(model_id).or_else(|| {
let (_, bare) = model_id.split_once('/')?;
TABLE.get(bare)
})?;
let (input, output) = (p.input_per_1k * 1000.0, p.output_per_1k * 1000.0);
(input > 0.0 || output > 0.0).then_some((input, output))
}