use std::io::{Read, Write};
use std::path::PathBuf;
use std::str::FromStr;
use anyhow::{Context, Result, bail};
use clap::{Parser, Subcommand};
use llmtrim::bench::{self, BenchCase};
use llmtrim::monitor;
use llmtrim::tracking::{Period, Record, Tracker};
use llmtrim::transport::Endpoint;
use llmtrim::ui::{self, Tone};
use llmtrim_core::config::DenseConfig;
use llmtrim_core::ir::ProviderKind;
const HELP_STYLES: clap::builder::Styles = clap::builder::Styles::styled()
.header(
clap::builder::styling::AnsiColor::BrightBlue
.on_default()
.bold(),
)
.usage(
clap::builder::styling::AnsiColor::BrightBlue
.on_default()
.bold(),
)
.literal(
clap::builder::styling::AnsiColor::BrightCyan
.on_default()
.bold(),
)
.placeholder(clap::builder::styling::AnsiColor::Cyan.on_default());
const HELP_TEMPLATE: &str = "\
{about-with-newline}
{usage-heading} {usage}
Get started:
setup Set everything up and start saving (CA, env, autostart, daemon)
status Show the savings dashboard + interceptor health [aliases: monitor, gain]
wrap Launch an agent (claude, codex, …) routed through the interceptor
sub Reroute Claude Code to another subscription's backend (codex|kimi)
compact Configure cheaper models for Claude Code /compact
tray Open the desktop tray app (savings menu-bar / system-tray)
Daemon:
start Start the background interceptor (no-op if already running)
stop Stop the background interceptor
autostart Run the interceptor at login (--off to disable)
When something's wrong:
doctor Check the install end-to-end and explain anything broken
update Update llmtrim to the latest release
uninstall Undo everything `setup` did
Pipes & one-shots:
compress Compress a request from stdin to stdout
send Compress a request, send it to the provider, print the response
serve Run the HTTPS interceptor in the foreground
ca Print the local CA certificate path and how to trust it
Measurement (dev):
eval Measure retrieval recall + token savings on a corpus
bench A/B benchmark: tokens saved vs quality retained, on a real model
discover Scan the capture corpus for where tokens still escape compression
Options:
{options}
Run `llmtrim help <command>` for details on any command.
";
#[derive(Parser)]
#[command(
name = "llmtrim",
version,
about = "Static, deterministic LLM prompt/payload compressor",
styles = HELP_STYLES,
help_template = HELP_TEMPLATE
)]
struct Cli {
#[command(subcommand)]
command: Commands,
}
#[derive(Subcommand)]
enum Commands {
Compress {
#[arg(long)]
provider: Option<String>,
},
Send {
#[arg(long)]
provider: Option<String>,
},
Sub {
#[command(subcommand)]
action: SubCmd,
},
Compact {
#[command(subcommand)]
action: CompactCmd,
},
Serve {
#[arg(long, default_value_t = llmtrim::setup::DEFAULT_PORT)]
port: u16,
#[arg(long)]
force: bool,
#[arg(long, hide = true)]
supervised: bool,
#[arg(long, hide = true)]
hide_console: bool,
},
Setup {
#[arg(long)]
port: Option<u16>,
#[arg(long)]
force: bool,
},
Uninstall {
#[arg(long)]
purge: bool,
#[arg(long)]
keep_binary: bool,
},
Start {
#[arg(long)]
port: Option<u16>,
#[arg(long)]
force: bool,
},
Wrap {
#[arg(trailing_var_arg = true, allow_hyphen_values = true)]
args: Vec<String>,
},
Stop,
#[command(name = "_alive", hide = true)]
Alive,
Update,
#[command(name = "status", visible_aliases = ["monitor", "gain"])]
Monitor {
#[arg(long, hide = true)]
watch: bool,
#[arg(long, default_value_t = 2)]
interval: u64,
#[arg(long)]
daily: bool,
#[arg(long)]
weekly: bool,
#[arg(long)]
monthly: bool,
#[arg(long)]
json: bool,
#[arg(long, requires = "json")]
breakdown: bool,
#[arg(long)]
csv: bool,
#[arg(long, short, conflicts_with_all = ["daily", "weekly", "monthly", "json", "csv"])]
quiet: bool,
},
Mcp {
#[command(subcommand)]
action: Option<McpAction>,
},
Statusline {
#[command(subcommand)]
action: Option<StatuslineCmd>,
},
Guard {
#[command(subcommand)]
action: Option<GuardCmd>,
},
Doctor,
Autostart {
#[arg(long)]
off: bool,
#[arg(long)]
port: Option<u16>,
#[arg(long)]
tray: bool,
#[arg(long)]
status: bool,
},
Tray,
Ca {
#[arg(long)]
pem: bool,
},
Eval {
#[arg(long)]
corpus: PathBuf,
#[arg(long, default_value = "openai")]
provider: String,
#[arg(long, default_value_t = 0.5)]
keep_ratio: f64,
},
Discover {
#[arg(long)]
dir: Option<PathBuf>,
#[arg(long)]
by_tool: bool,
#[arg(long)]
limit: Option<usize>,
#[arg(long)]
json: bool,
},
#[command(subcommand)]
Bench(BenchCmd),
#[command(hide = true)]
WindowSub {
#[arg(trailing_var_arg = true)]
args: Vec<String>,
},
}
#[derive(clap::Subcommand)]
enum BenchCmd {
Quality(Box<BenchArgs>),
Suite(Box<SuiteArgs>),
Agent(Box<BenchAgentArgs>),
Latency(LatencyArgs),
Compare(CompareArgs),
}
#[derive(Subcommand)]
enum SubCmd {
Setup {
provider: String,
},
#[command(visible_alias = "use", alias = "start")]
On {
provider: Option<String>,
#[arg(long)]
no_restart: bool,
},
#[command(alias = "stop")]
Off {
#[arg(long)]
no_restart: bool,
},
Mode {
mode: String,
#[arg(long)]
no_restart: bool,
},
Chain {
providers: String,
#[arg(long)]
no_restart: bool,
},
Effort {
level: String,
#[arg(long)]
no_restart: bool,
},
Map {
provider: String,
from: String,
to: String,
#[arg(long)]
no_restart: bool,
},
Unmap {
provider: String,
from: String,
#[arg(long)]
no_restart: bool,
},
Models {
provider: String,
#[arg(long)]
json: bool,
},
Status {
#[arg(long)]
json: bool,
},
Auth {
provider: String,
#[command(subcommand)]
action: AuthAction,
},
}
#[derive(Subcommand)]
enum CompactCmd {
Models {
#[arg(required = true)]
models: Vec<String>,
#[arg(long)]
no_restart: bool,
},
Off {
#[arg(long)]
no_restart: bool,
},
Status {
#[arg(long)]
json: bool,
},
}
#[derive(Subcommand)]
enum AuthAction {
Login,
Device,
Status {
#[arg(long)]
json: bool,
},
Logout,
}
#[derive(Subcommand)]
enum McpAction {
Install {
#[arg(long)]
print: bool,
#[arg(long)]
force: bool,
},
}
#[derive(Subcommand)]
enum StatuslineCmd {
Install {
#[arg(long)]
print: bool,
},
Uninstall,
}
#[derive(Subcommand)]
enum GuardCmd {
Install {
#[arg(long)]
print: bool,
},
Uninstall,
}
#[derive(clap::Args)]
struct BenchAgentArgs {
#[arg(long = "tasks", default_value = "bench/agent")]
tasks: Vec<PathBuf>,
#[arg(long, default_value = "baseline,agent")]
conditions: String,
#[arg(long, default_value_t = 1)]
repeats: usize,
#[arg(long)]
model: Option<String>,
#[arg(long, default_value = "bench/pricing.json")]
pricing: PathBuf,
#[arg(long)]
live: bool,
#[arg(long, default_value_t = 2)]
tool_turns: usize,
#[arg(long)]
json_out: Option<PathBuf>,
}
#[derive(clap::Args)]
struct BenchArgs {
#[arg(long)]
corpus: PathBuf,
#[arg(long, default_value = "openai")]
provider: String,
#[arg(long, default_value = "auto")]
preset: String,
#[arg(long, default_value = "openai/gpt-oss-20b")]
model: String,
#[arg(long, default_value = "openai/gpt-4o-mini")]
judge_model: String,
#[arg(long, default_value = "groq")]
route: String,
#[arg(long, default_value = "")]
reasoning_effort: String,
#[arg(long)]
n: Option<usize>,
#[arg(long)]
config: Option<PathBuf>,
#[arg(long, default_value = "bench/pricing.json")]
pricing: PathBuf,
#[arg(long)]
offline: bool,
#[arg(long)]
ablate: bool,
#[arg(long)]
json_out: Option<PathBuf>,
}
#[derive(clap::Args)]
struct SuiteArgs {
#[arg(long, default_value = "bench/data")]
data_dir: PathBuf,
#[arg(long, default_value = "bench/results")]
out: PathBuf,
#[arg(long, default_value = "openai/gpt-oss-20b")]
model: String,
#[arg(long, default_value = "openai/gpt-4o-mini")]
judge_model: String,
#[arg(long, default_value = "groq")]
route: String,
#[arg(long, default_value = "bench/pricing.json")]
pricing: PathBuf,
#[arg(long)]
n: Option<usize>,
#[arg(long)]
offline: bool,
}
#[derive(clap::Args)]
struct LatencyArgs {
#[arg(long)]
request: Option<PathBuf>,
#[arg(long)]
provider: Option<String>,
#[arg(long, default_value_t = 100)]
iterations: usize,
}
#[derive(clap::Args)]
struct CompareArgs {
tool: String,
#[arg(long)]
live: bool,
#[arg(last = true)]
extra: Vec<String>,
}
fn read_stdin() -> Result<String> {
let mut buf = String::new();
std::io::stdin()
.read_to_string(&mut buf)
.context("failed to read stdin")?;
Ok(buf)
}
#[cfg(feature = "intercept")]
fn window_sub_provider() -> Result<String> {
let configured = llmtrim_core::config::RuntimeConfig::get()
.sub
.clone()
.or_else(llmtrim_core::config::sub_reenable_provider)
.context("no provider to re-enable — run `llmtrim sub on codex` (or kimi) first")?;
let provider = llmtrim::reroute::SubProvider::parse(&configured)
.context("configured subscription provider is invalid")?;
let logged_in = llmtrim::reroute::auth::auth_status_json(provider)
.get("logged_in")
.and_then(serde_json::Value::as_bool)
.unwrap_or(false);
if !logged_in {
bail!(
"{} is not signed in — run `llmtrim sub auth {} login` first",
provider.as_str(),
provider.as_str()
);
}
Ok(provider.as_str().to_string())
}
#[cfg(not(feature = "intercept"))]
fn window_sub_provider() -> Result<String> {
bail!("window-scoped rerouting needs the `intercept` feature")
}
fn run_window_sub(args: Vec<String>) -> Result<()> {
let action = args.first().map(String::as_str).unwrap_or("status");
let session_from_env = || {
std::env::var("CLAUDE_CODE_SESSION_ID")
.ok()
.or_else(|| std::env::var("CLAUDE_SESSION_ID").ok())
};
match action {
"install" => {
llmtrim::window_sub::install(&std::env::current_exe()?.display().to_string())?;
println!("Claude Code /sub integration installed.");
}
"uninstall" => {
llmtrim::window_sub::uninstall()?;
println!("Claude Code /sub integration removed.");
}
"hook-start" => {
let mut raw = String::new();
std::io::stdin().read_to_string(&mut raw)?;
let value: serde_json::Value = serde_json::from_str(&raw)?;
let session = value
.get("session_id")
.and_then(serde_json::Value::as_str)
.context("SessionStart missing session_id")?;
let source = value
.get("source")
.and_then(serde_json::Value::as_str)
.unwrap_or("startup");
let existing = std::env::var("LLMTRIM_CLAUDE_WINDOW_TOKEN").ok();
let token = llmtrim::window_sub::session_start(session, source, existing.as_deref())?;
if let Ok(file) = std::env::var("CLAUDE_ENV_FILE") {
let mut f = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(file)?;
writeln!(f, "export LLMTRIM_CLAUDE_WINDOW_TOKEN={token}")?;
}
}
"hook-end" => {
let mut raw = String::new();
std::io::stdin().read_to_string(&mut raw)?;
let v: serde_json::Value = serde_json::from_str(&raw)?;
if let Some(s) = v.get("session_id").and_then(serde_json::Value::as_str) {
let reason = v
.get("reason")
.and_then(serde_json::Value::as_str)
.unwrap_or("");
let _ = llmtrim::window_sub::session_end(s, reason);
}
}
"slash" => {
let command = args.get(1).map(String::as_str).unwrap_or("status");
let session = args
.get(2)
.filter(|x| !x.is_empty())
.cloned()
.or_else(session_from_env)
.context("Claude Code session unavailable")?;
match command.trim() {
"on" => {
let provider = window_sub_provider()?;
llmtrim::window_sub::set(&session, true, Some(&provider))?;
println!("Subscription rerouting enabled for this window ({provider}).");
}
"off" => {
llmtrim::window_sub::set(&session, false, None)?;
println!(
"Subscription rerouting disabled for this window; Anthropic compression remains active."
);
}
"status" | "" => match llmtrim::window_sub::status(&session)? {
Some(llmtrim::window_sub::Intent::Enabled { provider }) => {
println!("This window: subscription rerouting enabled ({provider}).")
}
Some(llmtrim::window_sub::Intent::Disabled) => {
println!("This window: subscription rerouting disabled.")
}
None => println!("This window follows the global subscription policy."),
},
_ => bail!("usage: /sub on|off|status"),
}
}
_ => bail!("unknown window-sub action"),
};
Ok(())
}
fn main() {
if let Err(e) = run() {
eprint!("{}", ui::render_error(ui::color_stderr(), &e));
std::process::exit(1);
}
}
#[cfg(feature = "intercept")]
fn run_auth(provider: &str, action: AuthAction) -> Result<()> {
use llmtrim::reroute::{SubProvider, auth};
if let AuthAction::Status { json: true } = action {
let p = SubProvider::parse(provider).ok_or_else(|| anyhow::anyhow!("unknown provider"))?;
println!("{}", auth::auth_status_json(p));
return Ok(());
}
match (provider, action) {
("codex", AuthAction::Login) => auth::codex_login(),
("codex", AuthAction::Device) => auth::codex_device(),
("codex", AuthAction::Status { .. }) => auth::codex_status(),
("codex", AuthAction::Logout) => auth::codex_logout(),
("kimi", AuthAction::Login) => auth::kimi_login(),
("kimi", AuthAction::Device) => {
anyhow::bail!("kimi sign-in is device-code already — use `llmtrim sub auth kimi login`")
}
("kimi", AuthAction::Status { .. }) => auth::kimi_status(),
("kimi", AuthAction::Logout) => auth::kimi_logout(),
(other, _) => anyhow::bail!("unknown provider '{other}' (codex|kimi)"),
}
}
#[cfg(feature = "intercept")]
fn print_reroute_enabled(p: llmtrim::reroute::SubProvider) -> bool {
let logged_in =
llmtrim::reroute::auth::auth_status_json(p)["logged_in"].as_bool() == Some(true);
if logged_in {
println!("Reroute enabled: {}.", p.as_str());
} else {
println!(
"Reroute enabled: {}. Sign in with `llmtrim sub auth {} login`, then \
`llmtrim start --force`.",
p.as_str(),
p.as_str()
);
}
logged_in
}
#[cfg(feature = "intercept")]
fn apply_sub_change(no_restart: bool) {
let Some(state) = llmtrim::daemon::running() else {
return;
};
if no_restart {
println!("Restart to apply: `llmtrim start --force`.");
return;
}
let port = state.port;
let restarted = matches!(llmtrim::daemon::stop_and_wait_free(port), Ok(true))
.then(|| llmtrim::daemon::spawn_detached(port).ok())
.flatten();
match restarted {
Some(pid) => println!("↻ Restarted interceptor (pid {pid}) to apply."),
None => eprintln!(
"llmtrim: change saved, but the restart failed — run `llmtrim start --force`."
),
}
}
#[cfg(feature = "intercept")]
fn apply_sub_map_change(edited: llmtrim::reroute::SubProvider, no_restart: bool) {
let active = llmtrim_core::config::RuntimeConfig::get()
.sub
.as_deref()
.and_then(llmtrim::reroute::SubProvider::parse);
if active == Some(edited) {
apply_sub_change(no_restart);
}
}
#[cfg(feature = "intercept")]
fn run_compact(action: CompactCmd) -> Result<()> {
match action {
CompactCmd::Models { models, no_restart } => {
let mut normalized = Vec::new();
for model in models
.iter()
.flat_map(|model| model.split(','))
.map(str::trim)
.filter(|model| !model.is_empty())
{
if !normalized.iter().any(|existing| existing == model) {
normalized.push(model.to_string());
}
}
if normalized.is_empty() {
anyhow::bail!("compact model list is empty; use `llmtrim compact off` to disable");
}
llmtrim_core::config::write_compact_models(&normalized)?;
println!(
"Compact models: {} -> original model (implicit).",
normalized.join(" -> ")
);
apply_sub_change(no_restart);
Ok(())
}
CompactCmd::Off { no_restart } => {
llmtrim_core::config::write_compact_models(&[])?;
println!("Compact model substitution disabled; `/compact` uses the original model.");
apply_sub_change(no_restart);
Ok(())
}
CompactCmd::Status { json } => {
let cfg = llmtrim_core::config::RuntimeConfig::get();
let sub = cfg
.sub
.as_deref()
.and_then(llmtrim::reroute::SubProvider::parse);
let original = "<original model>";
let plan = llmtrim::compact::plan(&cfg.compact_models, original, sub, &cfg.sub_tiers);
if json {
let out = serde_json::json!({
"models": cfg.compact_models,
"original_fallback": true,
"resolved": plan.iter().map(|candidate| serde_json::json!({
"logical": candidate.logical_model,
"upstream": candidate.upstream_model,
"context_window": llmtrim_core::context_window(&candidate.upstream_model),
"original": candidate.is_original,
})).collect::<Vec<_>>(),
});
println!("{}", serde_json::to_string_pretty(&out)?);
} else if cfg.compact_models.is_empty() {
println!("Compact models: off (original model only).");
} else {
println!("Compact models: {}", cfg.compact_models.join(" -> "));
for candidate in plan.iter().filter(|candidate| !candidate.is_original) {
let window = llmtrim_core::context_window(&candidate.upstream_model)
.map(|tokens| format!("{tokens} tokens"))
.unwrap_or_else(|| "unknown context".to_string());
println!(
" {} -> {} ({window})",
candidate.logical_model, candidate.upstream_model
);
}
println!(" then the original requested model (implicit)");
}
Ok(())
}
}
}
#[cfg(feature = "intercept")]
fn run_sub(action: SubCmd) -> Result<()> {
use llmtrim::reroute::{SubProvider, Tier, default_codex_tier_model};
let parse = |p: &str| {
SubProvider::parse(p).ok_or_else(|| anyhow::anyhow!("unknown provider '{p}' (codex|kimi)"))
};
match action {
SubCmd::Setup { provider } => {
let p = parse(&provider)?;
#[cfg(feature = "breakdown")]
{
llmtrim::reroute::tui::run(p)
}
#[cfg(not(feature = "breakdown"))]
{
let _ = p;
anyhow::bail!("the mapping editor needs the `breakdown` feature")
}
}
SubCmd::On {
provider,
no_restart,
} => {
let logged_in = match provider {
Some(provider) => {
let p = parse(&provider)?;
let mut map = std::collections::BTreeMap::new();
if p == SubProvider::Codex {
for t in Tier::ALL {
map.insert(
t.as_str().to_string(),
default_codex_tier_model(t).to_string(),
);
}
}
llmtrim_core::config::write_sub_mapping(p.as_str(), &map)?;
print_reroute_enabled(p)
}
None => {
let provider = llmtrim_core::config::sub_reenable_provider().ok_or_else(|| {
anyhow::anyhow!(
"no provider to re-enable — run `llmtrim sub on codex` (or kimi) first"
)
})?;
let p = parse(&provider)?;
llmtrim_core::config::enable_sub(p.as_str())?;
print_reroute_enabled(p)
}
};
if logged_in {
apply_sub_change(no_restart);
}
Ok(())
}
SubCmd::Off { no_restart } => {
llmtrim_core::config::disable_sub()?;
println!("Reroute disabled — traffic goes to Anthropic (compression only).");
apply_sub_change(no_restart);
Ok(())
}
SubCmd::Mode { mode, no_restart } => {
let fallback = match mode.trim().to_ascii_lowercase().as_str() {
"fallback" | "on-error" | "on_error" | "onerror" => true,
"always" => false,
other => anyhow::bail!("unknown mode '{other}' (always|fallback)"),
};
llmtrim_core::config::write_sub_mode(fallback)?;
println!(
"Reroute mode: {}.",
if fallback { "fallback" } else { "always" }
);
apply_sub_change(no_restart);
Ok(())
}
SubCmd::Chain {
providers,
no_restart,
} => {
let mut chain = Vec::new();
for raw in providers.split(',') {
let raw = raw.trim();
if raw.is_empty() {
continue;
}
let p = parse(raw)?;
if !chain.contains(&p.as_str().to_string()) {
chain.push(p.as_str().to_string());
}
}
if chain.is_empty() {
anyhow::bail!("fallback chain is empty (expected codex,kimi)");
}
llmtrim_core::config::write_sub_chain(&chain)?;
println!("Fallback chain: {}.", chain.join(" -> "));
apply_sub_change(no_restart);
Ok(())
}
SubCmd::Effort { level, no_restart } => {
let level = match level.trim().to_ascii_lowercase().as_str() {
"max" => "xhigh".to_string(),
l @ ("none" | "low" | "medium" | "high" | "xhigh") => l.to_string(),
other => anyhow::bail!("unknown effort '{other}' (none|low|medium|high|xhigh)"),
};
llmtrim_core::config::write_sub_effort(SubProvider::Codex.as_str(), &level)?;
println!("Codex reasoning effort: {level}.");
let active = llmtrim_core::config::RuntimeConfig::get()
.sub
.as_deref()
.and_then(SubProvider::parse);
if active == Some(SubProvider::Codex) {
apply_sub_change(no_restart);
} else {
eprintln!(
"note: reroute is not on codex right now, so this effort has no effect until \
`llmtrim sub on codex`."
);
}
Ok(())
}
SubCmd::Map {
provider,
from,
to,
no_restart,
} => {
let p = parse(&provider)?;
llmtrim_core::config::write_sub_map_entry(p.as_str(), &from, &to)?;
println!("Mapped {from} -> {to} for {}.", p.as_str());
apply_sub_map_change(p, no_restart);
Ok(())
}
SubCmd::Unmap {
provider,
from,
no_restart,
} => {
let p = parse(&provider)?;
llmtrim_core::config::remove_sub_map_entry(p.as_str(), &from)?;
println!(
"Unmapped {from} for {} (back to the preset default).",
p.as_str()
);
apply_sub_map_change(p, no_restart);
Ok(())
}
SubCmd::Models { provider, json } => {
let p = parse(&provider)?;
let ids: Vec<String> = llmtrim::reroute::catalog::models_for(p)
.into_iter()
.map(|e| e.id)
.collect();
if json {
println!("{}", serde_json::to_string(&ids)?);
} else {
for id in ids {
println!("{id}");
}
}
Ok(())
}
SubCmd::Status { json } => {
let cfg = llmtrim_core::config::RuntimeConfig::get();
let active = cfg.sub.as_deref().and_then(SubProvider::parse);
if json {
let mut mapping: std::collections::BTreeMap<String, String> = match active {
Some(SubProvider::Codex) => Tier::ALL
.iter()
.map(|t| {
(
t.as_str().to_string(),
default_codex_tier_model(*t).to_string(),
)
})
.collect(),
_ => std::collections::BTreeMap::new(),
};
for (k, v) in &cfg.sub_tiers {
mapping.insert(k.clone(), v.clone());
}
let out = serde_json::json!({
"provider": active.map(|p| p.as_str()),
"mode": if cfg.sub_fallback { "fallback" } else { "always" },
"chain": cfg.sub_chain,
"effort": cfg.sub_effort,
"mapping": mapping,
"auth": {
"codex": llmtrim::reroute::auth::auth_status_json(SubProvider::Codex),
"kimi": llmtrim::reroute::auth::auth_status_json(SubProvider::Kimi),
},
});
println!("{}", serde_json::to_string_pretty(&out)?);
return Ok(());
}
match active {
None => println!("Reroute: off"),
Some(p) => {
println!(
"Reroute: {} ({})",
p.as_str(),
if cfg.sub_fallback {
"fallback"
} else {
"always"
}
);
if cfg.sub_fallback {
println!(
" chain -> {}",
if cfg.sub_chain.is_empty() {
p.as_str().to_string()
} else {
cfg.sub_chain.join(" -> ")
}
);
}
if p == SubProvider::Codex {
println!(
" effort -> {}",
cfg.sub_effort.as_deref().unwrap_or("none")
);
for t in Tier::ALL {
let model = cfg
.sub_tiers
.get(t.as_str())
.cloned()
.unwrap_or_else(|| default_codex_tier_model(t).to_string());
println!(" {:<7} -> {model}", t.as_str());
}
let tier_keys: [&str; 4] = Tier::ALL.map(|t| t.as_str());
for (from, to) in &cfg.sub_tiers {
if !tier_keys.contains(&from.as_str()) {
println!(" {from} -> {to}");
}
}
} else {
println!(" all tiers -> {}", llmtrim::reroute::KIMI_MODEL);
}
let auth = llmtrim::reroute::auth::auth_status_json(p);
if auth["logged_in"].as_bool().unwrap_or(false) {
println!(" auth: logged in");
} else {
println!(
" auth: NOT logged in — run `llmtrim sub auth {} login`",
p.as_str()
);
}
}
}
Ok(())
}
SubCmd::Auth { provider, action } => run_auth(&provider, action),
}
}
#[cfg(not(feature = "intercept"))]
fn run_sub(_action: SubCmd) -> Result<()> {
anyhow::bail!("this build has no interceptor; rebuild with `--features intercept`")
}
fn run() -> Result<()> {
let cli = Cli::parse();
match cli.command {
Commands::Compress { provider } => {
let input = read_stdin()?;
let kind = provider
.as_deref()
.map(ProviderKind::from_str)
.transpose()?;
let result = llmtrim_core::compress(&input, kind)?;
if let Ok(tracker) = Tracker::open() {
let _ = tracker.record(&Record {
provider: result.provider.as_str().to_string(),
model: result.model.clone(),
tokenizer: result.tokenizer_label.clone(),
exact: result.tokenizer_exact,
input_before: result.input_tokens_before.0 as i64,
input_after: result.input_tokens_after.0 as i64,
output_before: None,
output_after: None,
compress_micros: None,
cache_read_tokens: None,
fresh_input_tokens: None,
cache_write_tokens: None,
output_shaped: Some(result.output_shaped),
frozen_input_tokens: Some(result.frozen_input_tokens.0 as i64),
});
}
let mut stdout = std::io::stdout().lock();
stdout
.write_all(result.request_json.as_bytes())
.context("failed to write stdout")?;
let _ = stdout.write_all(b"\n");
}
Commands::Send { provider } => {
let input = read_stdin()?;
let kind = provider
.as_deref()
.map(ProviderKind::from_str)
.transpose()?;
let result = llmtrim_core::compress(&input, kind)?;
let endpoint = Endpoint::from_env(result.provider)?;
let proxy_url = llmtrim::transport::upstream_proxy_url(None)?;
let response = endpoint.send(&result.request_json, proxy_url.as_deref())?;
if let Ok(counter) =
llmtrim_core::tokenizer::counter_for(result.provider, result.model.as_deref())
{
let output_after = serde_json::from_str::<serde_json::Value>(&response)
.ok()
.and_then(|v| llmtrim_core::provider::for_kind(result.provider).answer_text(&v))
.map(|text| counter.count(&text) as i64);
if let Ok(tracker) = Tracker::open() {
let _ = tracker.record(&Record {
provider: result.provider.as_str().to_string(),
model: result.model.clone(),
tokenizer: counter.label().to_string(),
exact: counter.is_exact(),
input_before: result.input_tokens_before.0 as i64,
input_after: result.input_tokens_after.0 as i64,
output_before: None,
output_after,
compress_micros: None,
cache_read_tokens: None,
fresh_input_tokens: None,
cache_write_tokens: None,
output_shaped: Some(result.output_shaped),
frozen_input_tokens: Some(result.frozen_input_tokens.0 as i64),
});
}
}
println!("{response}");
}
Commands::Serve {
port,
force,
supervised,
hide_console,
} => {
if hide_console {
llmtrim::autostart::hide_console_window();
}
if supervised {
let _ = llmtrim::setup::heal_managed_env();
#[cfg(windows)]
let _ = llmtrim::setup::wire_env_windows(port);
llmtrim::serve::run_supervised(port, force)?;
} else {
llmtrim::serve::run(port, force)?;
}
}
Commands::Setup { port, force } => llmtrim::setup::run(port, force)?,
Commands::Uninstall { purge, keep_binary } => {
llmtrim::setup::uninstall(purge, keep_binary)?
}
Commands::Start { port, force } => {
let color = ui::color_stdout();
if force && let Some(state) = llmtrim::daemon::running() {
println!(
"{}",
ui::ok(
color,
&format!("Stopping existing daemon · pid {} (--force)", state.pid)
)
);
if !llmtrim::daemon::stop_and_wait_free(state.port)? {
anyhow::bail!(
"--force stopped pid {} but port {} was still held after 5s",
state.pid,
state.port
);
}
}
if let Some(state) = llmtrim::daemon::running() {
println!(
"{}",
ui::ok(
color,
&format!(
"Interceptor already running · pid {} · port {}",
state.pid, state.port
)
)
);
} else {
let port = llmtrim::setup::resolve_port(port, None)?;
let pid = llmtrim::daemon::spawn_detached(port)?;
println!(
"{}",
ui::ok(
color,
&format!("Interceptor running · pid {pid} · port {port}")
)
);
for (label, value) in [
("watch", "llmtrim status".to_string()),
("stop", "llmtrim stop".to_string()),
] {
println!(
" {} {value}",
ui::paint(color, Tone::Dim, &format!("{label:<5}"))
);
}
if !llmtrim::setup::profile_has_block() {
eprintln!(
"{}",
ui::note(
ui::color_stderr(),
"HTTPS_PROXY isn't set for your user yet — run `llmtrim setup` so \
tools actually route through the interceptor."
)
);
}
}
if let Some(msg) = llmtrim::tray::nudge_once() {
println!("{}", ui::paint(color, Tone::Dim, &msg));
}
}
Commands::Wrap { args } => llmtrim::wrap::run(args)?,
Commands::Alive => {
if llmtrim::daemon::running().is_none() {
std::process::exit(1);
}
}
Commands::Stop => match llmtrim::daemon::stop()? {
Some(pid) => {
println!(
"{}",
ui::ok(
ui::color_stdout(),
&format!("Stopped interceptor (pid {pid}).")
)
);
#[cfg(not(windows))]
if llmtrim::wrap::https_proxy_is_local() {
eprintln!(
"{}",
ui::warn(
ui::color_stderr(),
&format!(
"This shell still points at the stopped interceptor. New \
terminals are already clear; to fix this one, run:\n {}",
llmtrim::setup::UNSET_HINT
)
)
);
}
#[cfg(windows)]
match llmtrim::setup::unwire_env_windows() {
Ok(true) => eprintln!(
"{}",
ui::note(
ui::color_stderr(),
"Cleared HTTPS_PROXY from your user environment — new terminals and \
apps are clear. Apps already running keep the old proxy until you \
restart them."
)
),
Ok(false) => {}
Err(e) => eprintln!(
"{}",
ui::warn(
ui::color_stderr(),
&format!("could not clear the interceptor env: {e}")
)
),
}
}
None => println!(
"{}",
ui::note(ui::color_stdout(), "No interceptor daemon was running.")
),
},
Commands::Sub { action } => run_sub(action)?,
Commands::Compact { action } => run_compact(action)?,
Commands::WindowSub { args } => {
let hook = args.first().is_some_and(|arg| arg.starts_with("hook-"));
if let Err(error) = run_window_sub(args) {
if hook {
eprintln!("llmtrim: window /sub hook skipped: {error:#}");
} else {
return Err(error);
}
}
}
Commands::Update => llmtrim::update::run()?,
Commands::Mcp { action } => match action {
None => llmtrim::mcp::run()?,
Some(McpAction::Install { print, force }) => llmtrim::mcp::install(print, force)?,
},
Commands::Statusline { action } => match action {
None => llmtrim::statusline::run()?,
Some(StatuslineCmd::Install { print }) => llmtrim::statusline::install(print)?,
Some(StatuslineCmd::Uninstall) => llmtrim::statusline::uninstall()?,
},
Commands::Guard { action } => match action {
None => std::process::exit(llmtrim::guard::run()),
Some(GuardCmd::Install { print }) => llmtrim::guard::install(print)?,
Some(GuardCmd::Uninstall) => llmtrim::guard::uninstall()?,
},
Commands::Monitor {
watch: _,
interval,
daily,
weekly,
monthly,
json,
breakdown,
csv,
quiet,
} => run_monitor(
interval, daily, weekly, monthly, json, breakdown, csv, quiet,
)?,
Commands::Doctor => {
let report = llmtrim::doctor::gather();
print!("{}", llmtrim::doctor::render(ui::color_stdout(), &report));
if report.problems > 0 {
std::process::exit(1);
}
}
Commands::Tray => llmtrim::tray::run()?,
Commands::Autostart {
off,
port,
tray,
status,
} => {
let color = ui::color_stdout();
if status {
let enabled = if tray {
llmtrim::autostart::is_tray_enabled()
} else {
llmtrim::autostart::is_enabled()
};
println!("{}", if enabled { "enabled" } else { "disabled" });
return Ok(());
}
if tray {
llmtrim::autostart::configure_tray(!off)?;
let msg = if off {
"Tray autostart disabled."
} else {
"Tray autostart enabled — the llmtrim tray opens at login."
};
println!("{}", ui::ok(color, msg));
return Ok(());
}
if off {
llmtrim::autostart::configure(false, llmtrim::setup::DEFAULT_PORT)?;
println!("{}", ui::ok(color, "Autostart disabled."));
} else {
let running = llmtrim::daemon::running().map(|s| s.port);
let port = llmtrim::setup::resolve_port(port, running)
.context("llmtrim autostart: could not pick a port — pass --port explicitly")?;
llmtrim::autostart::configure(true, port)?;
println!(
"{}",
ui::ok(
color,
&format!("Autostart enabled — llmtrim serve --port {port} runs at login.")
)
);
println!(
" {} llmtrim autostart --off",
ui::paint(color, Tone::Dim, "undo ")
);
}
}
Commands::Ca { pem } => {
let path = llmtrim::serve::ca_cert_path()?;
llmtrim::serve::ensure_ca()?; if pem {
print!("{}", std::fs::read_to_string(&path)?);
return Ok(());
}
let color = ui::color_stdout();
print!(
"{}",
ui::panel(color, "llmtrim local CA", &[path.display().to_string()])
);
println!();
println!("Trust it for your tool, then route its traffic through llmtrim:");
#[cfg(windows)]
{
println!(" $env:NODE_EXTRA_CA_CERTS = \"{}\"", path.display());
println!(
" $env:HTTPS_PROXY = \"http://127.0.0.1:{}\"",
llmtrim::setup::DEFAULT_PORT
);
println!(" llmtrim serve");
println!();
println!("System-wide trust (non-PowerShell / GUI apps):");
println!(" certutil -addstore -user Root \"{}\"", path.display());
}
#[cfg(not(windows))]
{
println!(" export NODE_EXTRA_CA_CERTS={}", path.display());
println!(
" export HTTPS_PROXY=http://127.0.0.1:{}",
llmtrim::setup::DEFAULT_PORT
);
println!(" llmtrim serve");
}
println!();
println!(
"{}",
ui::note(
ui::color_stdout(),
"The CA is name-constrained to LLM API domains only."
)
);
}
Commands::Eval {
corpus,
provider,
keep_ratio,
} => {
let kind = ProviderKind::from_str(&provider)?;
let jsonl = std::fs::read_to_string(&corpus)
.with_context(|| format!("failed to read corpus {}", corpus.display()))?;
let cases = llmtrim::quality::load_corpus(&jsonl, kind)?;
let keep_ratio = keep_ratio.clamp(0.0, 1.0);
let config = llmtrim_core::config::DenseConfig {
retrieve: true,
retrieve_keep_ratio: keep_ratio,
retrieve_min_segment_chars: 120,
output_control: false,
..Default::default()
};
let results = llmtrim::quality::run_recall(&cases, &config)?;
let color = ui::color_stdout();
let saved_cell = |pct: f64| {
if pct > 0.0 {
ui::paint(color, Tone::Accent, &format!("-{pct:.1}%"))
} else {
ui::paint(color, Tone::Dim, &format!("{:.1}%", pct.abs()))
}
};
let mut t = ui::table(color, &["case", "recall", "tokens", "saved"]);
for r in &results {
t.add_row(vec![
comfy_table::Cell::new(&r.name),
ui::right(format!("{:.2}", r.recall)),
ui::right(format!("{} → {}", r.tokens_before, r.tokens_after)),
ui::right(saved_cell(r.savings_pct())),
]);
}
for line in t.to_string().lines() {
println!(" {line}");
}
let recall = llmtrim::quality::mean_recall(&results);
let savings = if results.is_empty() {
0.0
} else {
results.iter().map(|r| r.savings_pct()).sum::<f64>() / results.len() as f64
};
print!(
"\n{}",
ui::panel(
color,
"eval",
&[format!(
"{} cases mean recall {} mean savings {} keep_ratio {keep_ratio}",
results.len(),
ui::paint(color, Tone::Bold, &format!("{recall:.2}")),
saved_cell(savings),
)]
)
);
}
Commands::Discover {
dir,
by_tool,
limit,
json,
} => llmtrim::discover::run(dir, json, by_tool, limit)?,
Commands::Bench(cmd) => match cmd {
BenchCmd::Quality(args) => run_bench(*args)?,
BenchCmd::Suite(args) => run_bench_suite(*args)?,
BenchCmd::Agent(args) => run_bench_agent(*args)?,
BenchCmd::Latency(args) => run_bench_latency(args)?,
BenchCmd::Compare(args) => run_bench_compare(args)?,
},
}
Ok(())
}
#[cfg(feature = "live")]
fn dotenv_get(key: &str) -> Option<String> {
if let Ok(v) = std::env::var(key) {
return Some(v);
}
dotenvy::from_path_iter(".env")
.ok()?
.flatten()
.find(|(k, _)| k == key)
.map(|(_, v)| v)
}
fn prepare_request(request_json: &str, model: &str, route: &str, reasoning_effort: &str) -> String {
match serde_json::from_str::<serde_json::Value>(request_json) {
Ok(mut v) => {
if let Some(obj) = v.as_object_mut() {
obj.insert("model".to_string(), serde_json::json!(model));
if !route.is_empty() {
let (provider, quant) = match route.split_once('/') {
Some((p, q)) => (p, Some(q)),
None => (route, None),
};
let mut routing =
serde_json::json!({"order": [provider], "allow_fallbacks": false});
if let Some(q) = quant {
routing["quantizations"] = serde_json::json!([q]);
}
obj.insert("provider".to_string(), routing);
}
if !reasoning_effort.is_empty() {
obj.insert(
"reasoning".to_string(),
serde_json::json!({"effort": reasoning_effort}),
);
}
obj.insert("usage".to_string(), serde_json::json!({"include": true}));
}
v.to_string()
}
Err(_) => request_json.to_string(),
}
}
fn run_bench(args: BenchArgs) -> Result<()> {
let kind = ProviderKind::from_str(&args.provider)?;
let jsonl = std::fs::read_to_string(&args.corpus)
.with_context(|| format!("failed to read corpus {}", args.corpus.display()))?;
let mut cases = bench::load_bench_corpus(&jsonl, kind, &args.model)?;
if let Some(limit) = args.n {
cases.truncate(limit);
}
for c in &mut cases {
c.request = prepare_request(&c.request, &args.model, &args.route, &args.reasoning_effort);
}
let mut config = match &args.config {
Some(path) => {
let t = std::fs::read_to_string(path)
.with_context(|| format!("failed to read config {}", path.display()))?;
toml::from_str::<DenseConfig>(&t)
.with_context(|| format!("failed to parse config {}", path.display()))?
}
None => DenseConfig::preset(&args.preset).with_context(|| {
format!(
"unknown preset '{}' (auto|safe|lossless|rag|agent|code|aggressive|cache|reasoning)",
args.preset
)
})?,
};
config.retrieve_keep_ratio = config.retrieve_keep_ratio.clamp(0.0, 1.0);
config.retrieve_mmr_lambda = config.retrieve_mmr_lambda.clamp(0.0, 1.0);
if args.ablate {
run_ablation(&cases, &config, &args.preset)
} else if args.offline {
run_offline(&cases, kind, &config, &args)
} else {
run_live(&cases, kind, &config, &args)
}
}
const SUITE_MATRIX: &[(&str, &str, usize)] = &[
("gsm8k", "reasoning", 12), ("humaneval", "code", 12), ("dolly", "aggressive", 12), ("hotpotqa", "rag", 12), ("glaive", "agent", 12), ("chat", "aggressive", 12), ("cnn", "aggressive", 8), ("cache", "cache", 12), ("truthfulqa", "safe", 20), ("squad2", "rag", 20), ("bfcl", "agent", 20), ];
fn run_bench_suite(args: SuiteArgs) -> Result<()> {
if !args.offline {
for k in [
"HTTPS_PROXY",
"https_proxy",
"HTTP_PROXY",
"http_proxy",
"ALL_PROXY",
] {
if std::env::var_os(k).is_some() {
anyhow::bail!(
"{k} is set; the llmtrim proxy would re-compress the baseline arm and \
contaminate the A/B. Unset all *_PROXY vars before a live suite run."
);
}
}
}
std::fs::create_dir_all(&args.out)
.with_context(|| format!("failed to create {}", args.out.display()))?;
for (corpus, preset, default_n) in SUITE_MATRIX {
let n = args.n.or(Some(*default_n));
eprintln!(
"{}",
ui::note(
ui::color_stderr(),
&format!("=== {corpus} ({preset}, n={}) ===", n.unwrap_or(0))
)
);
let corpus_args = BenchArgs {
corpus: args.data_dir.join(format!("{corpus}.jsonl")),
provider: "openai".to_string(),
preset: (*preset).to_string(),
model: args.model.clone(),
judge_model: args.judge_model.clone(),
route: args.route.clone(),
reasoning_effort: String::new(),
n,
config: None,
pricing: args.pricing.clone(),
offline: args.offline,
ablate: false,
json_out: Some(args.out.join(format!("{corpus}.json"))),
};
if let Err(e) = run_bench(corpus_args) {
eprintln!(
"{}",
ui::render_error(ui::color_stderr(), &e.context(format!("corpus {corpus}")))
);
}
}
eprintln!("{}", ui::note(ui::color_stderr(), "suite complete"));
Ok(())
}
fn run_bench_latency(args: LatencyArgs) -> Result<()> {
use std::time::Instant;
const FIXTURE: &str = r#"{"model":"gpt-4o","messages":[
{"role":"system","content":"You are a meticulous coding assistant. Answer precisely."},
{"role":"user","content":"Review this function for bugs:\n```rust\nfn process(data: &[i32]) -> i32 {\n let mut total = 0;\n for x in data {\n if *x > 0 {\n total += x * 2;\n }\n }\n total\n}\n```\nDoes it handle the spec's edge cases?"}
]}"#;
let input = match &args.request {
Some(p) => {
std::fs::read_to_string(p).with_context(|| format!("failed to read {}", p.display()))?
}
None => FIXTURE.to_string(),
};
let kind = args
.provider
.as_deref()
.map(ProviderKind::from_str)
.transpose()?;
let config = DenseConfig::auto();
for _ in 0..5 {
let _ = llmtrim_core::compress_with_config(&input, kind, &config)?;
}
let n = args.iterations.max(1);
let mut r = llmtrim_core::compress_with_config(&input, kind, &config)?;
let t = Instant::now();
for _ in 0..n {
r = llmtrim_core::compress_with_config(&input, kind, &config)?;
}
let per_req_ms = t.elapsed().as_secs_f64() * 1000.0 / n as f64;
let counter = llmtrim_core::tokenizer::counter_for(r.provider, r.model.as_deref())?;
let value: serde_json::Value = serde_json::from_str(&input).context("parse request JSON")?;
let tools_str = value
.get("tools")
.map(|t| t.to_string())
.unwrap_or_default();
let bench = |label: &str, f: &dyn Fn()| {
let t = Instant::now();
for _ in 0..n {
f();
}
println!(
" {label}: {:.2} ms",
t.elapsed().as_secs_f64() * 1000.0 / n as f64
);
};
println!("request: {} bytes, provider={:?}", input.len(), r.provider);
println!(
"input tokens: {} -> {} ({:.1}% saved)",
r.input_tokens_before,
r.input_tokens_after,
100.0 * (1.0 - r.input_tokens_after.0 as f64 / r.input_tokens_before.0.max(1) as f64)
);
println!("compress latency: {per_req_ms:.2} ms/req (warm, avg of {n})");
println!("attribution (1x each):");
bench("full tokenize (content+tools)", &|| {
let _ = counter.count(&input);
});
bench("tools tokenize only", &|| {
let _ = counter.count(&tools_str);
});
bench("Value clone (per-stage snapshot)", &|| {
let _ = value.clone();
});
bench("JSON parse", &|| {
let _ = serde_json::from_str::<serde_json::Value>(&input);
});
bench("JSON serialize", &|| {
let _ = value.to_string();
});
Ok(())
}
fn run_bench_compare(args: CompareArgs) -> Result<()> {
match args.tool.as_str() {
"headroom" | "caveman" | "leanctx" | "entroly" | "rtk" | "snip" => {}
other => anyhow::bail!(
"unknown comparator '{other}' (expected: \
headroom|caveman|leanctx|entroly|rtk|snip)"
),
}
let script = "bench/scripts/bench.py";
if !std::path::Path::new(script).exists() {
anyhow::bail!(
"{script} not found - run `llmtrim bench compare` from the repo root (it dispatches \
the Python comparator, which needs its own deps; see bench/README.md)"
);
}
let mut cmd = std::process::Command::new("python3");
cmd.arg(script).arg(&args.tool);
if args.live {
cmd.arg("--live");
}
cmd.args(&args.extra);
let status = cmd
.status()
.with_context(|| format!("failed to run python3 {script}"))?;
if !status.success() {
std::process::exit(status.code().unwrap_or(1));
}
Ok(())
}
fn run_bench_agent(args: BenchAgentArgs) -> Result<()> {
use llmtrim::bench::agent::Condition;
let mut tasks = load_agent_tasks(&args.tasks)?;
if tasks.is_empty() {
anyhow::bail!("no agent tasks found under {:?}", args.tasks);
}
if let Some(m) = &args.model {
for t in &mut tasks {
t.model = m.clone();
}
}
let conditions: Vec<Condition> = args
.conditions
.split(',')
.map(|s| Condition::parse(s.trim()))
.collect();
let table = std::fs::read_to_string(&args.pricing)
.map(|s| bench::load_pricing(&s))
.unwrap_or_default();
if args.live {
run_agent_live(&tasks, &conditions, &args, &table)
} else {
run_agent_dry(&tasks, &conditions, &args, &table)
}
}
fn load_agent_tasks(paths: &[PathBuf]) -> Result<Vec<llmtrim::bench::agent::AgentTask>> {
use llmtrim::bench::agent::AgentTask;
let mut files = Vec::new();
for p in paths {
if p.is_dir() {
for entry in
std::fs::read_dir(p).with_context(|| format!("read dir {}", p.display()))?
{
let path = entry?.path();
let is_underscore = path
.file_name()
.and_then(|n| n.to_str())
.is_some_and(|n| n.starts_with('_'));
if !is_underscore && path.extension().and_then(|e| e.to_str()) == Some("json") {
files.push(path);
}
}
} else {
files.push(p.clone());
}
}
files.sort();
let mut shared: std::collections::HashMap<PathBuf, serde_json::Value> = Default::default();
let mut tasks = Vec::new();
for f in files {
let s = std::fs::read_to_string(&f).with_context(|| format!("read {}", f.display()))?;
let mut task =
AgentTask::from_json(&s).with_context(|| format!("parse {}", f.display()))?;
if task.tools.is_null() || task.tools.as_array().is_some_and(|a| a.is_empty()) {
let dir = f.parent().unwrap_or_else(|| std::path::Path::new("."));
let tools = match shared.get(dir) {
Some(v) => v.clone(),
None => {
let path = dir.join("_tools.json");
let v = std::fs::read_to_string(&path)
.ok()
.and_then(|t| serde_json::from_str(&t).ok())
.unwrap_or(serde_json::Value::Null);
shared.insert(dir.to_path_buf(), v.clone());
v
}
};
task.tools = tools;
}
tasks.push(task);
}
Ok(tasks)
}
fn run_agent_dry(
tasks: &[llmtrim::bench::agent::AgentTask],
conditions: &[llmtrim::bench::agent::Condition],
args: &BenchAgentArgs,
table: &bench::PriceTable,
) -> Result<()> {
use llmtrim::bench::agent::{OpenAiAgent, dry_run_transport, run_agent_loop};
let provider = OpenAiAgent;
let mut results = Vec::new();
for task in tasks {
let price = bench::resolve_pricing(table, &task.model);
for cond in conditions {
for _ in 0..args.repeats.max(1) {
let mut send = dry_run_transport(args.tool_turns);
results.push(run_agent_loop(&provider, task, cond, &price, &mut send)?);
}
}
}
print_agent_results(&results, true);
write_agent_json(&results, &args.json_out)
}
#[cfg(feature = "live")]
fn run_agent_live(
tasks: &[llmtrim::bench::agent::AgentTask],
conditions: &[llmtrim::bench::agent::Condition],
args: &BenchAgentArgs,
table: &bench::PriceTable,
) -> Result<()> {
use llmtrim::bench::agent::{OpenAiAgent, run_agent_loop};
let api_key =
dotenv_get("OPENROUTER_API_KEY").context("OPENROUTER_API_KEY not set (env or a .env)")?;
let model = llmtrim::quality::OpenRouterModel::new(api_key, ProviderKind::OpenAi)?;
let provider = OpenAiAgent;
let mut results = Vec::new();
for task in tasks {
let price = bench::resolve_pricing(table, &task.model);
for cond in conditions {
for _ in 0..args.repeats.max(1) {
let mut send = |req: &str| model.send_raw(req);
results.push(run_agent_loop(&provider, task, cond, &price, &mut send)?);
}
}
}
print_agent_results(&results, false);
write_agent_json(&results, &args.json_out)
}
#[cfg(not(feature = "live"))]
fn run_agent_live(
_tasks: &[llmtrim::bench::agent::AgentTask],
_conditions: &[llmtrim::bench::agent::Condition],
_args: &BenchAgentArgs,
_table: &bench::PriceTable,
) -> Result<()> {
anyhow::bail!(
"--live needs the `live` feature: rebuild with `cargo run --features live -- bench agent --live …`"
)
}
fn print_agent_results(results: &[llmtrim::bench::agent::AgentRunResult], dry: bool) {
let color = ui::color_stdout();
let mut t = ui::table(
color,
&[
"task",
"condition",
"iters",
"input",
"cached",
"output",
"cost$",
"done",
],
);
for r in results {
t.add_row(vec![
r.task_id.clone(),
r.condition.clone(),
r.iterations.to_string(),
r.input_tokens.to_string(),
r.cached_tokens.to_string(),
r.output_tokens.to_string(),
format!("{:.4}", r.cost_usd),
if r.completed { "yes" } else { "no" }.to_string(),
]);
}
println!("{t}");
if dry {
println!(
"(dry-run: synthetic usage, cached estimated by prefix overlap — no API calls; use --live for real numbers)"
);
}
}
fn write_agent_json(
results: &[llmtrim::bench::agent::AgentRunResult],
out: &Option<PathBuf>,
) -> Result<()> {
if let Some(path) = out {
let doc = bench::envelope::wrap(
"agent-v1",
serde_json::json!({ "runs": results.len() }),
serde_json::to_value(results).context("serialize agent results")?,
);
let json = serde_json::to_string_pretty(&doc).context("serialize agent results")?;
std::fs::write(path, json).with_context(|| format!("write {}", path.display()))?;
}
Ok(())
}
fn run_ablation(cases: &[BenchCase], config: &DenseConfig, preset: &str) -> Result<()> {
let configs = bench::ablation_configs(config);
let rows = bench::run_token_ablation(cases, &configs)?;
let full_after = rows.first().map(|(_, _, a)| *a).unwrap_or(0) as f64;
let color = ui::color_stdout();
println!(
"{}",
ui::paint(
color,
Tone::Bold,
&format!(
"ablation — input tokens, offline, preset={preset}, {} cases",
cases.len()
)
)
);
let mut t = ui::table(
color,
&["config", "before", "after", "saved", "stage saves"],
);
for (label, before, after) in &rows {
let saved = ui::saved_pct(*before as f64, *after as f64);
let contribution = if label == "full" {
String::new()
} else {
format!("{:+.0} tok", *after as f64 - full_after)
};
t.add_row(vec![
comfy_table::Cell::new(label),
ui::right(before.to_string()),
ui::right(after.to_string()),
ui::right(ui::paint(color, Tone::Accent, &format!("{saved:.1}%"))),
ui::right(contribution),
]);
}
for line in t.to_string().lines() {
println!(" {line}");
}
Ok(())
}
fn quality_meta(args: &BenchArgs, config: &DenseConfig) -> serde_json::Value {
serde_json::json!({
"preset": args.preset,
"config_path": args.config.as_ref().map(|p| p.display().to_string()),
"config_resolved": config,
"model": args.model,
"judge_model": args.judge_model,
"route": args.route,
"reasoning_effort": args.reasoning_effort,
"corpus": args.corpus.display().to_string(),
})
}
fn run_offline(
cases: &[BenchCase],
kind: ProviderKind,
config: &DenseConfig,
args: &BenchArgs,
) -> Result<()> {
let (mut before, mut after) = (0usize, 0usize);
let mut rows = Vec::with_capacity(cases.len());
for c in cases {
let r = llmtrim_core::compress_with_config(&c.request, Some(kind), config)?;
before += r.input_tokens_before.0;
after += r.input_tokens_after.0;
rows.push(serde_json::json!({
"name": c.name,
"tokens_in_before": r.input_tokens_before.0,
"tokens_in_after": r.input_tokens_after.0,
}));
}
println!(
"offline: {} cases input {before} -> {after} tok ({:.1}% saved) preset={}",
cases.len(),
ui::saved_pct(before as f64, after as f64),
args.preset,
);
if let Some(path) = &args.json_out {
let result = serde_json::json!({
"n": cases.len(),
"tokens_in_before": before,
"tokens_in_after": after,
"tokens_in_saved_pct": ui::saved_pct(before as f64, after as f64),
"cases": rows,
});
let doc = bench::envelope::wrap("quality-offline-v1", quality_meta(args, config), result);
let json = serde_json::to_string_pretty(&doc).context("serialize offline results")?;
std::fs::write(path, json)
.with_context(|| format!("failed to write {}", path.display()))?;
eprintln!(
"{}",
ui::note(ui::color_stderr(), &format!("wrote {}", path.display()))
);
}
Ok(())
}
#[cfg(feature = "live")]
fn run_live(
cases: &[BenchCase],
kind: ProviderKind,
config: &DenseConfig,
args: &BenchArgs,
) -> Result<()> {
if kind != ProviderKind::OpenAi {
anyhow::bail!(
"live A/B bench only supports --provider openai (the OpenRouter endpoint is \
OpenAI-shaped; a {} request would be sent OpenAI-style and mis-parsed, scoring \
nothing). Use --offline or --ablate to measure {}-shaped input-token savings, \
or re-run with --provider openai.",
args.provider,
args.provider
);
}
if args.judge_model == args.model
&& cases
.iter()
.any(|c| matches!(c.scorer, bench::Scorer::LlmJudge))
{
eprintln!(
"{}",
ui::warn(
ui::color_stderr(),
&format!(
"--judge-model == --model ({}) — the model is grading its own answers; \
pass a different --judge-model for an unbiased judge.",
args.model
)
)
);
}
let table = std::fs::read_to_string(&args.pricing)
.map(|s| bench::load_pricing(&s))
.unwrap_or_default();
let price = bench::resolve_pricing(&table, &args.model);
let counter = llmtrim_core::tokenizer::counter_for(kind, Some(&args.model))?;
let api_key = dotenv_get("OPENROUTER_API_KEY")
.context("OPENROUTER_API_KEY not set (in env or a local .env)")?;
let llm = llmtrim::quality::OpenRouterModel::new(api_key, kind)?;
let scorer = bench::BenchScorer {
exec_timeout: 10,
judge: Some(&llm),
judge_model: args.judge_model.clone(),
};
let run = bench::run_ab(cases, config, &llm, counter.as_ref(), &scorer, price)?;
let color = ui::color_stdout();
let mut t = ui::table(color, &["case", "quality", "input", "output"]);
for o in &run.outcomes {
let q_tone = if o.quality_comp < o.quality_orig {
Tone::Warn
} else {
Tone::Accent
};
t.add_row(vec![
comfy_table::Cell::new(ui::truncate(&o.name, 24)),
ui::right(ui::paint(
color,
q_tone,
&format!("{:.2} → {:.2}", o.quality_orig, o.quality_comp),
)),
ui::right(format!("{} → {}", o.tokens_in_before, o.tokens_in_after)),
ui::right(format!("{} → {}", o.tokens_out_orig, o.tokens_out_comp)),
]);
}
for line in t.to_string().lines() {
println!(" {line}");
}
let f = bench::summarize(&run);
let cache_note = if f.cache_busted {
"cache busted (per-arm nonce; cache stage off)"
} else {
"cache stage under test"
};
let metric = |v: f64| ui::paint(color, Tone::Accent, &format!("{v:.1}%"));
let lines = vec![
ui::paint(
color,
Tone::Dim,
&format!(
"{} · model {} · judge {}",
args.corpus.display(),
args.model,
args.judge_model
),
),
String::new(),
format!("input saved {}", metric(f.tokens_in_saved_pct)),
format!("output saved {}", metric(f.tokens_out_saved_pct)),
format!("cost saved {}", metric(f.cost_saved_pct)),
format!("cache used {:.1}% ({cache_note})", f.cache_used_pct),
format!(
"quality {} → {} (retention {:+.1}pp, paired 95%CI ±{:.1})",
ui::paint(
color,
Tone::Bold,
&format!("{:.1}%", f.quality_orig.mean * 100.0)
),
ui::paint(
color,
Tone::Bold,
&format!("{:.1}%", f.quality_comp.mean * 100.0)
),
f.retention_pp,
f.retention_ci95_pp,
),
format!(
"cases {} scored, {} failed (compressed 4xx → 0), {} skipped (transient)",
f.n, f.failed, f.skipped
),
];
print!(
"\n{}",
ui::panel(
color,
&format!("bench · {} · n={}", args.preset, f.n),
&lines
)
);
if let Some(path) = &args.json_out {
let rows: Vec<_> = run
.outcomes
.iter()
.map(|o| {
serde_json::json!({
"name": o.name,
"quality_orig": o.quality_orig, "quality_comp": o.quality_comp,
"tokens_in_before": o.tokens_in_before, "tokens_in_after": o.tokens_in_after,
"tokens_out_orig": o.tokens_out_orig, "tokens_out_comp": o.tokens_out_comp,
"cached_in_orig": o.cached_in_orig, "cached_in_comp": o.cached_in_comp,
"cost_orig": o.cost_orig, "cost_comp": o.cost_comp,
})
})
.collect();
let meta = quality_meta(args, config);
let result = serde_json::json!({
"n": f.n,
"failed": f.failed,
"skipped": f.skipped,
"cache_busted": f.cache_busted,
"tokens_in_saved_pct": f.tokens_in_saved_pct,
"tokens_out_saved_pct": f.tokens_out_saved_pct,
"cost_saved_pct": f.cost_saved_pct,
"cache_used_pct": f.cache_used_pct,
"quality_orig": f.quality_orig.mean, "quality_comp": f.quality_comp.mean,
"retention_pp": f.retention_pp,
"retention_ci95_pp": f.retention_ci95_pp,
"cases": rows,
});
let doc = bench::envelope::wrap("quality-v1", meta, result);
let json = serde_json::to_string_pretty(&doc).context("serialize quality results")?;
std::fs::write(path, json)
.with_context(|| format!("failed to write {}", path.display()))?;
eprintln!(
"{}",
ui::note(ui::color_stderr(), &format!("wrote {}", path.display()))
);
}
Ok(())
}
#[cfg(not(feature = "live"))]
fn run_live(
_cases: &[BenchCase],
_kind: ProviderKind,
_config: &DenseConfig,
_args: &BenchArgs,
) -> Result<()> {
anyhow::bail!(
"live A/B bench requires the `live` feature: rebuild with \
`cargo run --features live -- bench …`. Use `--offline` (or `--ablate`) for \
token-only measurement, which needs no extra feature."
)
}
fn period_flag(daily: bool, weekly: bool, monthly: bool) -> Option<Period> {
if daily {
Some(Period::Day)
} else if weekly {
Some(Period::Week)
} else if monthly {
Some(Period::Month)
} else {
None
}
}
fn daemon_view(summary: &llmtrim::tracking::Summary) -> monitor::DaemonView {
use llmtrim::daemon;
let ca_present = matches!(llmtrim::serve::ca_cert_path(), Ok(p) if p.exists());
let env_port = llmtrim::setup::configured_port();
let autostart = llmtrim::autostart::is_enabled();
let log_path = daemon::logfile().ok().map(|p| p.display().to_string());
let last_request = summary.last_ts.as_deref().and_then(daemon::human_age);
let binary_version = env!("CARGO_PKG_VERSION").to_string();
match daemon::running() {
Some(s) => monitor::DaemonView {
running: true,
pid: s.pid,
port: s.port,
uptime: daemon::human_uptime(daemon::uptime_secs(s.started_at)),
uptime_secs: daemon::uptime_secs(s.started_at),
ca_present,
port_accepting: daemon::probe_port(s.port),
env_port,
autostart,
restarts: s.restarts,
version: s.version,
binary_version,
log_path,
last_request,
},
None => match env_port.filter(|&p| daemon::probe_port(p)) {
Some(p) => monitor::DaemonView {
running: true,
pid: 0,
port: p,
uptime: String::new(),
uptime_secs: 0,
ca_present,
port_accepting: true,
env_port,
autostart,
restarts: 0,
version: None,
binary_version,
log_path,
last_request,
},
None => monitor::DaemonView {
running: false,
pid: 0,
port: 0,
uptime: String::new(),
uptime_secs: 0,
ca_present,
port_accepting: false,
env_port,
autostart,
restarts: 0,
version: None,
binary_version,
log_path,
last_request,
},
},
}
}
fn render_snapshot(
tracker: &Tracker,
color: bool,
cache_included: bool,
split_marker: bool,
) -> Result<(String, monitor::Health)> {
let summary = tracker.summary()?;
let models = monitor::model_views(tracker)?;
let cost = monitor::monitor_cost(tracker);
let daemon = daemon_view(&summary);
let health = monitor::health(&daemon);
let trend: Vec<i64> = tracker
.by_period(Period::Day)
.unwrap_or_default()
.iter()
.rev()
.take(7)
.rev()
.map(|r| (r.input_before - r.input_after).max(0))
.collect();
let mut out = monitor::snapshot(
color,
Some(&daemon),
&summary,
&models,
cost.as_ref(),
monitor::today_saved_usd(tracker),
&trend,
cache_included,
split_marker,
);
if let Some(v) = llmtrim::update::check(false) {
out.push_str(&format!(
"\n {} llmtrim v{v} available — run llmtrim update\n",
ui::paint(color, Tone::Accent, "↑")
));
}
Ok((out, health))
}
#[cfg(feature = "breakdown")]
fn overview_data(tracker: &Tracker) -> llmtrim::breakdown::app::OverviewData {
use llmtrim::breakdown::app::{StatusKind, StatusLine};
monitor::overview_data(tracker, |summary, has_traffic| {
let daemon = daemon_view(summary);
let version_stale = daemon
.version
.as_deref()
.is_some_and(|v| v != daemon.binary_version);
if daemon.running && version_stale {
StatusLine {
kind: StatusKind::Stale,
text: "llmtrim is on, but running an older version — restart to apply the update"
.into(),
fix: Some("llmtrim start --force".into()),
uninstall: None,
}
} else if daemon.running {
let text = match daemon.last_request {
Some(ago) => format!("llmtrim is on and working · last request {ago}"),
None => "llmtrim is on and ready · waiting for your first request".into(),
};
StatusLine {
kind: if has_traffic {
StatusKind::Working
} else {
StatusKind::Ready
},
text,
fix: None,
uninstall: None,
}
} else if daemon.env_port.is_some() {
StatusLine {
kind: StatusKind::Off,
text: "llmtrim is OFF — your AI tools can't reach the API right now".into(),
fix: Some("llmtrim start".into()),
uninstall: Some("llmtrim uninstall".into()),
}
} else {
StatusLine {
kind: StatusKind::Degraded,
text: "llmtrim is not set up — your AI traffic isn't routed through it".into(),
fix: Some("llmtrim setup".into()),
uninstall: None,
}
}
})
}
fn merge_breakdown_json(s: String, include: bool) -> String {
if !include {
return s;
}
#[cfg(feature = "breakdown")]
if let Some(bd) = llmtrim::breakdown::export::breakdown_json() {
let mut v: serde_json::Value = match serde_json::from_str(&s) {
Ok(v) => v,
Err(e) => {
eprintln!("llmtrim: could not parse status json for breakdown merge: {e}");
return s;
}
};
if let Some(obj) = v.as_object_mut() {
obj.insert("breakdown".to_string(), bd);
match serde_json::to_string_pretty(&v) {
Ok(out) => return out,
Err(e) => eprintln!("llmtrim: could not merge breakdown into json: {e}"),
}
} else {
eprintln!("llmtrim: status json is not an object; skipped breakdown merge");
}
}
s
}
fn print_sessions_csv() {
#[cfg(feature = "breakdown")]
if let Some(sc) = llmtrim::breakdown::export::sessions_csv() {
println!();
print!("{sc}");
}
}
fn print_top_sources() {
#[cfg(feature = "breakdown")]
if let Ok(block) = llmtrim::breakdown::export::top_sources_report(ui::color_stdout(), 10) {
print!("{block}");
}
}
#[allow(clippy::too_many_arguments)]
fn run_monitor(
interval: u64,
daily: bool,
weekly: bool,
monthly: bool,
json: bool,
breakdown: bool,
csv: bool,
quiet: bool,
) -> Result<()> {
#[cfg(not(feature = "breakdown"))]
let _ = interval;
let tracker = Tracker::open().context("failed to open savings ledger")?;
if quiet {
let summary = tracker.summary()?;
let health = monitor::health(&daemon_view(&summary));
println!("{}", health.label());
std::process::exit(health.exit_code());
}
if let Some(period) = period_flag(daily, weekly, monthly) {
let rows = tracker.by_period(period)?;
if csv {
print!("{}", monitor::export_csv(&rows));
print_sessions_csv();
} else if json {
let s = tracker.summary()?;
let models = monitor::model_views(&tracker)?;
let daemon = daemon_view(&s);
let out = monitor::export_json(
&s,
&models,
monitor::monitor_cost(&tracker).as_ref(),
&rows,
Some(&daemon),
);
println!("{}", merge_breakdown_json(out, breakdown));
} else {
print!(
"{}",
monitor::period_report(ui::color_stdout(), period.label(), &rows)
);
print_top_sources();
}
return Ok(());
}
if json || csv {
if csv {
let rows = tracker.by_period(Period::Day)?;
print!("{}", monitor::export_csv(&rows));
print_sessions_csv();
} else {
let daemon = daemon_view(&tracker.summary()?);
println!(
"{}",
merge_breakdown_json(monitor::stats_json(&tracker, Some(&daemon))?, breakdown)
);
}
return Ok(());
}
#[cfg(feature = "breakdown")]
{
use std::io::IsTerminal as _;
if std::io::stdout().is_terminal() {
drop(tracker);
let tracker = llmtrim::tracking::db_path()
.ok()
.as_deref()
.and_then(|p| Tracker::open_reader_at(p).ok());
let sessions_db = llmtrim::breakdown::db::BreakdownDb::open().ok();
let snapshot = move || {
let ov = tracker.as_ref().map(overview_data).unwrap_or_default();
let rows = sessions_db
.as_ref()
.and_then(|d| d.sessions().ok())
.unwrap_or_default();
(ov, rows)
};
use llmtrim::breakdown::app::PostAction;
match llmtrim::breakdown::app::run(interval.max(2), snapshot)? {
PostAction::Doctor => {
let report = llmtrim::doctor::gather();
print!("{}", llmtrim::doctor::render(ui::color_stdout(), &report));
}
PostAction::Update => llmtrim::update::run()?,
PostAction::Restart => {
if let Err(e) = llmtrim::update::restart_daemon(ui::color_stdout()) {
eprintln!("{e:#}");
std::process::exit(1);
}
}
PostAction::None => {}
}
return Ok(());
}
}
let (out, health) = render_snapshot(&tracker, ui::color_stdout(), false, false)?;
print!("{out}");
if health != monitor::Health::Healthy {
use std::io::Write as _;
let _ = std::io::stdout().flush();
std::process::exit(health.exit_code());
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use std::time::{SystemTime, UNIX_EPOCH};
#[test]
fn status_still_accepts_deprecated_watch_flag() {
let cli = Cli::try_parse_from(["llmtrim", "status", "--watch"])
.expect("status --watch must still parse for backward compatibility");
match cli.command {
Commands::Monitor { watch, .. } => assert!(watch),
_ => panic!("expected status to parse as the Monitor command"),
}
}
#[test]
fn breakdown_section_is_omitted_unless_requested() {
let base = r#"{"daemon":{"running":true}}"#.to_string();
assert_eq!(merge_breakdown_json(base.clone(), false), base);
}
#[test]
fn alive_subcommand_name_is_stable_and_hidden() {
let cli = Cli::try_parse_from(["llmtrim", "_alive"]).expect("`_alive` must parse");
assert!(matches!(cli.command, Commands::Alive));
let help = {
use clap::CommandFactory;
let mut out = Vec::new();
Cli::command().write_long_help(&mut out).unwrap();
String::from_utf8(out).unwrap()
};
assert!(
!help.contains("_alive"),
"`_alive` must stay hidden from help"
);
}
#[test]
fn breakdown_flag_requires_json() {
assert!(
Cli::try_parse_from(["llmtrim", "status", "--breakdown"]).is_err(),
"--breakdown without --json must be rejected"
);
Cli::try_parse_from(["llmtrim", "status", "--json", "--breakdown"])
.expect("status --json --breakdown must parse");
}
fn scratch_dir(tag: &str) -> PathBuf {
let nanos = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos();
let dir = std::env::temp_dir().join(format!("llmtrim_bench_{tag}_{nanos}"));
std::fs::create_dir_all(&dir).unwrap();
dir
}
#[test]
fn load_agent_tasks_injects_shared_tools_and_skips_underscore_files() {
let dir = scratch_dir("dedup");
std::fs::write(
dir.join("_tools.json"),
r#"[{"type":"function","function":{"name":"grep"}}]"#,
)
.unwrap();
std::fs::write(
dir.join("a.json"),
r#"{"id":"a","model":"m","system":"s","user":"u"}"#,
)
.unwrap();
std::fs::write(
dir.join("b.json"),
r#"{"id":"b","model":"m","system":"s","user":"u","tools":[{"type":"function","function":{"name":"own"}}]}"#,
)
.unwrap();
let tasks = load_agent_tasks(std::slice::from_ref(&dir)).unwrap();
assert_eq!(tasks.len(), 2, "_tools.json is a fragment, not a task");
let a = tasks.iter().find(|t| t.id == "a").unwrap();
let b = tasks.iter().find(|t| t.id == "b").unwrap();
assert_eq!(
a.tools.pointer("/0/function/name").unwrap(),
"grep",
"shared catalog injected when tools omitted"
);
assert_eq!(
b.tools.pointer("/0/function/name").unwrap(),
"own",
"task's own tools left intact"
);
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn suite_matrix_covers_the_documented_corpora() {
assert_eq!(
SUITE_MATRIX.len(),
11,
"eight workload corpora + three named benchmarks"
);
let cnn = SUITE_MATRIX.iter().find(|(c, _, _)| *c == "cnn").unwrap();
assert_eq!(cnn.2, 8, "cnn runs fewer cases (long docs)");
for (corpus, preset, n) in SUITE_MATRIX {
assert!(*n > 0, "{corpus}: case count must be positive");
assert!(
DenseConfig::preset(preset).is_some(),
"{corpus}: unknown preset '{preset}'"
);
}
}
#[test]
fn prepare_request_pins_model_route_and_reasoning() {
let base = r#"{"messages":[{"role":"user","content":"hi"}]}"#;
let out = prepare_request(base, "openai/gpt-oss-20b", "wandb/fp4", "medium");
let v: serde_json::Value = serde_json::from_str(&out).unwrap();
assert_eq!(v["model"], "openai/gpt-oss-20b");
assert_eq!(v["provider"]["order"], serde_json::json!(["wandb"]));
assert_eq!(v["provider"]["quantizations"], serde_json::json!(["fp4"]));
assert_eq!(v["reasoning"]["effort"], "medium");
}
#[test]
fn prepare_request_omits_reasoning_when_effort_is_empty() {
let base = r#"{"messages":[{"role":"user","content":"hi"}]}"#;
let out = prepare_request(base, "openai/gpt-4o-mini", "", "");
let v: serde_json::Value = serde_json::from_str(&out).unwrap();
assert!(v.get("reasoning").is_none(), "no reasoning field: {v}");
assert!(v.get("provider").is_none(), "no provider field: {v}");
}
const CORPUS_LINE: &str = r#"{"context":"The cat sat on the red mat.","question":"What color was the mat?","gold":"red","scorer":"token_f1"}"#;
fn offline_args(corpus: PathBuf, json_out: Option<PathBuf>) -> BenchArgs {
BenchArgs {
corpus,
provider: "openai".to_string(),
preset: "rag".to_string(),
model: "openai/gpt-4o-mini".to_string(),
judge_model: "openai/gpt-4o-mini".to_string(),
route: String::new(),
reasoning_effort: String::new(),
n: Some(1),
config: None,
pricing: PathBuf::new(), offline: true,
ablate: false,
json_out,
}
}
#[test]
fn offline_quality_writes_a_valid_enveloped_file() {
let dir = scratch_dir("offline");
let corpus = dir.join("c.jsonl");
std::fs::write(&corpus, format!("{CORPUS_LINE}\n")).unwrap();
let out = dir.join("out.json");
run_bench(offline_args(corpus, Some(out.clone()))).unwrap();
let doc: serde_json::Value =
serde_json::from_str(&std::fs::read_to_string(&out).unwrap()).unwrap();
assert_eq!(doc["schema"], "llmtrim-bench/quality-offline-v1");
assert_eq!(doc["meta"]["preset"], "rag");
assert_eq!(doc["result"]["n"], 1);
assert_eq!(doc["result"]["cases"].as_array().unwrap().len(), 1);
assert!(doc["result"]["tokens_in_before"].is_number());
assert!(doc["meta"]["model"].is_string());
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn suite_runs_offline_and_skips_missing_corpora() {
let data = scratch_dir("suite_data");
let out = scratch_dir("suite_out");
std::fs::write(data.join("gsm8k.jsonl"), format!("{CORPUS_LINE}\n")).unwrap();
let args = SuiteArgs {
data_dir: data.clone(),
out: out.clone(),
model: "openai/gpt-4o-mini".to_string(),
judge_model: "openai/gpt-4o-mini".to_string(),
route: String::new(),
pricing: PathBuf::new(), n: Some(1),
offline: true,
};
run_bench_suite(args).expect("missing corpora must not abort the matrix");
assert!(
out.join("gsm8k.json").is_file(),
"present corpus produced a result"
);
assert!(
!out.join("cnn.json").exists(),
"absent corpus left no result"
);
std::fs::remove_dir_all(&data).unwrap();
std::fs::remove_dir_all(&out).unwrap();
}
#[test]
fn latency_runs_on_the_builtin_fixture() {
let r = run_bench_latency(LatencyArgs {
request: None,
provider: None,
iterations: 3,
});
assert!(
r.is_ok(),
"latency on the built-in fixture should not error: {r:?}"
);
}
#[test]
fn compare_rejects_an_unknown_tool() {
let err = run_bench_compare(CompareArgs {
tool: "nope".to_string(),
live: false,
extra: vec![],
})
.unwrap_err();
assert!(err.to_string().contains("unknown comparator"), "{err}");
}
}