use std::io::{Read, Write};
use std::path::PathBuf;
use std::str::FromStr;
use anyhow::{Context, Result, bail};
use clap::{Parser, Subcommand};
use llmtrim::bench::{self, BenchCase};
use llmtrim::monitor;
use llmtrim::tracking::{Period, Record, Tracker};
use llmtrim::transport::Endpoint;
use llmtrim::ui::{self, Tone};
use llmtrim_core::config::DenseConfig;
use llmtrim_core::ir::ProviderKind;
const HELP_STYLES: clap::builder::Styles = clap::builder::Styles::styled()
.header(
clap::builder::styling::AnsiColor::BrightBlue
.on_default()
.bold(),
)
.usage(
clap::builder::styling::AnsiColor::BrightBlue
.on_default()
.bold(),
)
.literal(
clap::builder::styling::AnsiColor::BrightCyan
.on_default()
.bold(),
)
.placeholder(clap::builder::styling::AnsiColor::Cyan.on_default());
const HELP_TEMPLATE: &str = "\
{about-with-newline}
{usage-heading} {usage}
Get started:
setup Set everything up and start saving (CA, env, autostart, integrations, daemon)
status Show the savings dashboard + interceptor health [aliases: monitor, gain]
update Update to the latest release and refresh integrations
ensure Bring this machine to the recommended current state
wrap Launch an agent (claude, codex, …) routed through the interceptor
sub Reroute Claude Code to another subscription's backend (codex|kimi|grok)
tray Open the desktop tray app (savings menu-bar / system-tray)
Daemon:
start Start the background interceptor (no-op if already running)
stop Stop the background interceptor
autostart Run the interceptor at login (--off to disable)
When something's wrong:
doctor Check the install end-to-end; `--fix` applies repairs
uninstall Undo everything `setup` did
Pipes & one-shots:
compress Compress a request from stdin to stdout
send Compress a request, send it to the provider, print the response
serve Run the HTTPS interceptor in the foreground
ca Print the local CA certificate path and how to trust it
Measurement (dev):
eval Measure retrieval recall + token savings on a corpus
bench A/B benchmark: tokens saved vs quality retained, on a real model
discover Scan the capture corpus for where tokens still escape compression
Options:
{options}
Run `llmtrim help <command>` for details on any command.
";
#[derive(Parser)]
#[command(
name = "llmtrim",
version,
about = "Static, deterministic LLM prompt/payload compressor",
styles = HELP_STYLES,
help_template = HELP_TEMPLATE
)]
struct Cli {
#[command(subcommand)]
command: Commands,
}
#[derive(Subcommand)]
enum Commands {
Compress {
#[arg(long)]
provider: Option<String>,
},
Send {
#[arg(long)]
provider: Option<String>,
},
Sub {
#[command(subcommand)]
action: SubCmd,
},
#[command(hide = true)]
Compact {
#[command(subcommand)]
action: CompactCmd,
},
Serve {
#[arg(long, default_value_t = llmtrim::setup::DEFAULT_PORT)]
port: u16,
#[arg(long)]
force: bool,
#[arg(long, hide = true)]
supervised: bool,
#[arg(long, hide = true)]
hide_console: bool,
},
Setup {
#[arg(long)]
port: Option<u16>,
#[arg(long)]
force: bool,
#[arg(long, conflicts_with = "force")]
env: bool,
},
Ensure {
#[arg(long, short = 'q')]
quiet: bool,
},
Uninstall {
#[arg(long)]
purge: bool,
#[arg(long)]
keep_binary: bool,
},
Start {
#[arg(long)]
port: Option<u16>,
#[arg(long)]
force: bool,
},
Wrap {
#[arg(trailing_var_arg = true, allow_hyphen_values = true)]
args: Vec<String>,
},
Stop,
#[command(name = "_alive", hide = true)]
Alive,
Update,
#[command(name = "status", visible_aliases = ["monitor", "gain"])]
Monitor {
#[arg(long, hide = true)]
watch: bool,
#[arg(long, default_value_t = 2)]
interval: u64,
#[arg(long)]
daily: bool,
#[arg(long)]
weekly: bool,
#[arg(long)]
monthly: bool,
#[arg(long)]
json: bool,
#[arg(long, requires = "json")]
breakdown: bool,
#[arg(long)]
csv: bool,
#[arg(long, short, conflicts_with_all = ["daily", "weekly", "monthly", "json", "csv"])]
quiet: bool,
},
Mcp {
#[command(subcommand)]
action: Option<McpAction>,
},
#[command(hide = true)]
Statusline {
#[command(subcommand)]
action: Option<StatuslineCmd>,
},
#[command(hide = true)]
Guard {
#[command(subcommand)]
action: Option<GuardCmd>,
},
Doctor {
#[arg(long)]
fix: bool,
},
Autostart {
#[arg(long)]
off: bool,
#[arg(long)]
port: Option<u16>,
#[arg(long)]
tray: bool,
#[arg(long)]
status: bool,
},
Tray,
Ca {
#[arg(long)]
pem: bool,
},
Eval {
#[arg(long)]
corpus: PathBuf,
#[arg(long, default_value = "openai")]
provider: String,
#[arg(long, default_value_t = 0.5)]
keep_ratio: f64,
},
Discover {
#[arg(long)]
dir: Option<PathBuf>,
#[arg(long)]
by_tool: bool,
#[arg(long)]
limit: Option<usize>,
#[arg(long)]
json: bool,
},
#[command(subcommand)]
Bench(BenchCmd),
#[command(hide = true)]
WindowSub {
#[arg(trailing_var_arg = true)]
args: Vec<String>,
},
}
#[derive(clap::Subcommand)]
enum BenchCmd {
Quality(Box<BenchArgs>),
Suite(Box<SuiteArgs>),
Agent(Box<BenchAgentArgs>),
Latency(LatencyArgs),
Compare(CompareArgs),
}
#[derive(Subcommand)]
enum SubCmd {
Setup {
provider: String,
},
#[command(visible_alias = "use", alias = "start")]
On {
provider: Option<String>,
#[arg(long)]
no_restart: bool,
},
#[command(alias = "stop")]
Off {
#[arg(long)]
no_restart: bool,
},
Mode {
mode: String,
#[arg(long)]
no_restart: bool,
},
Chain {
providers: String,
#[arg(long)]
no_restart: bool,
},
Effort {
level: String,
#[arg(long)]
no_restart: bool,
},
Map {
provider: String,
from: String,
to: String,
#[arg(long)]
no_restart: bool,
},
Unmap {
provider: String,
from: String,
#[arg(long)]
no_restart: bool,
},
Models {
provider: String,
#[arg(long)]
json: bool,
},
Status {
#[arg(long)]
json: bool,
},
Auth {
provider: String,
#[command(subcommand)]
action: AuthAction,
},
}
#[derive(Subcommand)]
enum CompactCmd {
Models {
#[arg(required = true)]
models: Vec<String>,
#[arg(long)]
no_restart: bool,
},
Off {
#[arg(long)]
no_restart: bool,
},
Status {
#[arg(long)]
json: bool,
},
}
#[derive(Subcommand)]
enum AuthAction {
Login,
Device,
Status {
#[arg(long)]
json: bool,
},
Logout,
}
#[derive(Subcommand)]
enum McpAction {
Install {
#[arg(long)]
print: bool,
#[arg(long)]
force: bool,
},
}
#[derive(Subcommand)]
enum StatuslineCmd {
Install {
#[arg(long)]
print: bool,
},
Uninstall,
}
#[derive(Subcommand)]
enum GuardCmd {
Install {
#[arg(long)]
print: bool,
},
Uninstall,
}
#[derive(clap::Args)]
struct BenchAgentArgs {
#[arg(long = "tasks", default_value = "bench/agent")]
tasks: Vec<PathBuf>,
#[arg(long, default_value = "baseline,agent")]
conditions: String,
#[arg(long, default_value_t = 1)]
repeats: usize,
#[arg(long)]
model: Option<String>,
#[arg(long, default_value = "bench/pricing.json")]
pricing: PathBuf,
#[arg(long)]
live: bool,
#[arg(long, default_value_t = 2)]
tool_turns: usize,
#[arg(long)]
json_out: Option<PathBuf>,
}
#[derive(clap::Args)]
struct BenchArgs {
#[arg(long)]
corpus: PathBuf,
#[arg(long, default_value = "openai")]
provider: String,
#[arg(long, default_value = "auto")]
preset: String,
#[arg(long, default_value = "openai/gpt-oss-20b")]
model: String,
#[arg(long, default_value = "openai/gpt-4o-mini")]
judge_model: String,
#[arg(long, default_value = "groq")]
route: String,
#[arg(long, default_value = "")]
reasoning_effort: String,
#[arg(long)]
n: Option<usize>,
#[arg(long)]
config: Option<PathBuf>,
#[arg(long, default_value = "bench/pricing.json")]
pricing: PathBuf,
#[arg(long)]
offline: bool,
#[arg(long)]
ablate: bool,
#[arg(long)]
json_out: Option<PathBuf>,
}
#[derive(clap::Args)]
struct SuiteArgs {
#[arg(long, default_value = "bench/data")]
data_dir: PathBuf,
#[arg(long, default_value = "bench/results")]
out: PathBuf,
#[arg(long, default_value = "openai/gpt-oss-20b")]
model: String,
#[arg(long, default_value = "openai/gpt-4o-mini")]
judge_model: String,
#[arg(long, default_value = "groq")]
route: String,
#[arg(long, default_value = "bench/pricing.json")]
pricing: PathBuf,
#[arg(long)]
n: Option<usize>,
#[arg(long)]
offline: bool,
}
#[derive(clap::Args)]
struct LatencyArgs {
#[arg(long)]
request: Option<PathBuf>,
#[arg(long)]
provider: Option<String>,
#[arg(long, default_value_t = 100)]
iterations: usize,
}
#[derive(clap::Args)]
struct CompareArgs {
tool: String,
#[arg(long)]
live: bool,
#[arg(last = true)]
extra: Vec<String>,
}
fn read_stdin() -> Result<String> {
let mut buf = String::new();
std::io::stdin()
.read_to_string(&mut buf)
.context("failed to read stdin")?;
Ok(buf)
}
#[cfg(feature = "intercept")]
fn window_sub_provider(requested: Option<&str>) -> Result<String> {
use llmtrim::reroute::SubProvider;
let configured = match requested {
Some(raw) => {
let p = SubProvider::parse(raw).ok_or_else(|| {
anyhow::anyhow!("unknown provider '{raw}' (codex|kimi|grok)")
})?;
p.as_str().to_string()
}
None => llmtrim_core::config::RuntimeConfig::get()
.sub
.clone()
.or_else(llmtrim_core::config::sub_reenable_provider)
.context(
"no provider to re-enable — pass one (`/sub on grok`) or run `llmtrim sub on codex|kimi|grok` first",
)?,
};
let provider =
SubProvider::parse(&configured).context("configured subscription provider is invalid")?;
let logged_in = llmtrim::reroute::auth::auth_status_json(provider)
.get("logged_in")
.and_then(serde_json::Value::as_bool)
.unwrap_or(false);
if !logged_in {
bail!(
"{} is not signed in — run `llmtrim sub auth {} login` first",
provider.as_str(),
provider.as_str()
);
}
Ok(provider.as_str().to_string())
}
#[cfg(not(feature = "intercept"))]
fn window_sub_provider(_requested: Option<&str>) -> Result<String> {
bail!("window-scoped rerouting needs the `intercept` feature")
}
fn run_window_sub(args: Vec<String>) -> Result<()> {
let action = args.first().map(String::as_str).unwrap_or("status");
let session_from_env = || {
std::env::var("CLAUDE_CODE_SESSION_ID")
.ok()
.or_else(|| std::env::var("CLAUDE_SESSION_ID").ok())
};
match action {
"install" => {
llmtrim::window_sub::install(&llmtrim::statusline::stable_exe_string())?;
if let Err(e) = llmtrim::ensure::set_opt_out("window_sub", false) {
eprintln!("llmtrim: could not clear /sub opt-out: {e:#}");
}
println!("Claude Code /sub integration installed.");
}
"uninstall" => {
llmtrim::window_sub::uninstall()?;
if let Err(e) = llmtrim::ensure::set_opt_out("window_sub", true) {
eprintln!("llmtrim: could not record /sub opt-out: {e:#}");
}
println!("Claude Code /sub integration removed.");
}
"hook-start" => {
let mut raw = String::new();
std::io::stdin().read_to_string(&mut raw)?;
let value: serde_json::Value = serde_json::from_str(&raw)?;
let session = value
.get("session_id")
.and_then(serde_json::Value::as_str)
.context("SessionStart missing session_id")?;
let source = value
.get("source")
.and_then(serde_json::Value::as_str)
.unwrap_or("startup");
let existing = std::env::var("LLMTRIM_CLAUDE_WINDOW_TOKEN").ok();
let token = llmtrim::window_sub::session_start(session, source, existing.as_deref())?;
if let Ok(file) = std::env::var("CLAUDE_ENV_FILE") {
let mut f = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(file)?;
writeln!(f, "export LLMTRIM_CLAUDE_WINDOW_TOKEN={token}")?;
}
}
"hook-end" => {
let mut raw = String::new();
std::io::stdin().read_to_string(&mut raw)?;
let v: serde_json::Value = serde_json::from_str(&raw)?;
if let Some(s) = v.get("session_id").and_then(serde_json::Value::as_str) {
let reason = v
.get("reason")
.and_then(serde_json::Value::as_str)
.unwrap_or("");
let _ = llmtrim::window_sub::session_end(s, reason);
}
}
"slash" => {
let raw = args.get(1).map(String::as_str).unwrap_or("status");
let parts: Vec<&str> = raw.split_whitespace().collect();
let session = args
.get(2)
.filter(|x| !x.is_empty())
.cloned()
.or_else(session_from_env)
.context("Claude Code session unavailable")?;
match parts.as_slice() {
["on"] => {
let preferred = llmtrim::window_sub::last_provider(&session);
let provider = match preferred.as_deref() {
Some(p) => window_sub_provider(Some(p))?,
None => window_sub_provider(None)?,
};
llmtrim::window_sub::set(&session, true, Some(&provider))?;
println!("Subscription rerouting enabled for this window ({provider}).");
}
["on", provider_name] => {
let provider = window_sub_provider(Some(provider_name))?;
llmtrim::window_sub::set(&session, true, Some(&provider))?;
println!("Subscription rerouting enabled for this window ({provider}).");
}
["off"] => {
llmtrim::window_sub::set(&session, false, None)?;
println!(
"Subscription rerouting disabled for this window; Anthropic compression remains active."
);
}
["status"] | [] => match llmtrim::window_sub::status(&session)? {
Some(llmtrim::window_sub::Intent::Enabled { provider }) => {
println!("This window: subscription rerouting enabled ({provider}).")
}
Some(llmtrim::window_sub::Intent::Disabled) => {
println!("This window: subscription rerouting disabled.")
}
None => println!("This window follows the global subscription policy."),
},
_ => bail!("usage: /sub on [codex|kimi|grok] | off | status"),
}
}
_ => bail!("unknown window-sub action"),
};
Ok(())
}
fn main() {
if let Err(e) = run() {
eprint!("{}", ui::render_error(ui::color_stderr(), &e));
std::process::exit(1);
}
}
#[cfg(feature = "intercept")]
fn run_auth(provider: &str, action: AuthAction) -> Result<()> {
use llmtrim::reroute::{SubProvider, auth};
if let AuthAction::Status { json: true } = action {
let p = SubProvider::parse(provider).ok_or_else(|| anyhow::anyhow!("unknown provider"))?;
println!("{}", auth::auth_status_json(p));
return Ok(());
}
match (provider, action) {
("codex", AuthAction::Login) => auth::codex_login(),
("codex", AuthAction::Device) => auth::codex_device(),
("codex", AuthAction::Status { .. }) => auth::codex_status(),
("codex", AuthAction::Logout) => auth::codex_logout(),
("kimi", AuthAction::Login) => auth::kimi_login(),
("kimi", AuthAction::Device) => {
anyhow::bail!("kimi sign-in is device-code already — use `llmtrim sub auth kimi login`")
}
("kimi", AuthAction::Status { .. }) => auth::kimi_status(),
("kimi", AuthAction::Logout) => auth::kimi_logout(),
("grok", AuthAction::Login) => auth::grok_login(),
("grok", AuthAction::Device) => auth::grok_device(),
("grok", AuthAction::Status { .. }) => auth::grok_status(),
("grok", AuthAction::Logout) => auth::grok_logout(),
(other, _) => anyhow::bail!("unknown provider '{other}' (codex|kimi|grok)"),
}
}
#[cfg(feature = "intercept")]
fn print_reroute_enabled(p: llmtrim::reroute::SubProvider) -> bool {
let logged_in =
llmtrim::reroute::auth::auth_status_json(p)["logged_in"].as_bool() == Some(true);
if logged_in {
println!("Reroute enabled: {}.", p.as_str());
} else {
println!(
"Reroute enabled: {}. Sign in with `llmtrim sub auth {} login`, then \
`llmtrim start --force`.",
p.as_str(),
p.as_str()
);
}
logged_in
}
#[cfg(feature = "intercept")]
fn apply_sub_change(no_restart: bool) {
let Some(state) = llmtrim::daemon::running() else {
return;
};
if no_restart {
println!("Restart to apply: `llmtrim start --force`.");
return;
}
let port = state.port;
let restarted = matches!(llmtrim::daemon::stop_and_wait_free(port), Ok(true))
.then(|| llmtrim::daemon::spawn_detached(port).ok())
.flatten();
match restarted {
Some(pid) => println!("↻ Restarted interceptor (pid {pid}) to apply."),
None => eprintln!(
"llmtrim: change saved, but the restart failed — run `llmtrim start --force`."
),
}
}
#[cfg(feature = "intercept")]
fn apply_sub_map_change(edited: llmtrim::reroute::SubProvider, no_restart: bool) {
let active = llmtrim_core::config::RuntimeConfig::get()
.sub
.as_deref()
.and_then(llmtrim::reroute::SubProvider::parse);
if active == Some(edited) {
apply_sub_change(no_restart);
}
}
#[cfg(feature = "intercept")]
fn run_compact(action: CompactCmd) -> Result<()> {
match action {
CompactCmd::Models { models, no_restart } => {
let mut normalized = Vec::new();
for model in models
.iter()
.flat_map(|model| model.split(','))
.map(str::trim)
.filter(|model| !model.is_empty())
{
if !normalized.iter().any(|existing| existing == model) {
normalized.push(model.to_string());
}
}
if normalized.is_empty() {
anyhow::bail!("compact model list is empty; use `llmtrim compact off` to disable");
}
llmtrim_core::config::write_compact_models(&normalized)?;
if let Err(e) = llmtrim::ensure::set_opt_out("compact", false) {
eprintln!("llmtrim: could not clear compact opt-out: {e:#}");
}
println!(
"Compact models: {} -> original model (implicit).",
normalized.join(" -> ")
);
apply_sub_change(no_restart);
Ok(())
}
CompactCmd::Off { no_restart } => {
llmtrim_core::config::write_compact_models(&[])?;
if let Err(e) = llmtrim::ensure::set_opt_out("compact", true) {
eprintln!("llmtrim: could not record compact opt-out: {e:#}");
}
println!("Compact model substitution disabled; `/compact` uses the original model.");
apply_sub_change(no_restart);
Ok(())
}
CompactCmd::Status { json } => {
let cfg = llmtrim_core::config::RuntimeConfig::get();
let sub = cfg
.sub
.as_deref()
.and_then(llmtrim::reroute::SubProvider::parse);
let original = "<original model>";
let plan = llmtrim::compact::plan(&cfg.compact_models, original, sub, &cfg.sub_tiers);
if json {
let out = serde_json::json!({
"models": cfg.compact_models,
"original_fallback": true,
"resolved": plan.iter().map(|candidate| serde_json::json!({
"logical": candidate.logical_model,
"upstream": candidate.upstream_model,
"context_window": llmtrim_core::context_window(&candidate.upstream_model),
"original": candidate.is_original,
})).collect::<Vec<_>>(),
});
println!("{}", serde_json::to_string_pretty(&out)?);
} else if cfg.compact_models.is_empty() {
println!("Compact models: off (original model only).");
} else {
println!("Compact models: {}", cfg.compact_models.join(" -> "));
for candidate in plan.iter().filter(|candidate| !candidate.is_original) {
let window = llmtrim_core::context_window(&candidate.upstream_model)
.map(|tokens| format!("{tokens} tokens"))
.unwrap_or_else(|| "unknown context".to_string());
println!(
" {} -> {} ({window})",
candidate.logical_model, candidate.upstream_model
);
}
println!(" then the original requested model (implicit)");
}
Ok(())
}
}
}
#[cfg(feature = "intercept")]
fn run_sub(action: SubCmd) -> Result<()> {
use llmtrim::reroute::{SubProvider, Tier, default_codex_tier_model};
let parse = |p: &str| {
SubProvider::parse(p)
.ok_or_else(|| anyhow::anyhow!("unknown provider '{p}' (codex|kimi|grok)"))
};
match action {
SubCmd::Setup { provider } => {
let p = parse(&provider)?;
#[cfg(feature = "breakdown")]
{
llmtrim::reroute::tui::run(p)
}
#[cfg(not(feature = "breakdown"))]
{
let _ = p;
anyhow::bail!("the mapping editor needs the `breakdown` feature")
}
}
SubCmd::On {
provider,
no_restart,
} => {
let logged_in = match provider {
Some(provider) => {
let p = parse(&provider)?;
let mut map = std::collections::BTreeMap::new();
match p {
SubProvider::Codex => {
for t in Tier::ALL {
map.insert(
t.as_str().to_string(),
default_codex_tier_model(t).to_string(),
);
}
}
SubProvider::Grok => {
for t in Tier::ALL {
map.insert(
t.as_str().to_string(),
llmtrim::reroute::default_grok_tier_model(t).to_string(),
);
}
}
SubProvider::Kimi => {}
_ => {}
}
llmtrim_core::config::write_sub_mapping(p.as_str(), &map)?;
print_reroute_enabled(p)
}
None => {
let provider = llmtrim_core::config::sub_reenable_provider().ok_or_else(|| {
anyhow::anyhow!(
"no provider to re-enable — run `llmtrim sub on codex` (or kimi|grok) first"
)
})?;
let p = parse(&provider)?;
llmtrim_core::config::enable_sub(p.as_str())?;
print_reroute_enabled(p)
}
};
if logged_in {
apply_sub_change(no_restart);
}
Ok(())
}
SubCmd::Off { no_restart } => {
llmtrim_core::config::disable_sub()?;
println!("Reroute disabled — traffic goes to Anthropic (compression only).");
apply_sub_change(no_restart);
Ok(())
}
SubCmd::Mode { mode, no_restart } => {
let fallback = match mode.trim().to_ascii_lowercase().as_str() {
"fallback" | "on-error" | "on_error" | "onerror" => true,
"always" => false,
other => anyhow::bail!("unknown mode '{other}' (always|fallback)"),
};
llmtrim_core::config::write_sub_mode(fallback)?;
println!(
"Reroute mode: {}.",
if fallback { "fallback" } else { "always" }
);
apply_sub_change(no_restart);
Ok(())
}
SubCmd::Chain {
providers,
no_restart,
} => {
let mut chain = Vec::new();
for raw in providers.split(',') {
let raw = raw.trim();
if raw.is_empty() {
continue;
}
let p = parse(raw)?;
if !chain.contains(&p.as_str().to_string()) {
chain.push(p.as_str().to_string());
}
}
if chain.is_empty() {
anyhow::bail!("fallback chain is empty (expected codex,kimi)");
}
llmtrim_core::config::write_sub_chain(&chain)?;
println!("Fallback chain: {}.", chain.join(" -> "));
apply_sub_change(no_restart);
Ok(())
}
SubCmd::Effort { level, no_restart } => {
let level = match level.trim().to_ascii_lowercase().as_str() {
"max" => "xhigh".to_string(),
l @ ("none" | "low" | "medium" | "high" | "xhigh") => l.to_string(),
other => anyhow::bail!("unknown effort '{other}' (none|low|medium|high|xhigh)"),
};
llmtrim_core::config::write_sub_effort(SubProvider::Codex.as_str(), &level)?;
println!("Codex reasoning effort: {level}.");
let active = llmtrim_core::config::RuntimeConfig::get()
.sub
.as_deref()
.and_then(SubProvider::parse);
if active == Some(SubProvider::Codex) {
apply_sub_change(no_restart);
} else {
eprintln!(
"note: reroute is not on codex right now, so this effort has no effect until \
`llmtrim sub on codex`."
);
}
Ok(())
}
SubCmd::Map {
provider,
from,
to,
no_restart,
} => {
let p = parse(&provider)?;
llmtrim_core::config::write_sub_map_entry(p.as_str(), &from, &to)?;
println!("Mapped {from} -> {to} for {}.", p.as_str());
apply_sub_map_change(p, no_restart);
Ok(())
}
SubCmd::Unmap {
provider,
from,
no_restart,
} => {
let p = parse(&provider)?;
llmtrim_core::config::remove_sub_map_entry(p.as_str(), &from)?;
println!(
"Unmapped {from} for {} (back to the preset default).",
p.as_str()
);
apply_sub_map_change(p, no_restart);
Ok(())
}
SubCmd::Models { provider, json } => {
let p = parse(&provider)?;
let ids: Vec<String> = llmtrim::reroute::catalog::models_for(p)
.into_iter()
.map(|e| e.id)
.collect();
if json {
println!("{}", serde_json::to_string(&ids)?);
} else {
for id in ids {
println!("{id}");
}
}
Ok(())
}
SubCmd::Status { json } => {
let cfg = llmtrim_core::config::RuntimeConfig::get();
let active = cfg.sub.as_deref().and_then(SubProvider::parse);
if json {
let mut mapping: std::collections::BTreeMap<String, String> = match active {
Some(SubProvider::Codex) => Tier::ALL
.iter()
.map(|t| {
(
t.as_str().to_string(),
default_codex_tier_model(*t).to_string(),
)
})
.collect(),
Some(SubProvider::Grok) => Tier::ALL
.iter()
.map(|t| {
(
t.as_str().to_string(),
llmtrim::reroute::default_grok_tier_model(*t).to_string(),
)
})
.collect(),
_ => std::collections::BTreeMap::new(),
};
for (k, v) in &cfg.sub_tiers {
mapping.insert(k.clone(), v.clone());
}
let out = serde_json::json!({
"provider": active.map(|p| p.as_str()),
"mode": if cfg.sub_fallback { "fallback" } else { "always" },
"chain": cfg.sub_chain,
"effort": cfg.sub_effort,
"mapping": mapping,
"auth": {
"codex": llmtrim::reroute::auth::auth_status_json(SubProvider::Codex),
"kimi": llmtrim::reroute::auth::auth_status_json(SubProvider::Kimi),
"grok": llmtrim::reroute::auth::auth_status_json(SubProvider::Grok),
},
});
println!("{}", serde_json::to_string_pretty(&out)?);
return Ok(());
}
match active {
None => println!("Reroute: off"),
Some(p) => {
println!(
"Reroute: {} ({})",
p.as_str(),
if cfg.sub_fallback {
"fallback"
} else {
"always"
}
);
if cfg.sub_fallback {
println!(
" chain -> {}",
if cfg.sub_chain.is_empty() {
p.as_str().to_string()
} else {
cfg.sub_chain.join(" -> ")
}
);
}
match p {
SubProvider::Codex => {
println!(
" effort -> {}",
cfg.sub_effort.as_deref().unwrap_or("none")
);
for t in Tier::ALL {
let model = cfg
.sub_tiers
.get(t.as_str())
.cloned()
.unwrap_or_else(|| default_codex_tier_model(t).to_string());
println!(" {:<7} -> {model}", t.as_str());
}
let tier_keys: [&str; 4] = Tier::ALL.map(|t| t.as_str());
for (from, to) in &cfg.sub_tiers {
if !tier_keys.contains(&from.as_str()) {
println!(" {from} -> {to}");
}
}
}
SubProvider::Grok => {
for t in Tier::ALL {
let model =
cfg.sub_tiers.get(t.as_str()).cloned().unwrap_or_else(|| {
llmtrim::reroute::default_grok_tier_model(t).to_string()
});
println!(" {:<7} -> {model}", t.as_str());
}
let tier_keys: [&str; 4] = Tier::ALL.map(|t| t.as_str());
for (from, to) in &cfg.sub_tiers {
if !tier_keys.contains(&from.as_str()) {
println!(" {from} -> {to}");
}
}
}
SubProvider::Kimi => {
println!(" all tiers -> {}", llmtrim::reroute::KIMI_MODEL);
}
_ => println!(" (unknown provider)"),
}
let auth = llmtrim::reroute::auth::auth_status_json(p);
if auth["logged_in"].as_bool().unwrap_or(false) {
println!(" auth: logged in");
} else {
println!(
" auth: NOT logged in — run `llmtrim sub auth {} login`",
p.as_str()
);
}
}
}
Ok(())
}
SubCmd::Auth { provider, action } => run_auth(&provider, action),
}
}
#[cfg(not(feature = "intercept"))]
fn run_sub(_action: SubCmd) -> Result<()> {
anyhow::bail!("this build has no interceptor; rebuild with `--features intercept`")
}
fn run() -> Result<()> {
let cli = Cli::parse();
match cli.command {
Commands::Compress { provider } => {
let input = read_stdin()?;
let kind = provider
.as_deref()
.map(ProviderKind::from_str)
.transpose()?;
let result = llmtrim_core::compress(&input, kind)?;
if let Ok(tracker) = Tracker::open() {
let _ = tracker.record(&Record {
provider: result.provider.as_str().to_string(),
model: result.model.clone(),
tokenizer: result.tokenizer_label.clone(),
exact: result.tokenizer_exact,
input_before: result.input_tokens_before.0 as i64,
input_after: result.input_tokens_after.0 as i64,
output_before: None,
output_after: None,
compress_micros: None,
cache_read_tokens: None,
fresh_input_tokens: None,
cache_write_tokens: None,
output_shaped: Some(result.output_shaped),
frozen_input_tokens: Some(result.frozen_input_tokens.0 as i64),
outcome: None,
});
}
let mut stdout = std::io::stdout().lock();
stdout
.write_all(result.request_json.as_bytes())
.context("failed to write stdout")?;
let _ = stdout.write_all(b"\n");
}
Commands::Send { provider } => {
let input = read_stdin()?;
let kind = provider
.as_deref()
.map(ProviderKind::from_str)
.transpose()?;
let result = llmtrim_core::compress(&input, kind)?;
let endpoint = Endpoint::from_env(result.provider)?;
let proxy_url = llmtrim::transport::upstream_proxy_url(None)?;
let response = endpoint.send(&result.request_json, proxy_url.as_deref())?;
if let Ok(counter) =
llmtrim_core::tokenizer::counter_for(result.provider, result.model.as_deref())
{
let output_after = serde_json::from_str::<serde_json::Value>(&response)
.ok()
.and_then(|v| llmtrim_core::provider::for_kind(result.provider).answer_text(&v))
.map(|text| counter.count(&text) as i64);
if let Ok(tracker) = Tracker::open() {
let _ = tracker.record(&Record {
provider: result.provider.as_str().to_string(),
model: result.model.clone(),
tokenizer: counter.label().to_string(),
exact: counter.is_exact(),
input_before: result.input_tokens_before.0 as i64,
input_after: result.input_tokens_after.0 as i64,
output_before: None,
output_after,
compress_micros: None,
cache_read_tokens: None,
fresh_input_tokens: None,
cache_write_tokens: None,
output_shaped: Some(result.output_shaped),
frozen_input_tokens: Some(result.frozen_input_tokens.0 as i64),
outcome: None,
});
}
}
println!("{response}");
}
Commands::Serve {
port,
force,
supervised,
hide_console,
} => {
if hide_console {
llmtrim::autostart::hide_console_window();
}
if supervised {
let _ = llmtrim::setup::heal_managed_env();
#[cfg(windows)]
let _ = llmtrim::setup::wire_env_windows(port);
llmtrim::serve::run_supervised(port, force)?;
} else {
llmtrim::serve::run(port, force)?;
}
}
Commands::Setup { port, force, env } => {
if env {
llmtrim::setup::print_env(port)?
} else {
llmtrim::setup::run(port, force)?
}
}
Commands::Ensure { quiet } => llmtrim::ensure::run_cli(quiet)?,
Commands::Uninstall { purge, keep_binary } => {
llmtrim::setup::uninstall(purge, keep_binary)?
}
Commands::Start { port, force } => {
let color = ui::color_stdout();
if force && let Some(state) = llmtrim::daemon::running() {
println!(
"{}",
ui::ok(
color,
&format!("Stopping existing daemon · pid {} (--force)", state.pid)
)
);
if !llmtrim::daemon::stop_and_wait_free(state.port)? {
anyhow::bail!(
"--force stopped pid {} but port {} was still held after 5s",
state.pid,
state.port
);
}
}
if let Some(state) = llmtrim::daemon::running() {
println!(
"{}",
ui::ok(
color,
&format!(
"Interceptor already running · pid {} · port {}",
state.pid, state.port
)
)
);
let _ = llmtrim::ensure::maybe_auto();
} else {
let port = llmtrim::setup::resolve_port(port, None)?;
let pid = llmtrim::daemon::spawn_detached(port)?;
println!(
"{}",
ui::ok(
color,
&format!("Interceptor running · pid {pid} · port {port}")
)
);
let _ = llmtrim::ensure::maybe_auto();
for (label, value) in [
("watch", "llmtrim status".to_string()),
("stop", "llmtrim stop".to_string()),
] {
println!(
" {} {value}",
ui::paint(color, Tone::Dim, &format!("{label:<5}"))
);
}
if !llmtrim::setup::profile_has_block() {
eprintln!(
"{}",
ui::note(
ui::color_stderr(),
"HTTPS_PROXY isn't set for your user yet — run `llmtrim setup` so \
tools actually route through the interceptor."
)
);
}
}
if let Some(msg) = llmtrim::tray::nudge_once() {
println!("{}", ui::paint(color, Tone::Dim, &msg));
}
}
Commands::Wrap { args } => llmtrim::wrap::run(args)?,
Commands::Alive => {
if llmtrim::daemon::running().is_none() {
std::process::exit(1);
}
}
Commands::Stop => match llmtrim::daemon::stop()? {
Some(pid) => {
println!(
"{}",
ui::ok(
ui::color_stdout(),
&format!("Stopped interceptor (pid {pid}).")
)
);
#[cfg(not(windows))]
if llmtrim::wrap::https_proxy_is_local() {
eprintln!(
"{}",
ui::warn(
ui::color_stderr(),
&format!(
"This shell still points at the stopped interceptor. New \
terminals are already clear; to fix this one, run:\n {}",
llmtrim::setup::UNSET_HINT
)
)
);
}
#[cfg(windows)]
match llmtrim::setup::unwire_env_windows() {
Ok(true) => eprintln!(
"{}",
ui::note(
ui::color_stderr(),
"Cleared HTTPS_PROXY from your user environment — new terminals and \
apps are clear. Apps already running keep the old proxy until you \
restart them."
)
),
Ok(false) => {}
Err(e) => eprintln!(
"{}",
ui::warn(
ui::color_stderr(),
&format!("could not clear the interceptor env: {e}")
)
),
}
}
None => println!(
"{}",
ui::note(ui::color_stdout(), "No interceptor daemon was running.")
),
},
Commands::Sub { action } => run_sub(action)?,
Commands::Compact { action } => run_compact(action)?,
Commands::WindowSub { args } => {
let hook = args.first().is_some_and(|arg| arg.starts_with("hook-"));
if let Err(error) = run_window_sub(args) {
if hook {
eprintln!("llmtrim: window /sub hook skipped: {error:#}");
} else {
return Err(error);
}
}
}
Commands::Update => llmtrim::update::run()?,
Commands::Mcp { action } => match action {
None => llmtrim::mcp::run()?,
Some(McpAction::Install { print, force }) => llmtrim::mcp::install(print, force)?,
},
Commands::Statusline { action } => match action {
None => llmtrim::statusline::run()?,
Some(StatuslineCmd::Install { print }) => {
llmtrim::statusline::install(print)?;
if !print && let Err(e) = llmtrim::ensure::set_opt_out("statusline", false) {
eprintln!("llmtrim: could not clear statusline opt-out: {e:#}");
}
}
Some(StatuslineCmd::Uninstall) => {
llmtrim::statusline::uninstall()?;
if let Err(e) = llmtrim::ensure::set_opt_out("statusline", true) {
eprintln!("llmtrim: could not record statusline opt-out: {e:#}");
}
}
},
Commands::Guard { action } => match action {
None => std::process::exit(llmtrim::guard::run()),
Some(GuardCmd::Install { print }) => {
llmtrim::guard::install(print)?;
if !print && let Err(e) = llmtrim::ensure::set_opt_out("guard", false) {
eprintln!("llmtrim: could not clear guard opt-out: {e:#}");
}
}
Some(GuardCmd::Uninstall) => {
llmtrim::guard::uninstall()?;
if let Err(e) = llmtrim::ensure::set_opt_out("guard", true) {
eprintln!("llmtrim: could not record guard opt-out: {e:#}");
}
}
},
Commands::Monitor {
watch: _,
interval,
daily,
weekly,
monthly,
json,
breakdown,
csv,
quiet,
} => run_monitor(
interval, daily, weekly, monthly, json, breakdown, csv, quiet,
)?,
Commands::Doctor { fix } => {
if fix {
llmtrim::ensure::run_cli(false)?;
}
let report = llmtrim::doctor::gather();
print!("{}", llmtrim::doctor::render(ui::color_stdout(), &report));
if report.problems > 0 {
std::process::exit(1);
}
}
Commands::Tray => llmtrim::tray::run()?,
Commands::Autostart {
off,
port,
tray,
status,
} => {
let color = ui::color_stdout();
if status {
let enabled = if tray {
llmtrim::autostart::is_tray_enabled()
} else {
llmtrim::autostart::is_enabled()
};
println!("{}", if enabled { "enabled" } else { "disabled" });
return Ok(());
}
if tray {
llmtrim::autostart::configure_tray(!off)?;
let msg = if off {
"Tray autostart disabled."
} else {
"Tray autostart enabled — the llmtrim tray opens at login."
};
println!("{}", ui::ok(color, msg));
return Ok(());
}
if off {
llmtrim::autostart::configure(false, llmtrim::setup::DEFAULT_PORT)?;
println!("{}", ui::ok(color, "Autostart disabled."));
} else {
let running = llmtrim::daemon::running().map(|s| s.port);
let port = llmtrim::setup::resolve_port(port, running)
.context("llmtrim autostart: could not pick a port — pass --port explicitly")?;
llmtrim::autostart::configure(true, port)?;
println!(
"{}",
ui::ok(
color,
&format!("Autostart enabled — llmtrim serve --port {port} runs at login.")
)
);
println!(
" {} llmtrim autostart --off",
ui::paint(color, Tone::Dim, "undo ")
);
}
}
Commands::Ca { pem } => {
let path = llmtrim::serve::ca_cert_path()?;
llmtrim::serve::ensure_ca()?; if pem {
print!("{}", std::fs::read_to_string(&path)?);
return Ok(());
}
let color = ui::color_stdout();
print!(
"{}",
ui::panel(color, "llmtrim local CA", &[path.display().to_string()])
);
println!();
println!("Trust it for your tool, then route its traffic through llmtrim:");
#[cfg(windows)]
{
println!(" $env:NODE_EXTRA_CA_CERTS = \"{}\"", path.display());
println!(
" $env:HTTPS_PROXY = \"http://127.0.0.1:{}\"",
llmtrim::setup::DEFAULT_PORT
);
println!(" llmtrim serve");
println!();
println!("System-wide trust (non-PowerShell / GUI apps):");
println!(" certutil -addstore -user Root \"{}\"", path.display());
}
#[cfg(not(windows))]
{
println!(" export NODE_EXTRA_CA_CERTS={}", path.display());
println!(
" export HTTPS_PROXY=http://127.0.0.1:{}",
llmtrim::setup::DEFAULT_PORT
);
println!(" llmtrim serve");
}
println!();
println!(
"{}",
ui::note(
ui::color_stdout(),
"The CA is name-constrained to LLM API domains only."
)
);
}
Commands::Eval {
corpus,
provider,
keep_ratio,
} => {
let kind = ProviderKind::from_str(&provider)?;
let jsonl = std::fs::read_to_string(&corpus)
.with_context(|| format!("failed to read corpus {}", corpus.display()))?;
let cases = llmtrim::quality::load_corpus(&jsonl, kind)?;
let keep_ratio = keep_ratio.clamp(0.0, 1.0);
let config = llmtrim_core::config::DenseConfig {
retrieve: true,
retrieve_keep_ratio: keep_ratio,
retrieve_min_segment_chars: 120,
output_control: false,
..Default::default()
};
let results = llmtrim::quality::run_recall(&cases, &config)?;
let color = ui::color_stdout();
let saved_cell = |pct: f64| {
if pct > 0.0 {
ui::paint(color, Tone::Accent, &format!("-{pct:.1}%"))
} else {
ui::paint(color, Tone::Dim, &format!("{:.1}%", pct.abs()))
}
};
let mut t = ui::table(color, &["case", "recall", "tokens", "saved"]);
for r in &results {
t.add_row(vec![
comfy_table::Cell::new(&r.name),
ui::right(format!("{:.2}", r.recall)),
ui::right(format!("{} → {}", r.tokens_before, r.tokens_after)),
ui::right(saved_cell(r.savings_pct())),
]);
}
for line in t.to_string().lines() {
println!(" {line}");
}
let recall = llmtrim::quality::mean_recall(&results);
let savings = if results.is_empty() {
0.0
} else {
results.iter().map(|r| r.savings_pct()).sum::<f64>() / results.len() as f64
};
print!(
"\n{}",
ui::panel(
color,
"eval",
&[format!(
"{} cases mean recall {} mean savings {} keep_ratio {keep_ratio}",
results.len(),
ui::paint(color, Tone::Bold, &format!("{recall:.2}")),
saved_cell(savings),
)]
)
);
}
Commands::Discover {
dir,
by_tool,
limit,
json,
} => llmtrim::discover::run(dir, json, by_tool, limit)?,
Commands::Bench(cmd) => match cmd {
BenchCmd::Quality(args) => run_bench(*args)?,
BenchCmd::Suite(args) => run_bench_suite(*args)?,
BenchCmd::Agent(args) => run_bench_agent(*args)?,
BenchCmd::Latency(args) => run_bench_latency(args)?,
BenchCmd::Compare(args) => run_bench_compare(args)?,
},
}
Ok(())
}
#[cfg(feature = "live")]
fn dotenv_get(key: &str) -> Option<String> {
if let Ok(v) = std::env::var(key) {
return Some(v);
}
dotenvy::from_path_iter(".env")
.ok()?
.flatten()
.find(|(k, _)| k == key)
.map(|(_, v)| v)
}
fn prepare_request(request_json: &str, model: &str, route: &str, reasoning_effort: &str) -> String {
match serde_json::from_str::<serde_json::Value>(request_json) {
Ok(mut v) => {
if let Some(obj) = v.as_object_mut() {
obj.insert("model".to_string(), serde_json::json!(model));
if !route.is_empty() {
let (provider, quant) = match route.split_once('/') {
Some((p, q)) => (p, Some(q)),
None => (route, None),
};
let mut routing =
serde_json::json!({"order": [provider], "allow_fallbacks": false});
if let Some(q) = quant {
routing["quantizations"] = serde_json::json!([q]);
}
obj.insert("provider".to_string(), routing);
}
if !reasoning_effort.is_empty() {
obj.insert(
"reasoning".to_string(),
serde_json::json!({"effort": reasoning_effort}),
);
}
obj.insert("usage".to_string(), serde_json::json!({"include": true}));
}
v.to_string()
}
Err(_) => request_json.to_string(),
}
}
fn run_bench(args: BenchArgs) -> Result<()> {
let kind = ProviderKind::from_str(&args.provider)?;
let jsonl = std::fs::read_to_string(&args.corpus)
.with_context(|| format!("failed to read corpus {}", args.corpus.display()))?;
let mut cases = bench::load_bench_corpus(&jsonl, kind, &args.model)?;
if let Some(limit) = args.n {
cases.truncate(limit);
}
for c in &mut cases {
c.request = prepare_request(&c.request, &args.model, &args.route, &args.reasoning_effort);
}
let mut config = match &args.config {
Some(path) => {
let t = std::fs::read_to_string(path)
.with_context(|| format!("failed to read config {}", path.display()))?;
toml::from_str::<DenseConfig>(&t)
.with_context(|| format!("failed to parse config {}", path.display()))?
}
None => DenseConfig::preset(&args.preset).with_context(|| {
format!(
"unknown preset '{}' (auto|safe|lossless|rag|agent|code|aggressive|cache|reasoning)",
args.preset
)
})?,
};
config.retrieve_keep_ratio = config.retrieve_keep_ratio.clamp(0.0, 1.0);
config.retrieve_mmr_lambda = config.retrieve_mmr_lambda.clamp(0.0, 1.0);
if args.ablate {
run_ablation(&cases, &config, &args.preset)
} else if args.offline {
run_offline(&cases, kind, &config, &args)
} else {
run_live(&cases, kind, &config, &args)
}
}
const SUITE_MATRIX: &[(&str, &str, usize)] = &[
("gsm8k", "reasoning", 12), ("humaneval", "code", 12), ("dolly", "aggressive", 12), ("hotpotqa", "rag", 12), ("glaive", "agent", 12), ("chat", "aggressive", 12), ("cnn", "aggressive", 8), ("cache", "cache", 12), ("truthfulqa", "safe", 20), ("squad2", "rag", 20), ("bfcl", "agent", 20), ];
fn run_bench_suite(args: SuiteArgs) -> Result<()> {
if !args.offline {
for k in [
"HTTPS_PROXY",
"https_proxy",
"HTTP_PROXY",
"http_proxy",
"ALL_PROXY",
] {
if std::env::var_os(k).is_some() {
anyhow::bail!(
"{k} is set; the llmtrim proxy would re-compress the baseline arm and \
contaminate the A/B. Unset all *_PROXY vars before a live suite run."
);
}
}
}
std::fs::create_dir_all(&args.out)
.with_context(|| format!("failed to create {}", args.out.display()))?;
for (corpus, preset, default_n) in SUITE_MATRIX {
let n = args.n.or(Some(*default_n));
eprintln!(
"{}",
ui::note(
ui::color_stderr(),
&format!("=== {corpus} ({preset}, n={}) ===", n.unwrap_or(0))
)
);
let corpus_args = BenchArgs {
corpus: args.data_dir.join(format!("{corpus}.jsonl")),
provider: "openai".to_string(),
preset: (*preset).to_string(),
model: args.model.clone(),
judge_model: args.judge_model.clone(),
route: args.route.clone(),
reasoning_effort: String::new(),
n,
config: None,
pricing: args.pricing.clone(),
offline: args.offline,
ablate: false,
json_out: Some(args.out.join(format!("{corpus}.json"))),
};
if let Err(e) = run_bench(corpus_args) {
eprintln!(
"{}",
ui::render_error(ui::color_stderr(), &e.context(format!("corpus {corpus}")))
);
}
}
eprintln!("{}", ui::note(ui::color_stderr(), "suite complete"));
Ok(())
}
fn run_bench_latency(args: LatencyArgs) -> Result<()> {
use std::time::Instant;
const FIXTURE: &str = r#"{"model":"gpt-4o","messages":[
{"role":"system","content":"You are a meticulous coding assistant. Answer precisely."},
{"role":"user","content":"Review this function for bugs:\n```rust\nfn process(data: &[i32]) -> i32 {\n let mut total = 0;\n for x in data {\n if *x > 0 {\n total += x * 2;\n }\n }\n total\n}\n```\nDoes it handle the spec's edge cases?"}
]}"#;
let input = match &args.request {
Some(p) => {
std::fs::read_to_string(p).with_context(|| format!("failed to read {}", p.display()))?
}
None => FIXTURE.to_string(),
};
let kind = args
.provider
.as_deref()
.map(ProviderKind::from_str)
.transpose()?;
let config = DenseConfig::auto();
for _ in 0..5 {
let _ = llmtrim_core::compress_with_config(&input, kind, &config)?;
}
let n = args.iterations.max(1);
let mut r = llmtrim_core::compress_with_config(&input, kind, &config)?;
let t = Instant::now();
for _ in 0..n {
r = llmtrim_core::compress_with_config(&input, kind, &config)?;
}
let per_req_ms = t.elapsed().as_secs_f64() * 1000.0 / n as f64;
let counter = llmtrim_core::tokenizer::counter_for(r.provider, r.model.as_deref())?;
let value: serde_json::Value = serde_json::from_str(&input).context("parse request JSON")?;
let tools_str = value
.get("tools")
.map(|t| t.to_string())
.unwrap_or_default();
let bench = |label: &str, f: &dyn Fn()| {
let t = Instant::now();
for _ in 0..n {
f();
}
println!(
" {label}: {:.2} ms",
t.elapsed().as_secs_f64() * 1000.0 / n as f64
);
};
println!("request: {} bytes, provider={:?}", input.len(), r.provider);
println!(
"input tokens: {} -> {} ({:.1}% saved)",
r.input_tokens_before,
r.input_tokens_after,
100.0 * (1.0 - r.input_tokens_after.0 as f64 / r.input_tokens_before.0.max(1) as f64)
);
println!("compress latency: {per_req_ms:.2} ms/req (warm, avg of {n})");
println!("attribution (1x each):");
bench("full tokenize (content+tools)", &|| {
let _ = counter.count(&input);
});
bench("tools tokenize only", &|| {
let _ = counter.count(&tools_str);
});
bench("Value clone (per-stage snapshot)", &|| {
let _ = value.clone();
});
bench("JSON parse", &|| {
let _ = serde_json::from_str::<serde_json::Value>(&input);
});
bench("JSON serialize", &|| {
let _ = value.to_string();
});
Ok(())
}
fn run_bench_compare(args: CompareArgs) -> Result<()> {
match args.tool.as_str() {
"headroom" | "caveman" | "leanctx" | "entroly" | "rtk" | "snip" => {}
other => anyhow::bail!(
"unknown comparator '{other}' (expected: \
headroom|caveman|leanctx|entroly|rtk|snip)"
),
}
let script = "bench/scripts/bench.py";
if !std::path::Path::new(script).exists() {
anyhow::bail!(
"{script} not found - run `llmtrim bench compare` from the repo root (it dispatches \
the Python comparator, which needs its own deps; see bench/README.md)"
);
}
let mut cmd = std::process::Command::new("python3");
cmd.arg(script).arg(&args.tool);
if args.live {
cmd.arg("--live");
}
cmd.args(&args.extra);
let status = cmd
.status()
.with_context(|| format!("failed to run python3 {script}"))?;
if !status.success() {
std::process::exit(status.code().unwrap_or(1));
}
Ok(())
}
fn run_bench_agent(args: BenchAgentArgs) -> Result<()> {
use llmtrim::bench::agent::Condition;
let mut tasks = load_agent_tasks(&args.tasks)?;
if tasks.is_empty() {
anyhow::bail!("no agent tasks found under {:?}", args.tasks);
}
if let Some(m) = &args.model {
for t in &mut tasks {
t.model = m.clone();
}
}
let conditions: Vec<Condition> = args
.conditions
.split(',')
.map(|s| Condition::parse(s.trim()))
.collect();
let table = std::fs::read_to_string(&args.pricing)
.map(|s| bench::load_pricing(&s))
.unwrap_or_default();
if args.live {
run_agent_live(&tasks, &conditions, &args, &table)
} else {
run_agent_dry(&tasks, &conditions, &args, &table)
}
}
fn load_agent_tasks(paths: &[PathBuf]) -> Result<Vec<llmtrim::bench::agent::AgentTask>> {
use llmtrim::bench::agent::AgentTask;
let mut files = Vec::new();
for p in paths {
if p.is_dir() {
for entry in
std::fs::read_dir(p).with_context(|| format!("read dir {}", p.display()))?
{
let path = entry?.path();
let is_underscore = path
.file_name()
.and_then(|n| n.to_str())
.is_some_and(|n| n.starts_with('_'));
if !is_underscore && path.extension().and_then(|e| e.to_str()) == Some("json") {
files.push(path);
}
}
} else {
files.push(p.clone());
}
}
files.sort();
let mut shared: std::collections::HashMap<PathBuf, serde_json::Value> = Default::default();
let mut tasks = Vec::new();
for f in files {
let s = std::fs::read_to_string(&f).with_context(|| format!("read {}", f.display()))?;
let mut task =
AgentTask::from_json(&s).with_context(|| format!("parse {}", f.display()))?;
if task.tools.is_null() || task.tools.as_array().is_some_and(|a| a.is_empty()) {
let dir = f.parent().unwrap_or_else(|| std::path::Path::new("."));
let tools = match shared.get(dir) {
Some(v) => v.clone(),
None => {
let path = dir.join("_tools.json");
let v = std::fs::read_to_string(&path)
.ok()
.and_then(|t| serde_json::from_str(&t).ok())
.unwrap_or(serde_json::Value::Null);
shared.insert(dir.to_path_buf(), v.clone());
v
}
};
task.tools = tools;
}
tasks.push(task);
}
Ok(tasks)
}
fn run_agent_dry(
tasks: &[llmtrim::bench::agent::AgentTask],
conditions: &[llmtrim::bench::agent::Condition],
args: &BenchAgentArgs,
table: &bench::PriceTable,
) -> Result<()> {
use llmtrim::bench::agent::{OpenAiAgent, dry_run_transport, run_agent_loop};
let provider = OpenAiAgent;
let mut results = Vec::new();
for task in tasks {
let price = bench::resolve_pricing(table, &task.model);
for cond in conditions {
for _ in 0..args.repeats.max(1) {
let mut send = dry_run_transport(args.tool_turns);
results.push(run_agent_loop(&provider, task, cond, &price, &mut send)?);
}
}
}
print_agent_results(&results, true);
write_agent_json(&results, &args.json_out)
}
#[cfg(feature = "live")]
fn run_agent_live(
tasks: &[llmtrim::bench::agent::AgentTask],
conditions: &[llmtrim::bench::agent::Condition],
args: &BenchAgentArgs,
table: &bench::PriceTable,
) -> Result<()> {
use llmtrim::bench::agent::{OpenAiAgent, run_agent_loop};
let api_key =
dotenv_get("OPENROUTER_API_KEY").context("OPENROUTER_API_KEY not set (env or a .env)")?;
let model = llmtrim::quality::OpenRouterModel::new(api_key, ProviderKind::OpenAi)?;
let provider = OpenAiAgent;
let mut results = Vec::new();
for task in tasks {
let price = bench::resolve_pricing(table, &task.model);
for cond in conditions {
for _ in 0..args.repeats.max(1) {
let mut send = |req: &str| model.send_raw(req);
results.push(run_agent_loop(&provider, task, cond, &price, &mut send)?);
}
}
}
print_agent_results(&results, false);
write_agent_json(&results, &args.json_out)
}
#[cfg(not(feature = "live"))]
fn run_agent_live(
_tasks: &[llmtrim::bench::agent::AgentTask],
_conditions: &[llmtrim::bench::agent::Condition],
_args: &BenchAgentArgs,
_table: &bench::PriceTable,
) -> Result<()> {
anyhow::bail!(
"--live needs the `live` feature: rebuild with `cargo run --features live -- bench agent --live …`"
)
}
fn print_agent_results(results: &[llmtrim::bench::agent::AgentRunResult], dry: bool) {
let color = ui::color_stdout();
let mut t = ui::table(
color,
&[
"task",
"condition",
"iters",
"input",
"cached",
"output",
"cost$",
"done",
],
);
for r in results {
t.add_row(vec![
r.task_id.clone(),
r.condition.clone(),
r.iterations.to_string(),
r.input_tokens.to_string(),
r.cached_tokens.to_string(),
r.output_tokens.to_string(),
format!("{:.4}", r.cost_usd),
if r.completed { "yes" } else { "no" }.to_string(),
]);
}
println!("{t}");
if dry {
println!(
"(dry-run: synthetic usage, cached estimated by prefix overlap — no API calls; use --live for real numbers)"
);
}
}
fn write_agent_json(
results: &[llmtrim::bench::agent::AgentRunResult],
out: &Option<PathBuf>,
) -> Result<()> {
if let Some(path) = out {
let doc = bench::envelope::wrap(
"agent-v1",
serde_json::json!({ "runs": results.len() }),
serde_json::to_value(results).context("serialize agent results")?,
);
let json = serde_json::to_string_pretty(&doc).context("serialize agent results")?;
std::fs::write(path, json).with_context(|| format!("write {}", path.display()))?;
}
Ok(())
}
fn run_ablation(cases: &[BenchCase], config: &DenseConfig, preset: &str) -> Result<()> {
let configs = bench::ablation_configs(config);
let rows = bench::run_token_ablation(cases, &configs)?;
let full_after = rows.first().map(|(_, _, a)| *a).unwrap_or(0) as f64;
let color = ui::color_stdout();
println!(
"{}",
ui::paint(
color,
Tone::Bold,
&format!(
"ablation — input tokens, offline, preset={preset}, {} cases",
cases.len()
)
)
);
let mut t = ui::table(
color,
&["config", "before", "after", "saved", "stage saves"],
);
for (label, before, after) in &rows {
let saved = ui::saved_pct(*before as f64, *after as f64);
let contribution = if label == "full" {
String::new()
} else {
format!("{:+.0} tok", *after as f64 - full_after)
};
t.add_row(vec![
comfy_table::Cell::new(label),
ui::right(before.to_string()),
ui::right(after.to_string()),
ui::right(ui::paint(color, Tone::Accent, &format!("{saved:.1}%"))),
ui::right(contribution),
]);
}
for line in t.to_string().lines() {
println!(" {line}");
}
Ok(())
}
fn quality_meta(args: &BenchArgs, config: &DenseConfig) -> serde_json::Value {
serde_json::json!({
"preset": args.preset,
"config_path": args.config.as_ref().map(|p| p.display().to_string()),
"config_resolved": config,
"model": args.model,
"judge_model": args.judge_model,
"route": args.route,
"reasoning_effort": args.reasoning_effort,
"corpus": args.corpus.display().to_string(),
})
}
fn run_offline(
cases: &[BenchCase],
kind: ProviderKind,
config: &DenseConfig,
args: &BenchArgs,
) -> Result<()> {
let (mut before, mut after) = (0usize, 0usize);
let mut rows = Vec::with_capacity(cases.len());
for c in cases {
let r = llmtrim_core::compress_with_config(&c.request, Some(kind), config)?;
before += r.input_tokens_before.0;
after += r.input_tokens_after.0;
rows.push(serde_json::json!({
"name": c.name,
"tokens_in_before": r.input_tokens_before.0,
"tokens_in_after": r.input_tokens_after.0,
}));
}
println!(
"offline: {} cases input {before} -> {after} tok ({:.1}% saved) preset={}",
cases.len(),
ui::saved_pct(before as f64, after as f64),
args.preset,
);
if let Some(path) = &args.json_out {
let result = serde_json::json!({
"n": cases.len(),
"tokens_in_before": before,
"tokens_in_after": after,
"tokens_in_saved_pct": ui::saved_pct(before as f64, after as f64),
"cases": rows,
});
let doc = bench::envelope::wrap("quality-offline-v1", quality_meta(args, config), result);
let json = serde_json::to_string_pretty(&doc).context("serialize offline results")?;
std::fs::write(path, json)
.with_context(|| format!("failed to write {}", path.display()))?;
eprintln!(
"{}",
ui::note(ui::color_stderr(), &format!("wrote {}", path.display()))
);
}
Ok(())
}
#[cfg(feature = "live")]
fn run_live(
cases: &[BenchCase],
kind: ProviderKind,
config: &DenseConfig,
args: &BenchArgs,
) -> Result<()> {
if kind != ProviderKind::OpenAi {
anyhow::bail!(
"live A/B bench only supports --provider openai (the OpenRouter endpoint is \
OpenAI-shaped; a {} request would be sent OpenAI-style and mis-parsed, scoring \
nothing). Use --offline or --ablate to measure {}-shaped input-token savings, \
or re-run with --provider openai.",
args.provider,
args.provider
);
}
if args.judge_model == args.model
&& cases
.iter()
.any(|c| matches!(c.scorer, bench::Scorer::LlmJudge))
{
eprintln!(
"{}",
ui::warn(
ui::color_stderr(),
&format!(
"--judge-model == --model ({}) — the model is grading its own answers; \
pass a different --judge-model for an unbiased judge.",
args.model
)
)
);
}
let table = std::fs::read_to_string(&args.pricing)
.map(|s| bench::load_pricing(&s))
.unwrap_or_default();
let price = bench::resolve_pricing(&table, &args.model);
let counter = llmtrim_core::tokenizer::counter_for(kind, Some(&args.model))?;
let api_key = dotenv_get("OPENROUTER_API_KEY")
.context("OPENROUTER_API_KEY not set (in env or a local .env)")?;
let llm = llmtrim::quality::OpenRouterModel::new(api_key, kind)?;
let scorer = bench::BenchScorer {
exec_timeout: 10,
judge: Some(&llm),
judge_model: args.judge_model.clone(),
};
let run = bench::run_ab(cases, config, &llm, counter.as_ref(), &scorer, price)?;
let color = ui::color_stdout();
let mut t = ui::table(color, &["case", "quality", "input", "output"]);
for o in &run.outcomes {
let q_tone = if o.quality_comp < o.quality_orig {
Tone::Warn
} else {
Tone::Accent
};
t.add_row(vec![
comfy_table::Cell::new(ui::truncate(&o.name, 24)),
ui::right(ui::paint(
color,
q_tone,
&format!("{:.2} → {:.2}", o.quality_orig, o.quality_comp),
)),
ui::right(format!("{} → {}", o.tokens_in_before, o.tokens_in_after)),
ui::right(format!("{} → {}", o.tokens_out_orig, o.tokens_out_comp)),
]);
}
for line in t.to_string().lines() {
println!(" {line}");
}
let f = bench::summarize(&run);
let cache_note = if f.cache_busted {
"cache busted (per-arm nonce; cache stage off)"
} else {
"cache stage under test"
};
let metric = |v: f64| ui::paint(color, Tone::Accent, &format!("{v:.1}%"));
let lines = vec![
ui::paint(
color,
Tone::Dim,
&format!(
"{} · model {} · judge {}",
args.corpus.display(),
args.model,
args.judge_model
),
),
String::new(),
format!("input saved {}", metric(f.tokens_in_saved_pct)),
format!("output saved {}", metric(f.tokens_out_saved_pct)),
format!("cost saved {}", metric(f.cost_saved_pct)),
format!("cache used {:.1}% ({cache_note})", f.cache_used_pct),
format!(
"quality {} → {} (retention {:+.1}pp, paired 95%CI ±{:.1})",
ui::paint(
color,
Tone::Bold,
&format!("{:.1}%", f.quality_orig.mean * 100.0)
),
ui::paint(
color,
Tone::Bold,
&format!("{:.1}%", f.quality_comp.mean * 100.0)
),
f.retention_pp,
f.retention_ci95_pp,
),
format!(
"cases {} scored, {} failed (compressed 4xx → 0), {} skipped (transient)",
f.n, f.failed, f.skipped
),
];
print!(
"\n{}",
ui::panel(
color,
&format!("bench · {} · n={}", args.preset, f.n),
&lines
)
);
if let Some(path) = &args.json_out {
let rows: Vec<_> = run
.outcomes
.iter()
.map(|o| {
serde_json::json!({
"name": o.name,
"quality_orig": o.quality_orig, "quality_comp": o.quality_comp,
"tokens_in_before": o.tokens_in_before, "tokens_in_after": o.tokens_in_after,
"tokens_out_orig": o.tokens_out_orig, "tokens_out_comp": o.tokens_out_comp,
"cached_in_orig": o.cached_in_orig, "cached_in_comp": o.cached_in_comp,
"cost_orig": o.cost_orig, "cost_comp": o.cost_comp,
})
})
.collect();
let meta = quality_meta(args, config);
let result = serde_json::json!({
"n": f.n,
"failed": f.failed,
"skipped": f.skipped,
"cache_busted": f.cache_busted,
"tokens_in_saved_pct": f.tokens_in_saved_pct,
"tokens_out_saved_pct": f.tokens_out_saved_pct,
"cost_saved_pct": f.cost_saved_pct,
"cache_used_pct": f.cache_used_pct,
"quality_orig": f.quality_orig.mean, "quality_comp": f.quality_comp.mean,
"retention_pp": f.retention_pp,
"retention_ci95_pp": f.retention_ci95_pp,
"cases": rows,
});
let doc = bench::envelope::wrap("quality-v1", meta, result);
let json = serde_json::to_string_pretty(&doc).context("serialize quality results")?;
std::fs::write(path, json)
.with_context(|| format!("failed to write {}", path.display()))?;
eprintln!(
"{}",
ui::note(ui::color_stderr(), &format!("wrote {}", path.display()))
);
}
Ok(())
}
#[cfg(not(feature = "live"))]
fn run_live(
_cases: &[BenchCase],
_kind: ProviderKind,
_config: &DenseConfig,
_args: &BenchArgs,
) -> Result<()> {
anyhow::bail!(
"live A/B bench requires the `live` feature: rebuild with \
`cargo run --features live -- bench …`. Use `--offline` (or `--ablate`) for \
token-only measurement, which needs no extra feature."
)
}
fn period_flag(daily: bool, weekly: bool, monthly: bool) -> Option<Period> {
if daily {
Some(Period::Day)
} else if weekly {
Some(Period::Week)
} else if monthly {
Some(Period::Month)
} else {
None
}
}
fn daemon_view(summary: &llmtrim::tracking::Summary) -> monitor::DaemonView {
use llmtrim::daemon;
let ca_present = matches!(llmtrim::serve::ca_cert_path(), Ok(p) if p.exists());
let env_port = llmtrim::setup::configured_port();
let autostart = llmtrim::autostart::is_enabled();
let log_path = daemon::logfile().ok().map(|p| p.display().to_string());
let last_request = summary.last_ts.as_deref().and_then(daemon::human_age);
let binary_version = env!("CARGO_PKG_VERSION").to_string();
match daemon::running() {
Some(s) => monitor::DaemonView {
running: true,
pid: s.pid,
port: s.port,
uptime: daemon::human_uptime(daemon::uptime_secs(s.started_at)),
uptime_secs: daemon::uptime_secs(s.started_at),
ca_present,
port_accepting: daemon::probe_port(s.port),
env_port,
autostart,
restarts: s.restarts,
version: s.version,
binary_version,
log_path,
last_request,
},
None => match env_port.filter(|&p| daemon::probe_port(p)) {
Some(p) => monitor::DaemonView {
running: true,
pid: 0,
port: p,
uptime: String::new(),
uptime_secs: 0,
ca_present,
port_accepting: true,
env_port,
autostart,
restarts: 0,
version: None,
binary_version,
log_path,
last_request,
},
None => monitor::DaemonView {
running: false,
pid: 0,
port: 0,
uptime: String::new(),
uptime_secs: 0,
ca_present,
port_accepting: false,
env_port,
autostart,
restarts: 0,
version: None,
binary_version,
log_path,
last_request,
},
},
}
}
fn render_snapshot(
tracker: &Tracker,
color: bool,
cache_included: bool,
split_marker: bool,
) -> Result<(String, monitor::Health)> {
let summary = tracker.summary()?;
let models = monitor::model_views(tracker)?;
let daemon = daemon_view(&summary);
let health = monitor::health(&daemon);
let trend: Vec<i64> = tracker
.by_period(Period::Day)
.unwrap_or_default()
.iter()
.rev()
.take(7)
.rev()
.map(|r| (r.input_before - r.input_after).max(0))
.collect();
let (hero_saved, today_saved) = monitor::hero_money(tracker);
let mut out = monitor::snapshot(
color,
Some(&daemon),
&summary,
&models,
hero_saved,
today_saved,
&trend,
cache_included,
split_marker,
);
if let Some(v) = llmtrim::update::check(false) {
out.push_str(&format!(
"\n {} llmtrim v{v} available — run llmtrim update\n",
ui::paint(color, Tone::Accent, "↑")
));
}
Ok((out, health))
}
#[cfg(feature = "breakdown")]
fn overview_data(tracker: &Tracker) -> llmtrim::breakdown::app::OverviewData {
use llmtrim::breakdown::app::{StatusKind, StatusLine};
monitor::overview_data(tracker, |summary, has_traffic| {
let daemon = daemon_view(summary);
let version_stale = daemon
.version
.as_deref()
.is_some_and(|v| v != daemon.binary_version);
if daemon.running && version_stale {
StatusLine {
kind: StatusKind::Stale,
text: "llmtrim is on, but running an older version — restart to apply the update"
.into(),
fix: Some("llmtrim start --force".into()),
uninstall: None,
}
} else if daemon.running {
let text = match daemon.last_request {
Some(ago) => format!("llmtrim is on and working · last request {ago}"),
None => "llmtrim is on and ready · waiting for your first request".into(),
};
StatusLine {
kind: if has_traffic {
StatusKind::Working
} else {
StatusKind::Ready
},
text,
fix: None,
uninstall: None,
}
} else if daemon.env_port.is_some() {
StatusLine {
kind: StatusKind::Off,
text: "llmtrim is OFF — your AI tools can't reach the API right now".into(),
fix: Some("llmtrim start".into()),
uninstall: Some("llmtrim uninstall".into()),
}
} else {
StatusLine {
kind: StatusKind::Degraded,
text: "llmtrim is not set up — your AI traffic isn't routed through it".into(),
fix: Some("llmtrim setup".into()),
uninstall: None,
}
}
})
}
fn merge_breakdown_json(s: String, include: bool) -> String {
if !include {
return s;
}
#[cfg(feature = "breakdown")]
if let Some(bd) = llmtrim::breakdown::export::breakdown_json() {
let mut v: serde_json::Value = match serde_json::from_str(&s) {
Ok(v) => v,
Err(e) => {
eprintln!("llmtrim: could not parse status json for breakdown merge: {e}");
return s;
}
};
if let Some(obj) = v.as_object_mut() {
obj.insert("breakdown".to_string(), bd);
match serde_json::to_string_pretty(&v) {
Ok(out) => return out,
Err(e) => eprintln!("llmtrim: could not merge breakdown into json: {e}"),
}
} else {
eprintln!("llmtrim: status json is not an object; skipped breakdown merge");
}
}
s
}
fn print_sessions_csv() {
#[cfg(feature = "breakdown")]
if let Some(sc) = llmtrim::breakdown::export::sessions_csv() {
println!();
print!("{sc}");
}
}
fn print_top_sources() {
#[cfg(feature = "breakdown")]
if let Ok(block) = llmtrim::breakdown::export::top_sources_report(ui::color_stdout(), 10) {
print!("{block}");
}
}
#[allow(clippy::too_many_arguments)]
fn run_monitor(
interval: u64,
daily: bool,
weekly: bool,
monthly: bool,
json: bool,
breakdown: bool,
csv: bool,
quiet: bool,
) -> Result<()> {
#[cfg(not(feature = "breakdown"))]
let _ = interval;
let tracker = Tracker::open().context("failed to open savings ledger")?;
if quiet {
let summary = tracker.summary()?;
let health = monitor::health(&daemon_view(&summary));
println!("{}", health.label());
std::process::exit(health.exit_code());
}
if let Some(period) = period_flag(daily, weekly, monthly) {
let rows = tracker.by_period(period)?;
if csv {
print!("{}", monitor::export_csv(&rows));
print_sessions_csv();
} else if json {
let s = tracker.summary()?;
let models = monitor::model_views(&tracker)?;
let daemon = daemon_view(&s);
let out = monitor::export_json_with_money(
&s,
&models,
monitor::monitor_cost(&tracker).as_ref(),
&rows,
Some(&daemon),
monitor::money_json(&tracker),
);
println!("{}", merge_breakdown_json(out, breakdown));
} else {
print!(
"{}",
monitor::period_report(ui::color_stdout(), period.label(), &rows)
);
print_top_sources();
}
return Ok(());
}
if json || csv {
if csv {
let rows = tracker.by_period(Period::Day)?;
print!("{}", monitor::export_csv(&rows));
print_sessions_csv();
} else {
let daemon = daemon_view(&tracker.summary()?);
println!(
"{}",
merge_breakdown_json(monitor::stats_json(&tracker, Some(&daemon))?, breakdown)
);
}
return Ok(());
}
#[cfg(feature = "breakdown")]
{
use std::io::IsTerminal as _;
if std::io::stdout().is_terminal() {
drop(tracker);
let tracker = llmtrim::tracking::db_path()
.ok()
.as_deref()
.and_then(|p| Tracker::open_reader_at(p).ok());
let sessions_db = llmtrim::breakdown::db::BreakdownDb::open().ok();
let snapshot = move || {
let ov = tracker.as_ref().map(overview_data).unwrap_or_default();
let rows = sessions_db
.as_ref()
.and_then(|d| d.sessions().ok())
.unwrap_or_default();
(ov, rows)
};
use llmtrim::breakdown::app::PostAction;
let _ = llmtrim::ensure::maybe_auto();
match llmtrim::breakdown::app::run(interval.max(2), snapshot)? {
PostAction::Doctor => {
let report = llmtrim::doctor::gather();
print!("{}", llmtrim::doctor::render(ui::color_stdout(), &report));
}
PostAction::Update => llmtrim::update::run()?,
PostAction::Restart => {
if let Err(e) = llmtrim::update::restart_daemon(ui::color_stdout()) {
eprintln!("{e:#}");
std::process::exit(1);
}
}
PostAction::None => {}
}
return Ok(());
}
}
let (out, health) = render_snapshot(&tracker, ui::color_stdout(), false, false)?;
print!("{out}");
if health != monitor::Health::Healthy {
use std::io::Write as _;
let _ = std::io::stdout().flush();
std::process::exit(health.exit_code());
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use std::time::{SystemTime, UNIX_EPOCH};
#[test]
fn status_still_accepts_deprecated_watch_flag() {
let cli = Cli::try_parse_from(["llmtrim", "status", "--watch"])
.expect("status --watch must still parse for backward compatibility");
match cli.command {
Commands::Monitor { watch, .. } => assert!(watch),
_ => panic!("expected status to parse as the Monitor command"),
}
}
#[test]
fn breakdown_section_is_omitted_unless_requested() {
let base = r#"{"daemon":{"running":true}}"#.to_string();
assert_eq!(merge_breakdown_json(base.clone(), false), base);
}
#[test]
fn alive_subcommand_name_is_stable_and_hidden() {
let cli = Cli::try_parse_from(["llmtrim", "_alive"]).expect("`_alive` must parse");
assert!(matches!(cli.command, Commands::Alive));
let help = {
use clap::CommandFactory;
let mut out = Vec::new();
Cli::command().write_long_help(&mut out).unwrap();
String::from_utf8(out).unwrap()
};
assert!(
!help.contains("_alive"),
"`_alive` must stay hidden from help"
);
}
#[test]
fn breakdown_flag_requires_json() {
assert!(
Cli::try_parse_from(["llmtrim", "status", "--breakdown"]).is_err(),
"--breakdown without --json must be rejected"
);
Cli::try_parse_from(["llmtrim", "status", "--json", "--breakdown"])
.expect("status --json --breakdown must parse");
}
fn scratch_dir(tag: &str) -> PathBuf {
let nanos = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos();
let dir = std::env::temp_dir().join(format!("llmtrim_bench_{tag}_{nanos}"));
std::fs::create_dir_all(&dir).unwrap();
dir
}
#[test]
fn load_agent_tasks_injects_shared_tools_and_skips_underscore_files() {
let dir = scratch_dir("dedup");
std::fs::write(
dir.join("_tools.json"),
r#"[{"type":"function","function":{"name":"grep"}}]"#,
)
.unwrap();
std::fs::write(
dir.join("a.json"),
r#"{"id":"a","model":"m","system":"s","user":"u"}"#,
)
.unwrap();
std::fs::write(
dir.join("b.json"),
r#"{"id":"b","model":"m","system":"s","user":"u","tools":[{"type":"function","function":{"name":"own"}}]}"#,
)
.unwrap();
let tasks = load_agent_tasks(std::slice::from_ref(&dir)).unwrap();
assert_eq!(tasks.len(), 2, "_tools.json is a fragment, not a task");
let a = tasks.iter().find(|t| t.id == "a").unwrap();
let b = tasks.iter().find(|t| t.id == "b").unwrap();
assert_eq!(
a.tools.pointer("/0/function/name").unwrap(),
"grep",
"shared catalog injected when tools omitted"
);
assert_eq!(
b.tools.pointer("/0/function/name").unwrap(),
"own",
"task's own tools left intact"
);
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn suite_matrix_covers_the_documented_corpora() {
assert_eq!(
SUITE_MATRIX.len(),
11,
"eight workload corpora + three named benchmarks"
);
let cnn = SUITE_MATRIX.iter().find(|(c, _, _)| *c == "cnn").unwrap();
assert_eq!(cnn.2, 8, "cnn runs fewer cases (long docs)");
for (corpus, preset, n) in SUITE_MATRIX {
assert!(*n > 0, "{corpus}: case count must be positive");
assert!(
DenseConfig::preset(preset).is_some(),
"{corpus}: unknown preset '{preset}'"
);
}
}
#[test]
fn prepare_request_pins_model_route_and_reasoning() {
let base = r#"{"messages":[{"role":"user","content":"hi"}]}"#;
let out = prepare_request(base, "openai/gpt-oss-20b", "wandb/fp4", "medium");
let v: serde_json::Value = serde_json::from_str(&out).unwrap();
assert_eq!(v["model"], "openai/gpt-oss-20b");
assert_eq!(v["provider"]["order"], serde_json::json!(["wandb"]));
assert_eq!(v["provider"]["quantizations"], serde_json::json!(["fp4"]));
assert_eq!(v["reasoning"]["effort"], "medium");
}
#[test]
fn prepare_request_omits_reasoning_when_effort_is_empty() {
let base = r#"{"messages":[{"role":"user","content":"hi"}]}"#;
let out = prepare_request(base, "openai/gpt-4o-mini", "", "");
let v: serde_json::Value = serde_json::from_str(&out).unwrap();
assert!(v.get("reasoning").is_none(), "no reasoning field: {v}");
assert!(v.get("provider").is_none(), "no provider field: {v}");
}
const CORPUS_LINE: &str = r#"{"context":"The cat sat on the red mat.","question":"What color was the mat?","gold":"red","scorer":"token_f1"}"#;
fn offline_args(corpus: PathBuf, json_out: Option<PathBuf>) -> BenchArgs {
BenchArgs {
corpus,
provider: "openai".to_string(),
preset: "rag".to_string(),
model: "openai/gpt-4o-mini".to_string(),
judge_model: "openai/gpt-4o-mini".to_string(),
route: String::new(),
reasoning_effort: String::new(),
n: Some(1),
config: None,
pricing: PathBuf::new(), offline: true,
ablate: false,
json_out,
}
}
#[test]
fn offline_quality_writes_a_valid_enveloped_file() {
let dir = scratch_dir("offline");
let corpus = dir.join("c.jsonl");
std::fs::write(&corpus, format!("{CORPUS_LINE}\n")).unwrap();
let out = dir.join("out.json");
run_bench(offline_args(corpus, Some(out.clone()))).unwrap();
let doc: serde_json::Value =
serde_json::from_str(&std::fs::read_to_string(&out).unwrap()).unwrap();
assert_eq!(doc["schema"], "llmtrim-bench/quality-offline-v1");
assert_eq!(doc["meta"]["preset"], "rag");
assert_eq!(doc["result"]["n"], 1);
assert_eq!(doc["result"]["cases"].as_array().unwrap().len(), 1);
assert!(doc["result"]["tokens_in_before"].is_number());
assert!(doc["meta"]["model"].is_string());
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn suite_runs_offline_and_skips_missing_corpora() {
let data = scratch_dir("suite_data");
let out = scratch_dir("suite_out");
std::fs::write(data.join("gsm8k.jsonl"), format!("{CORPUS_LINE}\n")).unwrap();
let args = SuiteArgs {
data_dir: data.clone(),
out: out.clone(),
model: "openai/gpt-4o-mini".to_string(),
judge_model: "openai/gpt-4o-mini".to_string(),
route: String::new(),
pricing: PathBuf::new(), n: Some(1),
offline: true,
};
run_bench_suite(args).expect("missing corpora must not abort the matrix");
assert!(
out.join("gsm8k.json").is_file(),
"present corpus produced a result"
);
assert!(
!out.join("cnn.json").exists(),
"absent corpus left no result"
);
std::fs::remove_dir_all(&data).unwrap();
std::fs::remove_dir_all(&out).unwrap();
}
#[test]
fn latency_runs_on_the_builtin_fixture() {
let r = run_bench_latency(LatencyArgs {
request: None,
provider: None,
iterations: 3,
});
assert!(
r.is_ok(),
"latency on the built-in fixture should not error: {r:?}"
);
}
#[test]
fn compare_rejects_an_unknown_tool() {
let err = run_bench_compare(CompareArgs {
tool: "nope".to_string(),
live: false,
extra: vec![],
})
.unwrap_err();
assert!(err.to_string().contains("unknown comparator"), "{err}");
}
}