llm-test-bench 0.1.0

A production-grade CLI for testing and benchmarking LLM applications with support for GPT-5, Claude Opus 4, Gemini 2.5, and 65+ models
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128

use clap::{Parser, Subcommand};
use std::process;

mod commands;
mod error;
mod output;

use commands::{analyze, bench, compare, config, dashboard, eval, optimize, test};

/// LLM Test Bench - Production-grade CLI for testing and benchmarking LLM applications
#[derive(Parser)]
#[command(name = "llm-test-bench")]
#[command(author = "LLM Test Bench Contributors")]
#[command(version)]
#[command(about = "A production-grade CLI for testing and benchmarking LLM applications", long_about = None)]
#[command(propagate_version = true)]
#[command(arg_required_else_help = true)]
struct Cli {
    #[command(subcommand)]
    command: Commands,

    /// Enable verbose output
    #[arg(short, long, global = true)]
    verbose: bool,

    /// Disable colored output
    #[arg(long, global = true)]
    no_color: bool,
}

#[derive(Subcommand)]
enum Commands {
    /// Run a single test against an LLM provider
    #[command(visible_alias = "t")]
    Test(test::TestArgs),

    /// Run benchmark tests across multiple providers
    #[command(visible_alias = "b")]
    Bench(bench::BenchArgs),

    /// Evaluate test results with metrics
    #[command(visible_alias = "e")]
    Eval(eval::EvalArgs),

    /// Compare multiple models on the same prompt or dataset
    #[command(visible_alias = "c")]
    Compare(compare::CompareArgs),

    /// Generate interactive HTML dashboards from results
    #[command(visible_alias = "d")]
    Dashboard(dashboard::DashboardArgs),

    /// Perform statistical analysis comparing baseline and new results
    #[command(visible_alias = "a")]
    Analyze(analyze::AnalyzeArgs),

    /// Recommend cost-optimized model alternatives
    #[command(visible_alias = "o")]
    Optimize(optimize::OptimizeArgs),

    /// Configuration management commands
    #[command(subcommand)]
    Config(config::ConfigCommands),

    /// Generate shell completions
    Completions {
        /// The shell to generate completions for
        #[arg(value_enum)]
        shell: clap_complete::Shell,
    },
}

#[tokio::main]
async fn main() {
    // Initialize tracing
    tracing_subscriber::fmt()
        .with_env_filter(
            tracing_subscriber::EnvFilter::try_from_default_env()
                .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")),
        )
        .init();

    let cli = Cli::parse();

    // Handle color output
    if cli.no_color {
        colored::control::set_override(false);
    }

    // Execute command
    let result = match cli.command {
        Commands::Test(args) => test::execute(args, cli.verbose).await,
        Commands::Bench(args) => bench::execute(args, cli.verbose).await,
        Commands::Eval(args) => eval::execute(args, cli.verbose).await,
        Commands::Compare(args) => compare::execute(args, cli.verbose).await,
        Commands::Dashboard(args) => dashboard::execute(args, cli.verbose).await,
        Commands::Analyze(args) => analyze::execute(args, cli.verbose).await,
        Commands::Optimize(args) => optimize::execute(args, cli.verbose).await,
        Commands::Config(cmd) => config::execute(cmd, cli.verbose).await,
        Commands::Completions { shell } => {
            generate_completions(shell);
            Ok(())
        }
    };

    // Handle errors
    if let Err(e) = result {
        eprintln!("Error: {}", e);
        if cli.verbose {
            eprintln!("\nCaused by:");
            for cause in e.chain().skip(1) {
                eprintln!("  {}", cause);
            }
        }
        process::exit(1);
    }
}

fn generate_completions(shell: clap_complete::Shell) {
    use clap::CommandFactory;
    use clap_complete::generate;
    use std::io;

    let mut cmd = Cli::command();
    let bin_name = cmd.get_name().to_string();
    generate(shell, &mut cmd, bin_name, &mut io::stdout());
}