mesh_llm_cli/
benchmark.rs

1use clap::{Args, Subcommand, ValueEnum};
2use std::path::PathBuf;
3
4#[derive(Subcommand, Debug, Clone)]
5pub enum BenchmarkCommand {
6    /// Tune model-serving settings by running isolated throughput trials.
7    Tune(Box<BenchmarkTuneCommand>),
8    /// Import a prompt corpus from a supported online source into local JSONL.
9    #[command(name = "import-prompts")]
10    ImportPrompts {
11        /// Online source to import.
12        #[arg(long, value_enum)]
13        source: PromptImportSource,
14        /// Maximum number of prompts to import.
15        #[arg(long, default_value = "20")]
16        limit: usize,
17        /// Optional per-prompt decode budget hint written into the corpus.
18        #[arg(long)]
19        max_tokens: Option<u32>,
20        /// Output JSONL path.
21        #[arg(long)]
22        output: PathBuf,
23    },
24}
25
26#[derive(Args, Debug, Clone)]
27pub struct BenchmarkTuneCommand {
28    /// Tune exactly one local/configured model target.
29    #[arg(long, conflicts_with = "models")]
30    pub model: Option<String>,
31    /// Tune multiple local/configured model targets from a comma-separated list.
32    #[arg(long, value_delimiter = ',')]
33    pub models: Vec<String>,
34    /// Print machine-readable JSON output.
35    #[arg(long)]
36    pub json: bool,
37    /// Context sizes to benchmark, as a comma-separated token list.
38    #[arg(long, value_delimiter = ',')]
39    pub ctx_sizes: Vec<u32>,
40    /// Batch sizes to benchmark, as a comma-separated list.
41    #[arg(long, value_delimiter = ',')]
42    pub batch_sizes: Vec<u32>,
43    /// Micro-batch sizes to benchmark, as a comma-separated list.
44    #[arg(long, value_delimiter = ',')]
45    pub ubatch_sizes: Vec<u32>,
46    /// mmap values to benchmark independently: auto, enabled, disabled.
47    #[arg(long = "mmap-values", value_delimiter = ',')]
48    pub mmap_values: Vec<BenchmarkBoolOrAuto>,
49    /// mlock values to benchmark independently: enabled, disabled.
50    #[arg(long = "mlock-values", value_delimiter = ',')]
51    pub mlock_values: Vec<BenchmarkBool>,
52    /// Flash attention values to benchmark independently: on, off.
53    #[arg(long = "flash-attention", value_delimiter = ',')]
54    pub flash_attention: Vec<BenchmarkFlashAttention>,
55    /// Speculative decoding types to benchmark: auto, disabled, mtp, draft, mtp-ngram.
56    #[arg(
57        long = "speculative-types",
58        value_delimiter = ',',
59        conflicts_with = "no_speculative_tune"
60    )]
61    pub speculative_types: Vec<BenchmarkSpeculativeType>,
62    /// Disable speculative decoding sweeps and only benchmark the disabled baseline.
63    #[arg(
64        long = "no-speculative-tune",
65        conflicts_with_all = [
66            "speculative_types",
67            "spec_draft_models",
68            "spec_draft_max_tokens",
69            "spec_draft_min_tokens",
70            "spec_draft_acceptance_threshold",
71            "spec_draft_split_probability",
72            "spec_ngram_min",
73            "spec_ngram_max"
74        ]
75    )]
76    pub no_speculative_tune: bool,
77    /// Candidate draft GGUF paths to benchmark for speculative draft mode.
78    #[arg(long = "spec-draft-models", value_delimiter = ',')]
79    pub spec_draft_models: Vec<PathBuf>,
80    /// Candidate maximum draft-token windows for MTP and draft speculation.
81    #[arg(long = "spec-draft-max-tokens", value_delimiter = ',')]
82    pub spec_draft_max_tokens: Vec<u32>,
83    /// Candidate minimum draft-token windows for MTP and draft speculation.
84    #[arg(long = "spec-draft-min-tokens", value_delimiter = ',')]
85    pub spec_draft_min_tokens: Vec<u32>,
86    /// Candidate minimum match lengths for MTP + N-gram speculation.
87    #[arg(long = "spec-ngram-min", value_delimiter = ',')]
88    pub spec_ngram_min: Vec<u32>,
89    /// Candidate maximum match lengths for MTP + N-gram speculation.
90    #[arg(long = "spec-ngram-max", value_delimiter = ',')]
91    pub spec_ngram_max: Vec<u32>,
92    /// Candidate draft-acceptance-threshold values for speculative draft sweeps.
93    #[arg(long = "spec-draft-acceptance-threshold", value_delimiter = ',')]
94    pub spec_draft_acceptance_threshold: Vec<f64>,
95    /// Candidate draft-split-probability values for speculative draft sweeps.
96    #[arg(long = "spec-draft-split-probability", value_delimiter = ',')]
97    pub spec_draft_split_probability: Vec<f64>,
98    /// Persist the recommended settings to the local config file.
99    #[arg(long)]
100    pub apply: bool,
101    /// Replace existing writable config fields instead of preserving existing values.
102    #[arg(long, requires = "apply")]
103    pub replace_existing: bool,
104    /// Print launch-argument output instead of applying or reporting recommended fields.
105    #[arg(long)]
106    pub launch_args: bool,
107    /// Treat candidates within this percent of the raw best tok/s as throughput-equivalent.
108    #[arg(long, default_value_t = 10.0)]
109    pub throughput_tolerance_pct: f64,
110    /// Maximum generated tokens per benchmark request.
111    #[arg(long, default_value_t = 128)]
112    pub max_tokens: u32,
113    /// Startup wait limit for each benchmark trial.
114    #[arg(long, default_value_t = 600)]
115    pub startup_timeout_secs: u64,
116    /// HTTP request timeout for each benchmark request.
117    #[arg(long, default_value_t = 600)]
118    pub request_timeout_secs: u64,
119    /// Capture Skippy debug telemetry in each trial log.
120    #[arg(long)]
121    pub debug_telemetry: bool,
122    /// Prompt sent during benchmark trials.
123    #[arg(
124        long,
125        default_value = "Write a concise paragraph about distributed GPU inference."
126    )]
127    pub prompt: String,
128}
129
130#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)]
131pub enum BenchmarkBoolOrAuto {
132    Auto,
133    #[value(alias = "true")]
134    Enabled,
135    #[value(alias = "false")]
136    Disabled,
137}
138
139#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)]
140pub enum BenchmarkBool {
141    #[value(alias = "true")]
142    Enabled,
143    #[value(alias = "false")]
144    Disabled,
145}
146
147#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)]
148pub enum BenchmarkFlashAttention {
149    #[value(alias = "enabled", alias = "true", alias = "1")]
150    On,
151    #[value(alias = "disabled", alias = "false", alias = "0")]
152    Off,
153}
154
155#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)]
156pub enum BenchmarkSpeculativeType {
157    Auto,
158    Disabled,
159    Mtp,
160    Draft,
161    MtpNgram,
162}
163
164#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)]
165pub enum GpuBenchmarkBackend {
166    Metal,
167    Cuda,
168    Hip,
169    Intel,
170}
171
172#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)]
173pub enum PromptImportSource {
174    MtBench,
175    Gsm8k,
176    Humaneval,
177}
mesh_llm_cli/benchmark.rs

mesh_llm_cli/
benchmark.rs
