ferrum_cli/
config.rs

1//! CLI configuration management
2//!
3//! Handles loading and parsing of configuration files for the CLI tool.
4
5use ferrum_types::{
6    AttentionExecutionPolicy, Result, RuntimeConfigEntry, RuntimeConfigSource, SequenceFitPolicy,
7};
8use serde::{Deserialize, Serialize};
9use std::collections::HashMap;
10use std::path::Path;
11use tokio::fs;
12
13/// CLI configuration
14#[derive(Debug, Clone, Serialize, Deserialize, Default)]
15pub struct CliConfig {
16    /// Server configuration
17    pub server: ServerCliConfig,
18
19    /// Model configuration
20    pub models: ModelCliConfig,
21
22    /// Benchmark configuration
23    pub benchmark: BenchmarkConfig,
24
25    /// Client configuration
26    pub client: ClientConfig,
27
28    /// Development configuration
29    pub dev: DevConfig,
30
31    /// Runtime overrides loaded from the CLI config file.
32    #[serde(default)]
33    pub runtime: RuntimeCliConfig,
34}
35
36/// Server CLI configuration
37#[derive(Debug, Clone, Serialize, Deserialize)]
38pub struct ServerCliConfig {
39    /// Default host
40    pub host: String,
41
42    /// Default port
43    pub port: u16,
44
45    /// Configuration file path
46    pub config_path: String,
47
48    /// Log level
49    pub log_level: String,
50
51    /// Enable hot reload
52    pub hot_reload: bool,
53}
54
55/// Model CLI configuration
56#[derive(Debug, Clone, Serialize, Deserialize)]
57pub struct ModelCliConfig {
58    /// Default model directory
59    pub model_dir: String,
60
61    /// Model cache directory
62    pub cache_dir: String,
63
64    /// Default model
65    pub default_model: Option<String>,
66
67    /// Model aliases
68    pub aliases: HashMap<String, String>,
69
70    /// Download settings
71    pub download: DownloadConfig,
72}
73
74/// Download configuration
75#[derive(Debug, Clone, Serialize, Deserialize)]
76pub struct DownloadConfig {
77    /// HuggingFace cache directory
78    pub hf_cache_dir: String,
79
80    /// Download timeout in seconds
81    pub timeout_seconds: u64,
82
83    /// Max concurrent downloads
84    pub max_concurrent: usize,
85
86    /// Retry attempts
87    pub retry_attempts: u32,
88}
89
90/// Benchmark configuration
91#[derive(Debug, Clone, Serialize, Deserialize)]
92pub struct BenchmarkConfig {
93    /// Default number of requests
94    pub num_requests: usize,
95
96    /// Default concurrency level
97    pub concurrency: usize,
98
99    /// Default prompt length
100    pub prompt_length: usize,
101
102    /// Default max tokens
103    pub max_tokens: usize,
104
105    /// Warmup requests
106    pub warmup_requests: usize,
107
108    /// Output directory for reports
109    pub output_dir: String,
110}
111
112/// Client configuration
113#[derive(Debug, Clone, Serialize, Deserialize)]
114pub struct ClientConfig {
115    /// Default API base URL
116    pub base_url: String,
117
118    /// Default API key
119    pub api_key: Option<String>,
120
121    /// Request timeout
122    pub timeout_seconds: u64,
123
124    /// Retry configuration
125    pub retry: RetryConfig,
126}
127
128/// Retry configuration
129#[derive(Debug, Clone, Serialize, Deserialize)]
130pub struct RetryConfig {
131    /// Maximum retry attempts
132    pub max_attempts: u32,
133
134    /// Initial delay in milliseconds
135    pub initial_delay_ms: u64,
136
137    /// Maximum delay in milliseconds
138    pub max_delay_ms: u64,
139
140    /// Backoff multiplier
141    pub backoff_multiplier: f64,
142}
143
144/// Development configuration
145#[derive(Debug, Clone, Serialize, Deserialize)]
146pub struct DevConfig {
147    /// Enable debug mode
148    pub debug: bool,
149
150    /// Profile memory usage
151    pub profile_memory: bool,
152
153    /// Enable GPU profiling
154    pub profile_gpu: bool,
155
156    /// Mock backends for testing
157    pub mock_backends: bool,
158
159    /// Test data directory
160    pub test_data_dir: String,
161}
162
163/// Runtime knobs that can be sourced from the CLI config file.
164#[derive(Debug, Clone, Serialize, Deserialize, Default)]
165pub struct RuntimeCliConfig {
166    /// Named startup/runtime preset. Presets provide product-owned default
167    /// bundles and can still be overridden by explicit runtime keys below,
168    /// environment variables, or CLI flags.
169    #[serde(default)]
170    pub preset: Option<String>,
171
172    /// KV cache dtype override, equivalent to `--kv-dtype` or
173    /// `FERRUM_KV_DTYPE`.
174    #[serde(default)]
175    pub kv_dtype: Option<String>,
176
177    /// KV block budget, equivalent to `FERRUM_KV_MAX_BLOCKS`.
178    #[serde(default)]
179    pub kv_max_blocks: Option<usize>,
180
181    /// Per-sequence KV token capacity, equivalent to `FERRUM_KV_CAPACITY`.
182    #[serde(default)]
183    pub kv_capacity: Option<usize>,
184
185    /// Maximum paged-KV sequence count, equivalent to
186    /// `FERRUM_PAGED_MAX_SEQS`.
187    #[serde(default)]
188    pub paged_max_seqs: Option<usize>,
189
190    /// Generic recurrent-state slot-pool size, equivalent to
191    /// `FERRUM_RECURRENT_STATE_MAX_SLOTS`.
192    #[serde(default)]
193    pub recurrent_state_max_slots: Option<usize>,
194
195    /// Attention provider-family policy for the plan runtime. Physical
196    /// V1/V2/varlen selection remains adaptive inside the compiled provider.
197    #[serde(default)]
198    pub attention_policy: Option<AttentionExecutionPolicy>,
199
200    /// Scheduler/model max batched-token budget, equivalent to
201    /// `FERRUM_MAX_BATCHED_TOKENS`.
202    #[serde(default)]
203    pub max_batched_tokens: Option<usize>,
204
205    /// Prefer prefilling until this many requests are active, equivalent to
206    /// `FERRUM_SCHED_PREFILL_FIRST_UNTIL_ACTIVE`.
207    #[serde(default)]
208    pub scheduler_prefill_first_until_active: Option<usize>,
209
210    /// Cap prefill chunks while decode requests are active, equivalent to
211    /// `FERRUM_ACTIVE_DECODE_PREFILL_CHUNK`.
212    #[serde(default)]
213    pub scheduler_active_decode_prefill_chunk: Option<usize>,
214
215    /// Prefix cache opt-in, equivalent to `FERRUM_PREFIX_CACHE`.
216    #[serde(default)]
217    pub prefix_cache: Option<bool>,
218
219    /// Layer-split decode pipeline mode, equivalent to
220    /// `FERRUM_LAYER_SPLIT_PIPELINE_MODE`.
221    #[serde(default)]
222    pub layer_split_pipeline_mode: Option<String>,
223
224    /// MoE CUDA graph policy override, equivalent to `FERRUM_MOE_GRAPH`.
225    #[serde(default)]
226    pub moe_graph: Option<bool>,
227
228    /// Legacy Llama/Gemma batched decode CUDA graph policy override,
229    /// equivalent to `FERRUM_BATCHED_GRAPH`.
230    #[serde(default)]
231    pub batched_graph: Option<bool>,
232
233    /// vNext reusable device-program policy override, equivalent to
234    /// `FERRUM_REUSABLE_EXECUTION`.
235    #[serde(default)]
236    pub reusable_execution: Option<bool>,
237
238    /// Exact vNext reusable decode widths prepared at startup. Omitted means
239    /// automatic exact resolution up to the admission and startup hard bounds.
240    #[serde(default)]
241    pub reusable_execution_exact_decode_widths: Option<Vec<usize>>,
242
243    /// Configurable automatic ceiling, bounded by the independent hard startup
244    /// capture limit. It does not cap runtime concurrency.
245    #[serde(default)]
246    pub reusable_execution_max_automatic_exact_decode_width: Option<usize>,
247
248    /// Unified Llama/Gemma decode CUDA graph policy override,
249    /// equivalent to `FERRUM_UNIFIED_GRAPH`.
250    #[serde(default)]
251    pub unified_graph: Option<bool>,
252
253    /// Diagnostic unified graph scope that captures only transformer layers,
254    /// equivalent to `FERRUM_UNIFIED_GRAPH_LAYERS_ONLY`.
255    #[serde(default)]
256    pub unified_graph_layers_only: Option<bool>,
257
258    /// Diagnostic unified graph scope that leaves lm_head eager, equivalent to
259    /// `FERRUM_UNIFIED_GRAPH_LM_HEAD_EAGER`.
260    #[serde(default)]
261    pub unified_graph_lm_head_eager: Option<bool>,
262
263    /// Emit engine batch iteration profile logs, equivalent to
264    /// `FERRUM_BATCH_DECODE_PROF`.
265    #[serde(default)]
266    pub batch_decode_prof: Option<bool>,
267
268    /// Emit executor batch prefill profile logs, equivalent to
269    /// `FERRUM_BATCH_PREFILL_PROF`.
270    #[serde(default)]
271    pub batch_prefill_prof: Option<bool>,
272
273    /// Emit engine next-batch scheduler profile logs, equivalent to
274    /// `FERRUM_NEXT_BATCH_PROF`.
275    #[serde(default)]
276    pub next_batch_prof: Option<bool>,
277
278    /// Emit route/batched-decode profile logs, equivalent to
279    /// `FERRUM_RBD_PROF`.
280    #[serde(default)]
281    pub rbd_prof: Option<bool>,
282
283    /// Emit unified decode postprocess profile logs, equivalent to
284    /// `FERRUM_UNIFIED_POST_PROF`.
285    #[serde(default)]
286    pub unified_post_prof: Option<bool>,
287
288    /// Emit model decode operator profile logs, equivalent to
289    /// `FERRUM_DECODE_OP_PROFILE`.
290    #[serde(default)]
291    pub decode_op_profile: Option<bool>,
292
293    /// Emit model prefill operator profile logs, equivalent to
294    /// `FERRUM_PREFILL_OP_PROFILE`.
295    #[serde(default)]
296    pub prefill_op_profile: Option<bool>,
297
298    /// Emit dense Marlin inner-kernel timing counters, equivalent to
299    /// `FERRUM_MARLIN_PROFILE`.
300    #[serde(default)]
301    pub marlin_profile: Option<bool>,
302
303    /// Emit dense Marlin shape/label trace lines, equivalent to
304    /// `FERRUM_MARLIN_TRACE_SHAPES`.
305    #[serde(default)]
306    pub marlin_trace_shapes: Option<bool>,
307
308    /// Maximum dense Marlin shape/label trace lines, equivalent to
309    /// `FERRUM_MARLIN_TRACE_SHAPES_MAX`.
310    #[serde(default)]
311    pub marlin_trace_shapes_max: Option<usize>,
312
313    /// vLLM paged attention policy, equivalent to
314    /// `FERRUM_USE_VLLM_PAGED_ATTN`.
315    #[serde(default)]
316    pub use_vllm_paged_attn: Option<bool>,
317
318    /// Short-context vLLM paged-attention v1 policy, equivalent to
319    /// `FERRUM_VLLM_PAGED_ATTN_V1_SHORT`.
320    #[serde(default)]
321    pub vllm_paged_attn_v1_short: Option<bool>,
322
323    /// vLLM-Marlin MoE dispatch policy, equivalent to `FERRUM_VLLM_MOE`.
324    #[serde(default)]
325    pub vllm_moe: Option<bool>,
326
327    /// vLLM-MoE pair-id route layout policy, equivalent to
328    /// `FERRUM_VLLM_MOE_PAIR_IDS`.
329    #[serde(default)]
330    pub vllm_moe_pair_ids: Option<bool>,
331
332    /// GPU greedy argmax readback policy, equivalent to
333    /// `FERRUM_GREEDY_ARGMAX`.
334    #[serde(default)]
335    pub greedy_argmax: Option<bool>,
336
337    /// FA-compatible varlen K/V layout policy, equivalent to
338    /// `FERRUM_FA_LAYOUT_VARLEN`.
339    #[serde(default)]
340    pub fa_layout_varlen: Option<bool>,
341
342    /// Source-linked FA2 policy, equivalent to `FERRUM_FA2_SOURCE`.
343    #[serde(default)]
344    pub fa2_source: Option<bool>,
345
346    /// Runtime-loaded FA2 direct FFI policy, equivalent to
347    /// `FERRUM_FA2_DIRECT_FFI`.
348    #[serde(default)]
349    pub fa2_direct_ffi: Option<bool>,
350
351    /// Runtime-loaded FA2 direct FFI shim path, equivalent to
352    /// `FERRUM_FA2_DIRECT_FFI_SHIM`.
353    #[serde(default)]
354    pub fa2_direct_ffi_shim: Option<String>,
355
356    /// Ferrum native FA2 operator manifest path, equivalent to
357    /// `FERRUM_FA2_NATIVE_MANIFEST`.
358    #[serde(default)]
359    pub fa2_native_manifest: Option<String>,
360
361    /// Ferrum native FA2 operator artifact path, equivalent to
362    /// `FERRUM_FA2_NATIVE_ARTIFACT`.
363    #[serde(default)]
364    pub fa2_native_artifact: Option<String>,
365
366    /// Ferrum native FA2 source package sha256 pin, equivalent to
367    /// `FERRUM_FA2_NATIVE_SOURCE_SHA256`.
368    #[serde(default)]
369    pub fa2_native_source_sha256: Option<String>,
370
371    /// Ferrum native FA2 input tree sha256 pin, equivalent to
372    /// `FERRUM_FA2_NATIVE_INPUTS_SHA256`.
373    #[serde(default)]
374    pub fa2_native_inputs_sha256: Option<String>,
375
376    /// Requested max model length, equivalent to `FERRUM_MAX_MODEL_LEN`.
377    #[serde(default)]
378    pub max_model_len: Option<usize>,
379
380    /// Sequence fit gate used before prefill admission.
381    #[serde(default)]
382    pub sequence_fit_policy: Option<SequenceFitPolicy>,
383
384    /// Minimum MoE batch size for the batched expert path, equivalent to
385    /// `FERRUM_MOE_BATCH_THRESHOLD`.
386    #[serde(default)]
387    pub moe_batch_threshold: Option<usize>,
388}
389
390impl RuntimeCliConfig {
391    pub fn runtime_config_entries(&self) -> Vec<RuntimeConfigEntry> {
392        let mut entries = Vec::new();
393        push_string_entry(&mut entries, "FERRUM_KV_DTYPE", self.kv_dtype.as_deref());
394        push_usize_entry(&mut entries, "FERRUM_KV_MAX_BLOCKS", self.kv_max_blocks);
395        push_usize_entry(&mut entries, "FERRUM_KV_CAPACITY", self.kv_capacity);
396        push_usize_entry(&mut entries, "FERRUM_PAGED_MAX_SEQS", self.paged_max_seqs);
397        push_usize_entry(
398            &mut entries,
399            "FERRUM_RECURRENT_STATE_MAX_SLOTS",
400            self.recurrent_state_max_slots,
401        );
402        push_string_entry(
403            &mut entries,
404            "FERRUM_ATTENTION_POLICY",
405            self.attention_policy
406                .map(AttentionExecutionPolicy::as_runtime_value),
407        );
408        push_usize_entry(
409            &mut entries,
410            "FERRUM_MAX_BATCHED_TOKENS",
411            self.max_batched_tokens,
412        );
413        push_usize_entry(
414            &mut entries,
415            "FERRUM_SCHED_PREFILL_FIRST_UNTIL_ACTIVE",
416            self.scheduler_prefill_first_until_active,
417        );
418        push_usize_entry(
419            &mut entries,
420            "FERRUM_ACTIVE_DECODE_PREFILL_CHUNK",
421            self.scheduler_active_decode_prefill_chunk,
422        );
423        push_bool_entry(&mut entries, "FERRUM_PREFIX_CACHE", self.prefix_cache);
424        push_string_entry(
425            &mut entries,
426            "FERRUM_LAYER_SPLIT_PIPELINE_MODE",
427            self.layer_split_pipeline_mode.as_deref(),
428        );
429        push_bool_entry(&mut entries, "FERRUM_MOE_GRAPH", self.moe_graph);
430        push_bool_entry(&mut entries, "FERRUM_BATCHED_GRAPH", self.batched_graph);
431        push_bool_entry(
432            &mut entries,
433            "FERRUM_REUSABLE_EXECUTION",
434            self.reusable_execution,
435        );
436        push_usize_list_entry(
437            &mut entries,
438            "FERRUM_REUSABLE_EXECUTION_EXACT_DECODE_WIDTHS",
439            self.reusable_execution_exact_decode_widths.as_deref(),
440        );
441        push_usize_entry(
442            &mut entries,
443            "FERRUM_REUSABLE_EXECUTION_MAX_AUTOMATIC_EXACT_DECODE_WIDTH",
444            self.reusable_execution_max_automatic_exact_decode_width,
445        );
446        push_bool_entry(&mut entries, "FERRUM_UNIFIED_GRAPH", self.unified_graph);
447        push_bool_entry(
448            &mut entries,
449            "FERRUM_UNIFIED_GRAPH_LAYERS_ONLY",
450            self.unified_graph_layers_only,
451        );
452        push_bool_entry(
453            &mut entries,
454            "FERRUM_UNIFIED_GRAPH_LM_HEAD_EAGER",
455            self.unified_graph_lm_head_eager,
456        );
457        push_true_entry(
458            &mut entries,
459            "FERRUM_BATCH_DECODE_PROF",
460            self.batch_decode_prof,
461        );
462        push_true_entry(
463            &mut entries,
464            "FERRUM_BATCH_PREFILL_PROF",
465            self.batch_prefill_prof,
466        );
467        push_true_entry(&mut entries, "FERRUM_NEXT_BATCH_PROF", self.next_batch_prof);
468        push_true_entry(&mut entries, "FERRUM_RBD_PROF", self.rbd_prof);
469        push_true_entry(
470            &mut entries,
471            "FERRUM_UNIFIED_POST_PROF",
472            self.unified_post_prof,
473        );
474        push_true_entry(
475            &mut entries,
476            "FERRUM_DECODE_OP_PROFILE",
477            self.decode_op_profile,
478        );
479        push_true_entry(
480            &mut entries,
481            "FERRUM_PREFILL_OP_PROFILE",
482            self.prefill_op_profile,
483        );
484        push_true_entry(&mut entries, "FERRUM_MARLIN_PROFILE", self.marlin_profile);
485        push_true_entry(
486            &mut entries,
487            "FERRUM_MARLIN_TRACE_SHAPES",
488            self.marlin_trace_shapes,
489        );
490        push_usize_entry(
491            &mut entries,
492            "FERRUM_MARLIN_TRACE_SHAPES_MAX",
493            self.marlin_trace_shapes_max,
494        );
495        push_bool_entry(
496            &mut entries,
497            "FERRUM_USE_VLLM_PAGED_ATTN",
498            self.use_vllm_paged_attn,
499        );
500        push_bool_entry(
501            &mut entries,
502            "FERRUM_VLLM_PAGED_ATTN_V1_SHORT",
503            self.vllm_paged_attn_v1_short,
504        );
505        push_bool_entry(&mut entries, "FERRUM_VLLM_MOE", self.vllm_moe);
506        push_bool_entry(
507            &mut entries,
508            "FERRUM_VLLM_MOE_PAIR_IDS",
509            self.vllm_moe_pair_ids,
510        );
511        push_bool_entry(&mut entries, "FERRUM_GREEDY_ARGMAX", self.greedy_argmax);
512        push_bool_entry(
513            &mut entries,
514            "FERRUM_FA_LAYOUT_VARLEN",
515            self.fa_layout_varlen,
516        );
517        push_bool_entry(&mut entries, "FERRUM_FA2_SOURCE", self.fa2_source);
518        push_bool_entry(&mut entries, "FERRUM_FA2_DIRECT_FFI", self.fa2_direct_ffi);
519        push_string_entry(
520            &mut entries,
521            "FERRUM_FA2_DIRECT_FFI_SHIM",
522            self.fa2_direct_ffi_shim.as_deref(),
523        );
524        push_string_entry(
525            &mut entries,
526            "FERRUM_FA2_NATIVE_MANIFEST",
527            self.fa2_native_manifest.as_deref(),
528        );
529        push_string_entry(
530            &mut entries,
531            "FERRUM_FA2_NATIVE_ARTIFACT",
532            self.fa2_native_artifact.as_deref(),
533        );
534        push_string_entry(
535            &mut entries,
536            "FERRUM_FA2_NATIVE_SOURCE_SHA256",
537            self.fa2_native_source_sha256.as_deref(),
538        );
539        push_string_entry(
540            &mut entries,
541            "FERRUM_FA2_NATIVE_INPUTS_SHA256",
542            self.fa2_native_inputs_sha256.as_deref(),
543        );
544        push_usize_entry(&mut entries, "FERRUM_MAX_MODEL_LEN", self.max_model_len);
545        push_string_entry(
546            &mut entries,
547            "FERRUM_SEQUENCE_FIT_POLICY",
548            self.sequence_fit_policy
549                .map(SequenceFitPolicy::as_runtime_value),
550        );
551        push_usize_entry(
552            &mut entries,
553            "FERRUM_MOE_BATCH_THRESHOLD",
554            self.moe_batch_threshold,
555        );
556        entries
557    }
558}
559
560fn push_string_entry(entries: &mut Vec<RuntimeConfigEntry>, key: &str, value: Option<&str>) {
561    if let Some(value) = value.filter(|value| !value.trim().is_empty()) {
562        entries.push(RuntimeConfigEntry::new(
563            key,
564            value.to_string(),
565            RuntimeConfigSource::ConfigFile,
566        ));
567    }
568}
569
570fn push_usize_entry(entries: &mut Vec<RuntimeConfigEntry>, key: &str, value: Option<usize>) {
571    if let Some(value) = value {
572        entries.push(RuntimeConfigEntry::new(
573            key,
574            value.to_string(),
575            RuntimeConfigSource::ConfigFile,
576        ));
577    }
578}
579
580fn push_usize_list_entry(
581    entries: &mut Vec<RuntimeConfigEntry>,
582    key: &str,
583    value: Option<&[usize]>,
584) {
585    if let Some(value) = value {
586        entries.push(RuntimeConfigEntry::new(
587            key,
588            value
589                .iter()
590                .map(usize::to_string)
591                .collect::<Vec<_>>()
592                .join(","),
593            RuntimeConfigSource::ConfigFile,
594        ));
595    }
596}
597
598fn push_bool_entry(entries: &mut Vec<RuntimeConfigEntry>, key: &str, value: Option<bool>) {
599    if let Some(value) = value {
600        entries.push(RuntimeConfigEntry::new(
601            key,
602            if value { "1" } else { "0" },
603            RuntimeConfigSource::ConfigFile,
604        ));
605    }
606}
607
608fn push_true_entry(entries: &mut Vec<RuntimeConfigEntry>, key: &str, value: Option<bool>) {
609    if value == Some(true) {
610        entries.push(RuntimeConfigEntry::new(
611            key,
612            "1".to_string(),
613            RuntimeConfigSource::ConfigFile,
614        ));
615    }
616}
617
618impl CliConfig {
619    /// Load configuration from file, falling back to typed defaults when the
620    /// optional file does not exist. Loading config must not mutate the
621    /// caller's current directory.
622    pub async fn load<P: AsRef<Path>>(path: P) -> Result<Self> {
623        let path = path.as_ref();
624
625        if !path.exists() {
626            return Ok(Self::default());
627        }
628
629        let content = fs::read_to_string(path).await.map_err(|e| {
630            ferrum_types::FerrumError::io_str(format!("Failed to read config file: {}", e))
631        })?;
632
633        toml::from_str(&content).map_err(|e| {
634            ferrum_types::FerrumError::configuration(format!("Failed to parse config: {}", e))
635        })
636    }
637
638    /// Save configuration to file
639    pub async fn save<P: AsRef<Path>>(&self, path: P) -> Result<()> {
640        let content = toml::to_string_pretty(self).map_err(|e| {
641            ferrum_types::FerrumError::configuration(format!("Failed to serialize config: {}", e))
642        })?;
643
644        fs::write(path, content).await.map_err(|e| {
645            ferrum_types::FerrumError::io_str(format!("Failed to write config file: {}", e))
646        })
647    }
648
649    /// Validate configuration
650    pub fn validate(&self) -> Result<()> {
651        // Validate server config
652        if self.server.port == 0 {
653            return Err(ferrum_types::FerrumError::configuration(
654                "Server port cannot be 0".to_string(),
655            ));
656        }
657
658        // Validate model config
659        if !Path::new(&self.models.model_dir).exists() {
660            return Err(ferrum_types::FerrumError::configuration(format!(
661                "Model directory does not exist: {}",
662                self.models.model_dir
663            )));
664        }
665
666        // Validate benchmark config
667        if self.benchmark.num_requests == 0 {
668            return Err(ferrum_types::FerrumError::configuration(
669                "Number of requests cannot be 0".to_string(),
670            ));
671        }
672
673        if self.benchmark.concurrency == 0 {
674            return Err(ferrum_types::FerrumError::configuration(
675                "Concurrency cannot be 0".to_string(),
676            ));
677        }
678
679        Ok(())
680    }
681}
682
683impl Default for ServerCliConfig {
684    fn default() -> Self {
685        Self {
686            host: "127.0.0.1".to_string(),
687            port: 8000,
688            config_path: "server.toml".to_string(),
689            log_level: "info".to_string(),
690            hot_reload: false,
691        }
692    }
693}
694
695impl Default for ModelCliConfig {
696    fn default() -> Self {
697        Self {
698            model_dir: "./models".to_string(),
699            cache_dir: "./cache".to_string(),
700            default_model: None,
701            aliases: HashMap::new(),
702            download: DownloadConfig::default(),
703        }
704    }
705}
706
707impl Default for DownloadConfig {
708    fn default() -> Self {
709        Self {
710            hf_cache_dir: std::env::var("HF_HOME")
711                .ok()
712                .or_else(|| {
713                    dirs::home_dir()
714                        .map(|h| h.join(".cache/huggingface").to_string_lossy().to_string())
715                })
716                .unwrap_or_else(|| "./hf_cache".to_string()),
717            timeout_seconds: 300,
718            max_concurrent: 4,
719            retry_attempts: 3,
720        }
721    }
722}
723
724impl Default for BenchmarkConfig {
725    fn default() -> Self {
726        Self {
727            num_requests: 100,
728            concurrency: 10,
729            prompt_length: 512,
730            max_tokens: 256,
731            warmup_requests: 10,
732            output_dir: "./benchmark_results".to_string(),
733        }
734    }
735}
736
737impl Default for ClientConfig {
738    fn default() -> Self {
739        Self {
740            base_url: "http://127.0.0.1:8000".to_string(),
741            api_key: None,
742            timeout_seconds: 30,
743            retry: RetryConfig::default(),
744        }
745    }
746}
747
748impl Default for RetryConfig {
749    fn default() -> Self {
750        Self {
751            max_attempts: 3,
752            initial_delay_ms: 100,
753            max_delay_ms: 5000,
754            backoff_multiplier: 2.0,
755        }
756    }
757}
758
759impl Default for DevConfig {
760    fn default() -> Self {
761        Self {
762            debug: false,
763            profile_memory: false,
764            profile_gpu: false,
765            mock_backends: false,
766            test_data_dir: "./test_data".to_string(),
767        }
768    }
769}
770
771#[cfg(test)]
772mod tests {
773    use super::*;
774    use ferrum_types::RuntimeConfigEffect;
775
776    #[tokio::test]
777    async fn missing_optional_config_uses_defaults_without_creating_a_file() {
778        let nonce = std::time::SystemTime::now()
779            .duration_since(std::time::UNIX_EPOCH)
780            .unwrap()
781            .as_nanos();
782        let path = std::env::temp_dir().join(format!(
783            "ferrum-missing-config-{}-{nonce}.toml",
784            std::process::id()
785        ));
786
787        assert!(!path.exists());
788        let config = CliConfig::load(&path).await.unwrap();
789
790        assert!(
791            !path.exists(),
792            "loading an optional config must be read-only"
793        );
794        assert_eq!(config.server.port, 8000);
795        assert!(config.models.default_model.is_none());
796    }
797
798    #[test]
799    fn runtime_cli_config_emits_config_file_source_entries() {
800        let runtime = RuntimeCliConfig {
801            preset: Some("m3_qwen3_30b_a3b_int4".to_string()),
802            kv_dtype: Some("int8".to_string()),
803            kv_max_blocks: Some(4096),
804            kv_capacity: Some(2048),
805            paged_max_seqs: Some(64),
806            recurrent_state_max_slots: Some(16),
807            attention_policy: Some(AttentionExecutionPolicy::NativeAdaptive),
808            max_batched_tokens: Some(2048),
809            scheduler_prefill_first_until_active: Some(16),
810            scheduler_active_decode_prefill_chunk: Some(24),
811            prefix_cache: Some(false),
812            layer_split_pipeline_mode: Some("batch".to_string()),
813            moe_graph: Some(true),
814            batched_graph: Some(true),
815            reusable_execution: Some(false),
816            reusable_execution_exact_decode_widths: Some(vec![1, 2, 4, 8, 16, 24, 32]),
817            reusable_execution_max_automatic_exact_decode_width: Some(32),
818            unified_graph: Some(true),
819            unified_graph_layers_only: Some(true),
820            unified_graph_lm_head_eager: Some(true),
821            batch_decode_prof: Some(true),
822            batch_prefill_prof: Some(true),
823            next_batch_prof: Some(true),
824            rbd_prof: Some(true),
825            unified_post_prof: Some(true),
826            decode_op_profile: Some(true),
827            prefill_op_profile: Some(true),
828            marlin_profile: Some(true),
829            marlin_trace_shapes: Some(true),
830            marlin_trace_shapes_max: Some(17),
831            use_vllm_paged_attn: Some(true),
832            vllm_paged_attn_v1_short: Some(false),
833            vllm_moe: Some(true),
834            vllm_moe_pair_ids: Some(true),
835            greedy_argmax: Some(true),
836            fa_layout_varlen: Some(true),
837            fa2_source: Some(true),
838            fa2_direct_ffi: Some(false),
839            fa2_direct_ffi_shim: Some("/tmp/libferrum_fa2_shim.so".to_string()),
840            fa2_native_manifest: Some("/tmp/native/fa2/native_operator_manifest.json".to_string()),
841            fa2_native_artifact: Some("/tmp/native/fa2/libferrum_native_fa2.a".to_string()),
842            fa2_native_source_sha256: Some(
843                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".to_string(),
844            ),
845            fa2_native_inputs_sha256: Some(
846                "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb".to_string(),
847            ),
848            max_model_len: Some(4096),
849            sequence_fit_policy: Some(SequenceFitPolicy::FullInputMustFit),
850            moe_batch_threshold: Some(4),
851            ..Default::default()
852        };
853        let entries = runtime.runtime_config_entries();
854        assert_eq!(entries.len(), 45);
855        let entry = |key: &str| {
856            entries
857                .iter()
858                .find(|entry| entry.key == key)
859                .unwrap_or_else(|| panic!("missing {key}"))
860        };
861        assert_eq!(entry("FERRUM_KV_DTYPE").effective_value, "int8");
862        assert_eq!(
863            entry("FERRUM_KV_DTYPE").source,
864            RuntimeConfigSource::ConfigFile
865        );
866        assert!(entry("FERRUM_KV_DTYPE")
867            .affects
868            .contains(&RuntimeConfigEffect::Correctness));
869        assert_eq!(entry("FERRUM_KV_MAX_BLOCKS").effective_value, "4096");
870        assert_eq!(entry("FERRUM_KV_CAPACITY").effective_value, "2048");
871        assert_eq!(entry("FERRUM_PAGED_MAX_SEQS").effective_value, "64");
872        assert_eq!(
873            entry("FERRUM_RECURRENT_STATE_MAX_SLOTS").effective_value,
874            "16"
875        );
876        assert!(entry("FERRUM_RECURRENT_STATE_MAX_SLOTS")
877            .affects
878            .contains(&RuntimeConfigEffect::Memory));
879        assert_eq!(
880            entry("FERRUM_ATTENTION_POLICY").effective_value,
881            "native-adaptive"
882        );
883        assert!(entry("FERRUM_ATTENTION_POLICY")
884            .affects
885            .contains(&RuntimeConfigEffect::Correctness));
886        assert!(entry("FERRUM_ATTENTION_POLICY")
887            .affects
888            .contains(&RuntimeConfigEffect::Performance));
889        assert_eq!(entry("FERRUM_MAX_BATCHED_TOKENS").effective_value, "2048");
890        assert_eq!(
891            entry("FERRUM_SCHED_PREFILL_FIRST_UNTIL_ACTIVE").effective_value,
892            "16"
893        );
894        assert_eq!(
895            entry("FERRUM_ACTIVE_DECODE_PREFILL_CHUNK").effective_value,
896            "24"
897        );
898        assert_eq!(
899            entry("FERRUM_LAYER_SPLIT_PIPELINE_MODE").effective_value,
900            "batch"
901        );
902        assert_eq!(entry("FERRUM_PREFIX_CACHE").effective_value, "0");
903        assert_eq!(entry("FERRUM_MOE_GRAPH").effective_value, "1");
904        assert_eq!(entry("FERRUM_BATCHED_GRAPH").effective_value, "1");
905        assert_eq!(entry("FERRUM_REUSABLE_EXECUTION").effective_value, "0");
906        assert_eq!(
907            entry("FERRUM_REUSABLE_EXECUTION_EXACT_DECODE_WIDTHS").effective_value,
908            "1,2,4,8,16,24,32"
909        );
910        assert_eq!(
911            entry("FERRUM_REUSABLE_EXECUTION_MAX_AUTOMATIC_EXACT_DECODE_WIDTH").effective_value,
912            "32"
913        );
914        assert_eq!(entry("FERRUM_UNIFIED_GRAPH").effective_value, "1");
915        assert_eq!(
916            entry("FERRUM_UNIFIED_GRAPH_LAYERS_ONLY").effective_value,
917            "1"
918        );
919        assert_eq!(
920            entry("FERRUM_UNIFIED_GRAPH_LM_HEAD_EAGER").effective_value,
921            "1"
922        );
923        assert_eq!(entry("FERRUM_BATCH_DECODE_PROF").effective_value, "1");
924        assert!(entry("FERRUM_BATCH_DECODE_PROF")
925            .affects
926            .contains(&RuntimeConfigEffect::Diagnostics));
927        assert_eq!(entry("FERRUM_BATCH_PREFILL_PROF").effective_value, "1");
928        assert_eq!(entry("FERRUM_NEXT_BATCH_PROF").effective_value, "1");
929        assert_eq!(entry("FERRUM_RBD_PROF").effective_value, "1");
930        assert_eq!(entry("FERRUM_UNIFIED_POST_PROF").effective_value, "1");
931        assert_eq!(entry("FERRUM_DECODE_OP_PROFILE").effective_value, "1");
932        assert_eq!(entry("FERRUM_PREFILL_OP_PROFILE").effective_value, "1");
933        assert_eq!(entry("FERRUM_MARLIN_PROFILE").effective_value, "1");
934        assert!(entry("FERRUM_MARLIN_PROFILE")
935            .affects
936            .contains(&RuntimeConfigEffect::Diagnostics));
937        assert_eq!(entry("FERRUM_MARLIN_TRACE_SHAPES").effective_value, "1");
938        assert_eq!(
939            entry("FERRUM_MARLIN_TRACE_SHAPES_MAX").effective_value,
940            "17"
941        );
942        assert_eq!(entry("FERRUM_USE_VLLM_PAGED_ATTN").effective_value, "1");
943        assert_eq!(
944            entry("FERRUM_VLLM_PAGED_ATTN_V1_SHORT").effective_value,
945            "0"
946        );
947        assert_eq!(entry("FERRUM_VLLM_MOE").effective_value, "1");
948        assert_eq!(entry("FERRUM_VLLM_MOE_PAIR_IDS").effective_value, "1");
949        assert_eq!(entry("FERRUM_GREEDY_ARGMAX").effective_value, "1");
950        assert_eq!(entry("FERRUM_FA_LAYOUT_VARLEN").effective_value, "1");
951        assert_eq!(entry("FERRUM_FA2_SOURCE").effective_value, "1");
952        assert_eq!(entry("FERRUM_FA2_DIRECT_FFI").effective_value, "0");
953        assert_eq!(
954            entry("FERRUM_FA2_DIRECT_FFI_SHIM").effective_value,
955            "/tmp/libferrum_fa2_shim.so"
956        );
957        assert_eq!(
958            entry("FERRUM_FA2_NATIVE_MANIFEST").effective_value,
959            "/tmp/native/fa2/native_operator_manifest.json"
960        );
961        assert_eq!(
962            entry("FERRUM_FA2_NATIVE_ARTIFACT").effective_value,
963            "/tmp/native/fa2/libferrum_native_fa2.a"
964        );
965        assert_eq!(
966            entry("FERRUM_FA2_NATIVE_SOURCE_SHA256").effective_value,
967            "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
968        );
969        assert_eq!(
970            entry("FERRUM_FA2_NATIVE_INPUTS_SHA256").effective_value,
971            "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
972        );
973        assert_eq!(entry("FERRUM_MAX_MODEL_LEN").effective_value, "4096");
974        assert_eq!(
975            entry("FERRUM_SEQUENCE_FIT_POLICY").effective_value,
976            "full-input-must-fit"
977        );
978        assert_eq!(entry("FERRUM_MOE_BATCH_THRESHOLD").effective_value, "4");
979    }
980
981    #[test]
982    fn runtime_cli_config_diagnostic_presence_flags_are_opt_in() {
983        let entries = RuntimeCliConfig {
984            batch_decode_prof: Some(false),
985            batch_prefill_prof: Some(false),
986            next_batch_prof: Some(false),
987            rbd_prof: Some(false),
988            unified_post_prof: Some(false),
989            decode_op_profile: Some(false),
990            prefill_op_profile: Some(false),
991            marlin_profile: Some(false),
992            marlin_trace_shapes: Some(false),
993            ..Default::default()
994        }
995        .runtime_config_entries();
996
997        assert!(
998            entries.is_empty(),
999            "false diagnostic presence flags must not materialize as FERRUM_*_PROF=0"
1000        );
1001    }
1002
1003    #[test]
1004    fn runtime_cli_config_defaults_when_missing_from_toml() {
1005        let config: CliConfig = toml::from_str(
1006            r#"
1007            [server]
1008            host = "127.0.0.1"
1009            port = 8000
1010            config_path = "server.toml"
1011            log_level = "info"
1012            hot_reload = false
1013
1014            [models]
1015            model_dir = "./models"
1016            cache_dir = "./cache"
1017
1018            [models.aliases]
1019
1020            [models.download]
1021            hf_cache_dir = "./hf_cache"
1022            timeout_seconds = 300
1023            max_concurrent = 4
1024            retry_attempts = 3
1025
1026            [benchmark]
1027            num_requests = 100
1028            concurrency = 10
1029            prompt_length = 512
1030            max_tokens = 256
1031            warmup_requests = 10
1032            output_dir = "./benchmark_results"
1033
1034            [client]
1035            base_url = "http://127.0.0.1:8000"
1036            timeout_seconds = 30
1037
1038            [client.retry]
1039            max_attempts = 3
1040            initial_delay_ms = 100
1041            max_delay_ms = 5000
1042            backoff_multiplier = 2.0
1043
1044            [dev]
1045            debug = false
1046            profile_memory = false
1047            profile_gpu = false
1048            mock_backends = false
1049            test_data_dir = "./test_data"
1050            "#,
1051        )
1052        .unwrap();
1053        assert!(config.runtime.preset.is_none());
1054        assert!(config.runtime.kv_dtype.is_none());
1055        assert!(config.runtime.runtime_config_entries().is_empty());
1056    }
1057}
ferrum_cli/config.rs

ferrum_cli/
config.rs
