mesh_llm_config/
model.rs

1mod built_in_schema;
2mod runtime;
3mod schema_types;
4
5pub use built_in_schema::{
6    BuiltInConfigPathResolution, built_in_config_schema_descriptor, built_in_config_settings,
7    canonicalize_built_in_config_identifier, canonicalize_built_in_config_path,
8    resolve_built_in_config_identifier, resolve_built_in_config_path,
9};
10pub use runtime::{
11    ActivityAdvertisement, ActivityResponse, DEFAULT_DRAIN_TIMEOUT_MAX_SECS,
12    DEFAULT_DRAIN_TIMEOUT_SECS, RuntimeActivityConfig, RuntimeMode, StartupFailurePolicy,
13};
14pub use schema_types::*;
15
16pub use mesh_llm_types::runtime::ModelRuntimeKind;
17use serde::ser::SerializeStruct;
18use serde::{Deserialize, Serialize};
19pub use skippy_protocol::FlashAttentionType;
20use std::collections::BTreeMap;
21
22#[derive(Clone, Debug, Default, Serialize)]
23pub struct MeshConfig {
24    #[serde(default)]
25    pub version: Option<u32>,
26    #[serde(default)]
27    pub gpu: GpuConfig,
28    #[serde(default)]
29    pub mesh_requirements: MeshRequirementsConfig,
30    #[serde(default)]
31    pub owner_control: OwnerControlConfig,
32    #[serde(default)]
33    pub telemetry: TelemetryConfig,
34    #[serde(default)]
35    pub defaults: Option<ModelConfigDefaults>,
36    #[serde(default)]
37    pub runtime: RuntimeConfig,
38    #[serde(default)]
39    pub models: Vec<ModelConfigEntry>,
40    #[serde(rename = "plugin", default)]
41    pub plugins: Vec<PluginConfigEntry>,
42    #[serde(flatten, default)]
43    pub extra: BTreeMap<String, toml::Value>,
44}
45
46#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
47pub struct OwnerControlConfig {
48    #[serde(default)]
49    pub bind: Option<std::net::SocketAddr>,
50    #[serde(default)]
51    pub advertise_addr: Option<std::net::SocketAddr>,
52}
53
54#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
55pub struct GpuConfig {
56    #[serde(default)]
57    pub assignment: GpuAssignment,
58    #[serde(default)]
59    pub parallel: Option<usize>,
60}
61
62pub const DEFAULT_MODEL_TARGET_DEMAND_UPGRADE_MIN_REQUESTS: u64 = 2;
63pub const DEFAULT_MODEL_TARGET_DEMAND_UPGRADE_MAX_AGE_SECS: u64 = 60 * 60;
64
65fn default_model_target_demand_upgrade_min_requests() -> u64 {
66    DEFAULT_MODEL_TARGET_DEMAND_UPGRADE_MIN_REQUESTS
67}
68
69fn default_model_target_demand_upgrade_max_age_secs() -> u64 {
70    DEFAULT_MODEL_TARGET_DEMAND_UPGRADE_MAX_AGE_SECS
71}
72
73fn default_drain_timeout_secs() -> u64 {
74    runtime::DEFAULT_DRAIN_TIMEOUT_SECS
75}
76
77fn default_drain_timeout_max_secs() -> u64 {
78    runtime::DEFAULT_DRAIN_TIMEOUT_MAX_SECS
79}
80
81#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
82pub struct RuntimeConfig {
83    #[serde(default)]
84    pub debug: bool,
85    #[serde(default)]
86    pub listen_all: bool,
87    /// Operating mode. Absent resolves to `Serve` for backward compatibility.
88    #[serde(default)]
89    pub mode: runtime::RuntimeMode,
90    /// How the runtime reacts when a model fails to load during startup.
91    #[serde(default)]
92    pub startup_failure_policy: runtime::StartupFailurePolicy,
93    /// Seconds before forcibly unloading a draining instance (default 30).
94    #[serde(default = "default_drain_timeout_secs")]
95    pub drain_timeout_secs: u64,
96    /// Maximum allowed drain timeout cap in seconds (default 300).
97    #[serde(default = "default_drain_timeout_max_secs")]
98    pub drain_timeout_max_secs: u64,
99    /// Host activity detection and response policy.
100    #[serde(default)]
101    pub activity: runtime::RuntimeActivityConfig,
102    #[serde(default)]
103    pub reconcile_model_targets: bool,
104    #[serde(default)]
105    pub reconcile_model_target_demand_upgrades: bool,
106    #[serde(default)]
107    pub native_runtime: NativeRuntimeConfig,
108    #[serde(default = "default_model_target_demand_upgrade_min_requests")]
109    pub model_target_demand_upgrade_min_requests: u64,
110    #[serde(default = "default_model_target_demand_upgrade_max_age_secs")]
111    pub model_target_demand_upgrade_max_age_secs: u64,
112}
113
114impl Default for RuntimeConfig {
115    fn default() -> Self {
116        Self {
117            debug: false,
118            listen_all: false,
119            mode: runtime::RuntimeMode::default(),
120            startup_failure_policy: runtime::StartupFailurePolicy::default(),
121            drain_timeout_secs: runtime::DEFAULT_DRAIN_TIMEOUT_SECS,
122            drain_timeout_max_secs: runtime::DEFAULT_DRAIN_TIMEOUT_MAX_SECS,
123            activity: runtime::RuntimeActivityConfig::default(),
124            reconcile_model_targets: false,
125            reconcile_model_target_demand_upgrades: false,
126            native_runtime: NativeRuntimeConfig::default(),
127            model_target_demand_upgrade_min_requests:
128                DEFAULT_MODEL_TARGET_DEMAND_UPGRADE_MIN_REQUESTS,
129            model_target_demand_upgrade_max_age_secs:
130                DEFAULT_MODEL_TARGET_DEMAND_UPGRADE_MAX_AGE_SECS,
131        }
132    }
133}
134
135#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
136pub struct NativeRuntimeConfig {
137    #[serde(default)]
138    pub mesh_version: Option<String>,
139    #[serde(default)]
140    pub skippy_abi: Option<String>,
141    #[serde(default)]
142    pub selection: Option<String>,
143}
144
145#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
146pub struct MeshRequirementsConfig {
147    #[serde(default)]
148    pub min_node_version: Option<String>,
149    #[serde(default)]
150    pub max_node_version: Option<String>,
151    #[serde(default)]
152    pub min_protocol_version: Option<u32>,
153    #[serde(default)]
154    pub max_protocol_version: Option<u32>,
155    #[serde(default)]
156    pub require_release_attestation: bool,
157    #[serde(default)]
158    pub release_signer_keys: Vec<String>,
159}
160
161#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
162#[serde(rename_all = "lowercase")]
163pub enum GpuAssignment {
164    #[default]
165    Auto,
166    Pinned,
167}
168
169#[derive(Clone, Debug, Default, Serialize)]
170pub struct ModelConfigDefaults {
171    #[serde(default)]
172    pub model_fit: Option<ModelFitConfig>,
173    #[serde(default)]
174    pub hardware: Option<HardwareConfig>,
175    #[serde(default)]
176    pub throughput: Option<ThroughputConfig>,
177    #[serde(default)]
178    pub skippy: Option<SkippyConfig>,
179    #[serde(default)]
180    pub speculative: Option<SpeculativeConfig>,
181    #[serde(default)]
182    pub request_defaults: Option<RequestDefaultsConfig>,
183    #[serde(default)]
184    pub multimodal: Option<MultimodalConfig>,
185    #[serde(default)]
186    pub advanced: Option<AdvancedConfig>,
187}
188
189#[derive(Clone, Debug, Default)]
190pub struct ModelConfigEntry {
191    pub model: String,
192    pub mmproj: Option<String>,
193    pub ctx_size: Option<u32>,
194    pub gpu_id: Option<String>,
195    pub parallel: Option<usize>,
196    pub cache_type_k: Option<String>,
197    pub cache_type_v: Option<String>,
198    pub batch: Option<u32>,
199    pub ubatch: Option<u32>,
200    pub flash_attention: Option<FlashAttentionType>,
201    pub model_fit: Option<ModelFitConfig>,
202    pub hardware: Option<HardwareConfig>,
203    pub throughput: Option<ThroughputConfig>,
204    pub skippy: Option<SkippyConfig>,
205    pub speculative: Option<SpeculativeConfig>,
206    pub request_defaults: Option<RequestDefaultsConfig>,
207    pub multimodal: Option<MultimodalConfig>,
208    pub advanced: Option<AdvancedConfig>,
209    pub gpu_id_from_legacy_shim: bool,
210}
211
212impl Serialize for ModelConfigEntry {
213    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
214    where
215        S: serde::Serializer,
216    {
217        let mut state = serializer.serialize_struct("ModelConfigEntry", 18)?;
218        state.serialize_field("model", &self.model)?;
219        if let Some(value) = &self.mmproj {
220            state.serialize_field("mmproj", value)?;
221        }
222        if let Some(value) = &self.ctx_size {
223            state.serialize_field("ctx_size", value)?;
224        }
225        if self.gpu_id_from_legacy_shim
226            && let Some(value) = &self.gpu_id
227        {
228            state.serialize_field("gpu_id", value)?;
229        }
230        if let Some(value) = &self.parallel {
231            state.serialize_field("parallel", value)?;
232        }
233        if let Some(value) = &self.cache_type_k {
234            state.serialize_field("cache_type_k", value)?;
235        }
236        if let Some(value) = &self.cache_type_v {
237            state.serialize_field("cache_type_v", value)?;
238        }
239        if let Some(value) = &self.batch {
240            state.serialize_field("batch", value)?;
241        }
242        if let Some(value) = &self.ubatch {
243            state.serialize_field("ubatch", value)?;
244        }
245        if let Some(value) = &self.flash_attention {
246            state.serialize_field("flash_attention", value)?;
247        }
248        if let Some(value) = &self.model_fit {
249            state.serialize_field("model_fit", value)?;
250        }
251        if let Some(value) = &self.hardware {
252            state.serialize_field("hardware", value)?;
253        }
254        if let Some(value) = &self.throughput {
255            state.serialize_field("throughput", value)?;
256        }
257        if let Some(value) = &self.skippy {
258            state.serialize_field("skippy", value)?;
259        }
260        if let Some(value) = &self.speculative {
261            state.serialize_field("speculative", value)?;
262        }
263        if let Some(value) = &self.request_defaults {
264            state.serialize_field("request_defaults", value)?;
265        }
266        if let Some(value) = &self.multimodal {
267            state.serialize_field("multimodal", value)?;
268        }
269        if let Some(value) = &self.advanced {
270            state.serialize_field("advanced", value)?;
271        }
272        state.end()
273    }
274}
275
276impl ModelConfigEntry {
277    /// Compute a derived profile hash from the runtime-shaping fields of this entry.
278    ///
279    /// The profile is derived from the fields that materially affect runtime
280    /// behavior: ModelFitConfig (ctx_size, batch, ubatch, cache_type_k,
281    /// cache_type_v, flash_attention), HardwareConfig (model_runtime, device,
282    /// gpu_layers, tensor_split, split_mode, main_gpu, cpu_moe, n_cpu_moe,
283    /// fit_target_mib, mmap, mlock), and ThroughputConfig (parallel,
284    /// continuous_batching, threads, threads_batch).
285    ///
286    /// Returns an 8-hex-character string (e.g. "a3f2b9c1"), or empty string
287    /// if all profile-input fields are at their defaults.
288    /// Derive a stable profile string from the runtime-shaping config fields.
289    ///
290    /// Returns an 8-hex-char hash when any profile-input field is set,
291    /// or an empty string (profile = default) when all inputs are at defaults.
292    pub fn derived_profile(&self) -> String {
293        let mut buf = Vec::new();
294        Self::write_effective_fit_profile(&mut buf, self);
295        Self::write_effective_hw_profile(&mut buf, self);
296        Self::write_effective_tp_profile(&mut buf, self);
297
298        if buf.is_empty() {
299            return String::new();
300        }
301
302        use std::hash::{Hash, Hasher};
303        let mut hasher = std::collections::hash_map::DefaultHasher::new();
304        buf.hash(&mut hasher);
305        let hash = hasher.finish();
306        format!("{:08x}", hash & 0xFFFFFFFF)
307    }
308
309    fn write_effective_fit_profile(buf: &mut Vec<u8>, entry: &ModelConfigEntry) {
310        use std::io::Write;
311        macro_rules! wo {
312            ($key:literal, $val:expr) => {
313                if let Some(ref v) = $val {
314                    let _ = write!(buf, concat!($key, "={:?}\0"), v);
315                }
316            };
317        }
318        // Effective fit fields: sub-config (set by ConfigEditor) preferred,
319        // top-level (set by direct Rust construction) as fallback.
320        let fit = entry.model_fit.as_ref();
321        wo!("ctx_size", fit.and_then(|f| f.ctx_size).or(entry.ctx_size));
322        wo!("batch", fit.and_then(|f| f.batch).or(entry.batch));
323        wo!("ubatch", fit.and_then(|f| f.ubatch).or(entry.ubatch));
324        wo!(
325            "cache_type_k",
326            fit.and_then(|f| f.cache_type_k.as_ref())
327                .or(entry.cache_type_k.as_ref())
328        );
329        wo!(
330            "cache_type_v",
331            fit.and_then(|f| f.cache_type_v.as_ref())
332                .or(entry.cache_type_v.as_ref())
333        );
334        wo!(
335            "flash_attention",
336            fit.and_then(|f| f.flash_attention)
337                .or(entry.flash_attention)
338        );
339    }
340
341    fn write_effective_hw_profile(buf: &mut Vec<u8>, entry: &ModelConfigEntry) {
342        use std::io::Write;
343        macro_rules! wo {
344            ($key:literal, $val:expr) => {
345                if let Some(ref v) = $val {
346                    let _ = write!(buf, concat!($key, "={:?}\0"), v);
347                }
348            };
349        }
350        let hw = entry.hardware.as_ref();
351        wo!(
352            "gpu_id",
353            hw.and_then(|h| h.device.as_ref()).or(entry.gpu_id.as_ref())
354        );
355        if let Some(hw) = hw {
356            wo!("model_runtime", hw.model_runtime);
357            wo!("gpu_layers", hw.gpu_layers);
358            wo!("tensor_split", hw.tensor_split);
359            wo!("split_mode", hw.split_mode);
360            wo!("main_gpu", hw.main_gpu);
361            wo!("cpu_moe", hw.cpu_moe);
362            wo!("n_cpu_moe", hw.n_cpu_moe);
363            wo!("fit_target_mib", hw.fit_target_mib);
364            wo!("mmap", hw.mmap);
365            wo!("mlock", hw.mlock);
366        }
367    }
368
369    fn write_effective_tp_profile(buf: &mut Vec<u8>, entry: &ModelConfigEntry) {
370        use std::io::Write;
371        macro_rules! wo {
372            ($key:literal, $val:expr) => {
373                if let Some(ref v) = $val {
374                    let _ = write!(buf, concat!($key, "={:?}\0"), v);
375                }
376            };
377        }
378        let tp = entry.throughput.as_ref();
379        wo!("parallel", tp.and_then(|t| t.parallel).or(entry.parallel));
380        if let Some(tp) = tp {
381            wo!("continuous_batching", tp.continuous_batching);
382            wo!("threads", tp.threads);
383            wo!("threads_batch", tp.threads_batch);
384        }
385    }
386}
387
388#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)]
389#[serde(deny_unknown_fields)]
390pub struct ModelFitConfig {
391    #[serde(default)]
392    pub ctx_size: Option<u32>,
393    #[serde(default)]
394    pub batch: Option<u32>,
395    #[serde(default)]
396    pub ubatch: Option<u32>,
397    #[serde(default)]
398    pub cache_type_k: Option<String>,
399    #[serde(default)]
400    pub cache_type_v: Option<String>,
401    #[serde(default)]
402    pub kv_cache_policy: Option<String>,
403    #[serde(default)]
404    pub kv_offload: Option<BoolOrAuto>,
405    #[serde(default)]
406    pub kv_unified: Option<BoolOrAuto>,
407    #[serde(default)]
408    pub cache_ram_mib: Option<u64>,
409    #[serde(default)]
410    pub cache_idle_slots: Option<u32>,
411    #[serde(default)]
412    pub prompt_cache: Option<BoolOrAuto>,
413    #[serde(default)]
414    pub prefix_cache: Option<PrefixCacheConfig>,
415    #[serde(default)]
416    pub keep_tokens: Option<u32>,
417    #[serde(default)]
418    pub context_shift: Option<BoolOrAuto>,
419    #[serde(default)]
420    pub swa_full: Option<bool>,
421    #[serde(default)]
422    pub checkpoint_interval: Option<u32>,
423    #[serde(default)]
424    pub checkpoint_count: Option<u32>,
425    #[serde(default)]
426    pub lookup_cache_static: Option<String>,
427    #[serde(default)]
428    pub lookup_cache_dynamic: Option<String>,
429    #[serde(default)]
430    pub flash_attention: Option<FlashAttentionType>,
431}
432
433#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
434#[serde(deny_unknown_fields)]
435pub struct PrefixCacheConfig {
436    #[serde(default)]
437    pub enabled: Option<bool>,
438    #[serde(default)]
439    pub max_entries: Option<u32>,
440    #[serde(default)]
441    pub max_bytes: Option<u64>,
442    #[serde(default)]
443    pub min_tokens: Option<u32>,
444    #[serde(default)]
445    pub shared_stride_tokens: Option<u32>,
446    #[serde(default)]
447    pub shared_record_limit: Option<u32>,
448    #[serde(default)]
449    pub payload_mode: Option<String>,
450}
451
452#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)]
453#[serde(deny_unknown_fields)]
454pub struct HardwareConfig {
455    #[serde(default)]
456    pub model_runtime: Option<ModelRuntimeKind>,
457    #[serde(default)]
458    pub device: Option<String>,
459    #[serde(default)]
460    pub gpu_layers: Option<IntegerOrString>,
461    #[serde(default)]
462    pub stage_layer_start: Option<u32>,
463    #[serde(default)]
464    pub stage_layer_end: Option<u32>,
465    #[serde(default)]
466    pub placement: Option<String>,
467    #[serde(default)]
468    pub tensor_split: Option<TensorSplitConfig>,
469    #[serde(default)]
470    pub split_mode: Option<String>,
471    #[serde(default)]
472    pub main_gpu: Option<u32>,
473    #[serde(default)]
474    pub cpu_moe: Option<BoolOrAuto>,
475    #[serde(default)]
476    pub n_cpu_moe: Option<u32>,
477    #[serde(default)]
478    pub rpc_backend: Option<toml::Value>,
479    #[serde(default)]
480    pub fit_target_mib: Option<u64>,
481    #[serde(default)]
482    pub safety_margin_gb: Option<f64>,
483    #[serde(default)]
484    pub fit_context: Option<BoolOrAuto>,
485    #[serde(default)]
486    pub model_path: Option<String>,
487    #[serde(default)]
488    pub hf_repo: Option<String>,
489    #[serde(default)]
490    pub hf_file: Option<String>,
491    #[serde(default)]
492    pub mmproj: Option<String>,
493    #[serde(default)]
494    pub mmproj_offload: Option<BoolOrAuto>,
495    #[serde(default)]
496    pub lora_adapters: Vec<String>,
497    #[serde(default)]
498    pub control_vectors: Vec<String>,
499    #[serde(default)]
500    pub check_tensors: Option<bool>,
501    #[serde(default)]
502    pub mmap: Option<BoolOrAuto>,
503    #[serde(default)]
504    pub mlock: Option<bool>,
505    #[serde(default)]
506    pub direct_io: Option<bool>,
507    #[serde(default)]
508    pub repack: Option<bool>,
509    #[serde(default)]
510    pub op_offload: Option<bool>,
511    #[serde(default)]
512    pub no_host_buffer: Option<bool>,
513    #[serde(default)]
514    pub warmup: Option<BoolOrAuto>,
515}
516
517#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)]
518#[serde(deny_unknown_fields)]
519pub struct ThroughputConfig {
520    #[serde(default)]
521    pub parallel: Option<usize>,
522    #[serde(default)]
523    pub continuous_batching: Option<BoolOrAuto>,
524    #[serde(default)]
525    pub threads: Option<usize>,
526    #[serde(default)]
527    pub threads_batch: Option<usize>,
528    #[serde(default)]
529    pub threads_http: Option<usize>,
530    #[serde(default)]
531    pub priority: Option<IntegerOrString>,
532    #[serde(default)]
533    pub poll: Option<BoolOrString>,
534    #[serde(default)]
535    pub cpu_affinity: Option<StringOrStringList>,
536    #[serde(default)]
537    pub numa: Option<String>,
538    #[serde(default)]
539    pub slot_prompt_similarity: Option<f64>,
540    #[serde(default)]
541    pub sleep_idle_seconds: Option<u64>,
542    #[serde(default)]
543    pub tuning_profile: Option<String>,
544}
545
546#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)]
547#[serde(deny_unknown_fields)]
548pub struct SkippyConfig {
549    #[serde(default)]
550    pub stage_model_path: Option<String>,
551    #[serde(default)]
552    pub stage_role: Option<String>,
553    #[serde(default)]
554    pub stage_topology: Option<String>,
555    #[serde(default)]
556    pub activation_wire_dtype: Option<String>,
557    #[serde(default)]
558    pub binary_stage_transport: Option<String>,
559    #[serde(default)]
560    pub openai_frontend_mode: Option<toml::Value>,
561    #[serde(default)]
562    pub lifecycle_startup_timeout_ms: Option<u64>,
563    #[serde(default)]
564    pub lifecycle_readiness_interval_ms: Option<u64>,
565    #[serde(default)]
566    pub lifecycle_health_interval_ms: Option<u64>,
567    #[serde(default)]
568    pub prefill_chunking: Option<String>,
569    #[serde(default)]
570    pub prefill_chunk_size: Option<u32>,
571    #[serde(default)]
572    pub prefill_chunk_schedule: Option<String>,
573}
574
575#[derive(Clone, Debug, Default, PartialEq)]
576pub struct SpeculativeConfig {
577    pub strategy: Option<String>,
578    pub mode: Option<String>,
579    pub draft_model: Option<String>,
580    pub draft_hf_repo: Option<String>,
581    pub draft_hf_file: Option<String>,
582    pub draft_selection_policy: Option<String>,
583    pub pairing_fault: Option<String>,
584    pub draft_max_tokens: Option<u32>,
585    pub draft_min_tokens: Option<u32>,
586    pub draft_acceptance_threshold: Option<f64>,
587    pub draft_split_probability: Option<f64>,
588    pub draft_gpu_layers: Option<i32>,
589    pub draft_device: Option<String>,
590    pub draft_threads: Option<usize>,
591    pub draft_cache_type_k: Option<String>,
592    pub draft_cache_type_v: Option<String>,
593    pub ngram_min: Option<u32>,
594    pub ngram_max: Option<u32>,
595    pub ngram_max_proposal_tokens: Option<u32>,
596    pub ngram_proposer: Option<String>,
597    pub extension_max_tokens: Option<u32>,
598    pub native_mtp_reject_cooldown_tokens: Option<u32>,
599    pub native_mtp_suppress_cooldown_drafts: Option<bool>,
600    pub native_mtp_suppress_cooldown_draft_limit: Option<u32>,
601    pub verify_window_min_tokens: Option<u32>,
602    pub verify_window_max_tokens: Option<u32>,
603    pub verify_window_pipeline_depth: Option<u32>,
604    pub spec_default: Option<BoolOrAuto>,
605    pub(crate) legacy_draft_model_path_used: bool,
606}
607
608impl SpeculativeConfig {
609    /// Resolves the three supported policy layers without discarding fields
610    /// that are not overridden by a more specific layer.
611    pub fn with_precedence(
612        overrides: Option<&Self>,
613        model: Option<&Self>,
614        defaults: Option<&Self>,
615    ) -> Self {
616        macro_rules! pick {
617            ($field:ident) => {
618                overrides
619                    .and_then(|config| config.$field.clone())
620                    .or_else(|| model.and_then(|config| config.$field.clone()))
621                    .or_else(|| defaults.and_then(|config| config.$field.clone()))
622            };
623        }
624
625        Self {
626            strategy: pick!(strategy),
627            mode: pick!(mode),
628            draft_model: pick!(draft_model),
629            draft_hf_repo: pick!(draft_hf_repo),
630            draft_hf_file: pick!(draft_hf_file),
631            draft_selection_policy: pick!(draft_selection_policy),
632            pairing_fault: pick!(pairing_fault),
633            draft_max_tokens: pick!(draft_max_tokens),
634            draft_min_tokens: pick!(draft_min_tokens),
635            draft_acceptance_threshold: pick!(draft_acceptance_threshold),
636            draft_split_probability: pick!(draft_split_probability),
637            draft_gpu_layers: pick!(draft_gpu_layers),
638            draft_device: pick!(draft_device),
639            draft_threads: pick!(draft_threads),
640            draft_cache_type_k: pick!(draft_cache_type_k),
641            draft_cache_type_v: pick!(draft_cache_type_v),
642            ngram_min: pick!(ngram_min),
643            ngram_max: pick!(ngram_max),
644            ngram_max_proposal_tokens: pick!(ngram_max_proposal_tokens),
645            ngram_proposer: pick!(ngram_proposer),
646            extension_max_tokens: pick!(extension_max_tokens),
647            native_mtp_reject_cooldown_tokens: pick!(native_mtp_reject_cooldown_tokens),
648            native_mtp_suppress_cooldown_drafts: pick!(native_mtp_suppress_cooldown_drafts),
649            native_mtp_suppress_cooldown_draft_limit: pick!(
650                native_mtp_suppress_cooldown_draft_limit
651            ),
652            verify_window_min_tokens: pick!(verify_window_min_tokens),
653            verify_window_max_tokens: pick!(verify_window_max_tokens),
654            verify_window_pipeline_depth: pick!(verify_window_pipeline_depth),
655            spec_default: pick!(spec_default),
656            legacy_draft_model_path_used: overrides
657                .filter(|config| config.draft_model.is_some())
658                .or_else(|| model.filter(|config| config.draft_model.is_some()))
659                .or_else(|| defaults.filter(|config| config.draft_model.is_some()))
660                .is_some_and(|config| config.legacy_draft_model_path_used),
661        }
662    }
663}
664
665/// Raw deserialization helper that accepts both `draft_model` and the legacy
666/// `draft_model_path` key. The public `SpeculativeConfig` is constructed from
667/// this after detecting which key was used.
668#[derive(Deserialize)]
669#[serde(deny_unknown_fields)]
670struct SpeculativeConfigRaw {
671    #[serde(default)]
672    strategy: Option<String>,
673    #[serde(default)]
674    mode: Option<String>,
675    #[serde(default)]
676    draft_model: Option<String>,
677    #[serde(default)]
678    draft_model_path: Option<String>,
679    #[serde(default)]
680    draft_hf_repo: Option<String>,
681    #[serde(default)]
682    draft_hf_file: Option<String>,
683    #[serde(default)]
684    draft_selection_policy: Option<String>,
685    #[serde(default)]
686    pairing_fault: Option<String>,
687    #[serde(default)]
688    draft_max_tokens: Option<u32>,
689    #[serde(default)]
690    draft_min_tokens: Option<u32>,
691    #[serde(default)]
692    draft_acceptance_threshold: Option<f64>,
693    #[serde(default)]
694    draft_split_probability: Option<f64>,
695    #[serde(default)]
696    draft_gpu_layers: Option<i32>,
697    #[serde(default)]
698    draft_device: Option<String>,
699    #[serde(default)]
700    draft_threads: Option<usize>,
701    #[serde(default)]
702    draft_cache_type_k: Option<String>,
703    #[serde(default)]
704    draft_cache_type_v: Option<String>,
705    #[serde(default)]
706    ngram_min: Option<u32>,
707    #[serde(default)]
708    ngram_max: Option<u32>,
709    #[serde(default)]
710    ngram_max_proposal_tokens: Option<u32>,
711    #[serde(default)]
712    ngram_proposer: Option<String>,
713    #[serde(default)]
714    extension_max_tokens: Option<u32>,
715    #[serde(default)]
716    native_mtp_reject_cooldown_tokens: Option<u32>,
717    #[serde(default)]
718    native_mtp_suppress_cooldown_drafts: Option<bool>,
719    #[serde(default)]
720    native_mtp_suppress_cooldown_draft_limit: Option<u32>,
721    #[serde(default)]
722    verify_window_min_tokens: Option<u32>,
723    #[serde(default)]
724    verify_window_max_tokens: Option<u32>,
725    #[serde(default)]
726    verify_window_pipeline_depth: Option<u32>,
727    #[serde(default)]
728    spec_default: Option<BoolOrAuto>,
729}
730
731impl<'de> Deserialize<'de> for SpeculativeConfig {
732    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
733    where
734        D: serde::Deserializer<'de>,
735    {
736        let raw = SpeculativeConfigRaw::deserialize(deserializer)?;
737        let legacy_used = raw.draft_model_path.is_some();
738        if raw.draft_model.is_some() && raw.draft_model_path.is_some() {
739            return Err(serde::de::Error::custom(
740                "speculative config cannot set both `draft_model` and the legacy `draft_model_path`; \
741                 use `draft_model` only",
742            ));
743        }
744        Ok(SpeculativeConfig {
745            strategy: raw.strategy,
746            mode: raw.mode,
747            draft_model: raw.draft_model.or(raw.draft_model_path),
748            draft_hf_repo: raw.draft_hf_repo,
749            draft_hf_file: raw.draft_hf_file,
750            draft_selection_policy: raw.draft_selection_policy,
751            pairing_fault: raw.pairing_fault,
752            draft_max_tokens: raw.draft_max_tokens,
753            draft_min_tokens: raw.draft_min_tokens,
754            draft_acceptance_threshold: raw.draft_acceptance_threshold,
755            draft_split_probability: raw.draft_split_probability,
756            draft_gpu_layers: raw.draft_gpu_layers,
757            draft_device: raw.draft_device,
758            draft_threads: raw.draft_threads,
759            draft_cache_type_k: raw.draft_cache_type_k,
760            draft_cache_type_v: raw.draft_cache_type_v,
761            ngram_min: raw.ngram_min,
762            ngram_max: raw.ngram_max,
763            ngram_max_proposal_tokens: raw.ngram_max_proposal_tokens,
764            ngram_proposer: raw.ngram_proposer,
765            extension_max_tokens: raw.extension_max_tokens,
766            native_mtp_reject_cooldown_tokens: raw.native_mtp_reject_cooldown_tokens,
767            native_mtp_suppress_cooldown_drafts: raw.native_mtp_suppress_cooldown_drafts,
768            native_mtp_suppress_cooldown_draft_limit: raw.native_mtp_suppress_cooldown_draft_limit,
769            verify_window_min_tokens: raw.verify_window_min_tokens,
770            verify_window_max_tokens: raw.verify_window_max_tokens,
771            verify_window_pipeline_depth: raw.verify_window_pipeline_depth,
772            spec_default: raw.spec_default,
773            legacy_draft_model_path_used: legacy_used,
774        })
775    }
776}
777
778impl Serialize for SpeculativeConfig {
779    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
780    where
781        S: serde::Serializer,
782    {
783        use serde::ser::SerializeMap;
784
785        let mut map = serializer.serialize_map(Some(32))?;
786        map.serialize_entry("strategy", &self.strategy)?;
787        map.serialize_entry("mode", &self.mode)?;
788        if self.legacy_draft_model_path_used {
789            if let Some(ref v) = self.draft_model {
790                map.serialize_entry("draft_model_path", v)?;
791            }
792        } else if let Some(ref v) = self.draft_model {
793            map.serialize_entry("draft_model", v)?;
794        }
795        map.serialize_entry("draft_hf_repo", &self.draft_hf_repo)?;
796        map.serialize_entry("draft_hf_file", &self.draft_hf_file)?;
797        map.serialize_entry("draft_selection_policy", &self.draft_selection_policy)?;
798        map.serialize_entry("pairing_fault", &self.pairing_fault)?;
799        map.serialize_entry("draft_max_tokens", &self.draft_max_tokens)?;
800        map.serialize_entry("draft_min_tokens", &self.draft_min_tokens)?;
801        map.serialize_entry(
802            "draft_acceptance_threshold",
803            &self.draft_acceptance_threshold,
804        )?;
805        map.serialize_entry("draft_split_probability", &self.draft_split_probability)?;
806        map.serialize_entry("draft_gpu_layers", &self.draft_gpu_layers)?;
807        map.serialize_entry("draft_device", &self.draft_device)?;
808        map.serialize_entry("draft_threads", &self.draft_threads)?;
809        map.serialize_entry("draft_cache_type_k", &self.draft_cache_type_k)?;
810        map.serialize_entry("draft_cache_type_v", &self.draft_cache_type_v)?;
811        map.serialize_entry("ngram_min", &self.ngram_min)?;
812        map.serialize_entry("ngram_max", &self.ngram_max)?;
813        map.serialize_entry("ngram_max_proposal_tokens", &self.ngram_max_proposal_tokens)?;
814        map.serialize_entry("ngram_proposer", &self.ngram_proposer)?;
815        map.serialize_entry("extension_max_tokens", &self.extension_max_tokens)?;
816        map.serialize_entry(
817            "native_mtp_reject_cooldown_tokens",
818            &self.native_mtp_reject_cooldown_tokens,
819        )?;
820        map.serialize_entry(
821            "native_mtp_suppress_cooldown_drafts",
822            &self.native_mtp_suppress_cooldown_drafts,
823        )?;
824        map.serialize_entry(
825            "native_mtp_suppress_cooldown_draft_limit",
826            &self.native_mtp_suppress_cooldown_draft_limit,
827        )?;
828        map.serialize_entry("verify_window_min_tokens", &self.verify_window_min_tokens)?;
829        map.serialize_entry("verify_window_max_tokens", &self.verify_window_max_tokens)?;
830        map.serialize_entry(
831            "verify_window_pipeline_depth",
832            &self.verify_window_pipeline_depth,
833        )?;
834        map.serialize_entry("spec_default", &self.spec_default)?;
835        map.end()
836    }
837}
838
839#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)]
840#[serde(deny_unknown_fields)]
841pub struct RequestDefaultsConfig {
842    #[serde(default)]
843    pub max_tokens: Option<u32>,
844    #[serde(default)]
845    pub stop: Option<StringOrStringList>,
846    #[serde(default)]
847    pub temperature: Option<f64>,
848    #[serde(default)]
849    pub top_p: Option<f64>,
850    #[serde(default)]
851    pub top_k: Option<i64>,
852    #[serde(default)]
853    pub min_p: Option<f64>,
854    #[serde(default)]
855    pub typical_p: Option<f64>,
856    #[serde(default)]
857    pub top_nsigma: Option<f64>,
858    #[serde(default)]
859    pub dynatemp_range: Option<f64>,
860    #[serde(default)]
861    pub dynatemp_exponent: Option<f64>,
862    #[serde(default)]
863    pub repeat_penalty: Option<f64>,
864    #[serde(default)]
865    pub repeat_last_n: Option<i64>,
866    #[serde(default)]
867    pub presence_penalty: Option<f64>,
868    #[serde(default)]
869    pub frequency_penalty: Option<f64>,
870    #[serde(default)]
871    pub dry: Option<ReservedObjectConfig>,
872    #[serde(default)]
873    pub xtc: Option<ReservedObjectConfig>,
874    #[serde(default)]
875    pub adaptive: Option<ReservedObjectConfig>,
876    #[serde(default)]
877    pub mirostat_mode: Option<IntegerOrString>,
878    #[serde(default)]
879    pub mirostat_entropy: Option<f64>,
880    #[serde(default)]
881    pub mirostat_learning_rate: Option<f64>,
882    #[serde(default)]
883    pub samplers: Option<Vec<String>>,
884    #[serde(default)]
885    pub sampler_sequence: Option<String>,
886    #[serde(default)]
887    pub seed: Option<i64>,
888    #[serde(default)]
889    pub logit_bias: Option<toml::Value>,
890    #[serde(default)]
891    pub ignore_eos: Option<bool>,
892    #[serde(default)]
893    pub backend_sampling: Option<toml::Value>,
894    #[serde(default)]
895    pub reasoning_format: Option<String>,
896    #[serde(default)]
897    pub reasoning_enabled: Option<ReasoningEnabled>,
898    #[serde(default)]
899    pub reasoning_budget: Option<ReasoningBudget>,
900    #[serde(default)]
901    pub chat_template: Option<String>,
902    #[serde(default)]
903    pub chat_template_file: Option<String>,
904    #[serde(default)]
905    pub jinja: Option<bool>,
906    #[serde(default)]
907    pub chat_template_kwargs: Option<toml::Value>,
908    #[serde(default)]
909    pub skip_chat_parsing: Option<bool>,
910    #[serde(default)]
911    pub prefill_assistant: Option<toml::Value>,
912    #[serde(default)]
913    pub system_prompt: Option<String>,
914    #[serde(default)]
915    pub grammar: Option<toml::Value>,
916    #[serde(default)]
917    pub json_schema: Option<toml::Value>,
918    #[serde(default)]
919    pub logprobs: Option<toml::Value>,
920}
921
922#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)]
923#[serde(deny_unknown_fields)]
924pub struct MultimodalConfig {
925    #[serde(default)]
926    pub mmproj: Option<String>,
927    #[serde(default)]
928    pub mmproj_url: Option<String>,
929    #[serde(default)]
930    pub mmproj_offload: Option<BoolOrAuto>,
931    #[serde(default)]
932    pub image_min_tokens: Option<u32>,
933    #[serde(default)]
934    pub image_max_tokens: Option<u32>,
935    #[serde(default)]
936    pub embeddings: Option<toml::Value>,
937    #[serde(default)]
938    pub reranking: Option<toml::Value>,
939    #[serde(default)]
940    pub pooling: Option<toml::Value>,
941    #[serde(default)]
942    pub vocoder: Option<toml::Value>,
943}
944
945#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
946#[serde(deny_unknown_fields)]
947pub struct AdvancedConfig {
948    #[serde(default)]
949    pub server: Option<AdvancedServerConfig>,
950}
951
952#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
953#[serde(deny_unknown_fields)]
954pub struct AdvancedServerConfig {
955    #[serde(default)]
956    pub host: Option<String>,
957    #[serde(default)]
958    pub port: Option<u16>,
959    #[serde(default)]
960    pub reuse_port: Option<bool>,
961    #[serde(default)]
962    pub timeout: Option<u64>,
963    #[serde(default)]
964    pub metrics: Option<bool>,
965    #[serde(default)]
966    pub slots: Option<bool>,
967    #[serde(default)]
968    pub props: Option<bool>,
969    #[serde(default)]
970    pub alias: Option<String>,
971    #[serde(default)]
972    pub api_prefix: Option<String>,
973}
974
975#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
976#[serde(untagged)]
977pub enum BoolOrAuto {
978    Bool(bool),
979    String(String),
980}
981
982#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
983#[serde(untagged)]
984pub enum BoolOrString {
985    Bool(bool),
986    String(String),
987}
988
989#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
990#[serde(untagged)]
991pub enum IntegerOrString {
992    Integer(i64),
993    String(String),
994}
995
996#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
997#[serde(untagged)]
998pub enum StringOrStringList {
999    String(String),
1000    List(Vec<String>),
1001}
1002
1003#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
1004#[serde(untagged)]
1005pub enum TensorSplitConfig {
1006    Ratios(Vec<f64>),
1007    String(String),
1008}
1009
1010#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
1011#[serde(untagged)]
1012pub enum ReasoningEnabled {
1013    Bool(bool),
1014    String(String),
1015}
1016
1017#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
1018#[serde(untagged)]
1019pub enum ReasoningBudget {
1020    Integer(u32),
1021    String(String),
1022}
1023
1024#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
1025#[serde(deny_unknown_fields)]
1026pub struct ReservedObjectConfig {}
1027
1028#[derive(Clone, Debug, Default, Deserialize)]
1029struct RawMeshConfig {
1030    #[serde(default)]
1031    version: Option<u32>,
1032    #[serde(default)]
1033    gpu: GpuConfig,
1034    #[serde(default)]
1035    mesh_requirements: MeshRequirementsConfig,
1036    #[serde(default)]
1037    owner_control: OwnerControlConfig,
1038    #[serde(default)]
1039    telemetry: TelemetryConfig,
1040    #[serde(default)]
1041    defaults: Option<ModelConfigDefaults>,
1042    #[serde(default)]
1043    runtime: RuntimeConfig,
1044    #[serde(default)]
1045    models: Vec<ModelConfigEntry>,
1046    #[serde(rename = "plugin", default)]
1047    plugins: Vec<PluginConfigEntry>,
1048    #[serde(flatten, default)]
1049    extra: BTreeMap<String, toml::Value>,
1050}
1051
1052#[derive(Clone, Debug, Default, Deserialize)]
1053struct RawModelConfigDefaults {
1054    #[serde(default)]
1055    model_fit: Option<ModelFitConfig>,
1056    #[serde(default)]
1057    hardware: Option<HardwareConfig>,
1058    #[serde(default)]
1059    throughput: Option<ThroughputConfig>,
1060    #[serde(default)]
1061    skippy: Option<SkippyConfig>,
1062    #[serde(default)]
1063    speculative: Option<SpeculativeConfig>,
1064    #[serde(default)]
1065    request_defaults: Option<RequestDefaultsConfig>,
1066    #[serde(default)]
1067    multimodal: Option<MultimodalConfig>,
1068    #[serde(default)]
1069    advanced: Option<AdvancedConfig>,
1070    #[serde(default)]
1071    mmproj: Option<String>,
1072    #[serde(default)]
1073    ctx_size: Option<u32>,
1074    #[serde(default)]
1075    gpu_id: Option<String>,
1076    #[serde(default)]
1077    parallel: Option<usize>,
1078    #[serde(default)]
1079    cache_type_k: Option<String>,
1080    #[serde(default)]
1081    cache_type_v: Option<String>,
1082    #[serde(default)]
1083    batch: Option<u32>,
1084    #[serde(default)]
1085    ubatch: Option<u32>,
1086    #[serde(default)]
1087    flash_attention: Option<FlashAttentionType>,
1088}
1089
1090#[derive(Clone, Debug, Default, Deserialize)]
1091struct RawModelConfigEntry {
1092    model: String,
1093    #[serde(default)]
1094    mmproj: Option<String>,
1095    #[serde(default)]
1096    ctx_size: Option<u32>,
1097    #[serde(default)]
1098    gpu_id: Option<String>,
1099    #[serde(default)]
1100    parallel: Option<usize>,
1101    #[serde(default)]
1102    cache_type_k: Option<String>,
1103    #[serde(default)]
1104    cache_type_v: Option<String>,
1105    #[serde(default)]
1106    batch: Option<u32>,
1107    #[serde(default)]
1108    ubatch: Option<u32>,
1109    #[serde(default)]
1110    flash_attention: Option<FlashAttentionType>,
1111    #[serde(default)]
1112    model_fit: Option<ModelFitConfig>,
1113    #[serde(default)]
1114    hardware: Option<HardwareConfig>,
1115    #[serde(default)]
1116    throughput: Option<ThroughputConfig>,
1117    #[serde(default)]
1118    skippy: Option<SkippyConfig>,
1119    #[serde(default)]
1120    speculative: Option<SpeculativeConfig>,
1121    #[serde(default)]
1122    request_defaults: Option<RequestDefaultsConfig>,
1123    #[serde(default)]
1124    multimodal: Option<MultimodalConfig>,
1125    #[serde(default)]
1126    advanced: Option<AdvancedConfig>,
1127}
1128
1129impl<'de> Deserialize<'de> for MeshConfig {
1130    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
1131    where
1132        D: serde::Deserializer<'de>,
1133    {
1134        let raw = RawMeshConfig::deserialize(deserializer)?;
1135        Ok(Self {
1136            version: raw.version,
1137            gpu: raw.gpu,
1138            mesh_requirements: raw.mesh_requirements,
1139            owner_control: raw.owner_control,
1140            telemetry: raw.telemetry,
1141            defaults: raw.defaults,
1142            runtime: raw.runtime,
1143            models: raw.models,
1144            plugins: raw.plugins,
1145            extra: raw.extra,
1146        })
1147    }
1148}
1149
1150impl<'de> Deserialize<'de> for ModelConfigDefaults {
1151    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
1152    where
1153        D: serde::Deserializer<'de>,
1154    {
1155        let raw = RawModelConfigDefaults::deserialize(deserializer)?;
1156        Ok(Self::from_raw(raw))
1157    }
1158}
1159
1160impl<'de> Deserialize<'de> for ModelConfigEntry {
1161    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
1162    where
1163        D: serde::Deserializer<'de>,
1164    {
1165        let raw = RawModelConfigEntry::deserialize(deserializer)?;
1166        Ok(Self::from_raw(raw))
1167    }
1168}
1169
1170impl ModelConfigDefaults {
1171    fn from_raw(raw: RawModelConfigDefaults) -> Self {
1172        let model_fit = merge_model_fit(
1173            raw.model_fit,
1174            raw.ctx_size,
1175            raw.cache_type_k,
1176            raw.cache_type_v,
1177            raw.batch,
1178            raw.ubatch,
1179            raw.flash_attention,
1180        );
1181        let hardware = merge_hardware(raw.hardware, raw.gpu_id, None, None);
1182        let throughput = merge_throughput(raw.throughput, raw.parallel);
1183        let multimodal = merge_multimodal(raw.multimodal, raw.mmproj);
1184        Self {
1185            model_fit,
1186            hardware,
1187            throughput,
1188            skippy: raw.skippy,
1189            speculative: raw.speculative,
1190            request_defaults: raw.request_defaults,
1191            multimodal,
1192            advanced: raw.advanced,
1193        }
1194    }
1195}
1196
1197impl ModelConfigEntry {
1198    fn from_raw(raw: RawModelConfigEntry) -> Self {
1199        let gpu_id_from_legacy_shim = raw.gpu_id.is_some();
1200        let model_fit = merge_model_fit(
1201            raw.model_fit,
1202            raw.ctx_size,
1203            raw.cache_type_k.clone(),
1204            raw.cache_type_v.clone(),
1205            raw.batch,
1206            raw.ubatch,
1207            raw.flash_attention,
1208        );
1209        let multimodal = merge_multimodal(raw.multimodal, raw.mmproj.clone());
1210        let hardware = merge_hardware(
1211            raw.hardware,
1212            raw.gpu_id.clone(),
1213            multimodal.as_ref().and_then(|m| m.mmproj.clone()),
1214            multimodal.as_ref().and_then(|m| m.mmproj_offload.clone()),
1215        );
1216        let throughput = merge_throughput(raw.throughput, raw.parallel);
1217
1218        Self {
1219            model: raw.model,
1220            mmproj: multimodal
1221                .as_ref()
1222                .and_then(|config| config.mmproj.clone())
1223                .or_else(|| hardware.as_ref().and_then(|config| config.mmproj.clone()))
1224                .or(raw.mmproj),
1225            ctx_size: model_fit.as_ref().and_then(|config| config.ctx_size),
1226            gpu_id: hardware
1227                .as_ref()
1228                .and_then(|config| config.device.clone())
1229                .or(raw.gpu_id),
1230            parallel: throughput.as_ref().and_then(|config| config.parallel),
1231            cache_type_k: model_fit
1232                .as_ref()
1233                .and_then(|config| config.cache_type_k.clone())
1234                .or(raw.cache_type_k),
1235            cache_type_v: model_fit
1236                .as_ref()
1237                .and_then(|config| config.cache_type_v.clone())
1238                .or(raw.cache_type_v),
1239            batch: model_fit.as_ref().and_then(|config| config.batch),
1240            ubatch: model_fit.as_ref().and_then(|config| config.ubatch),
1241            flash_attention: model_fit
1242                .as_ref()
1243                .and_then(|config| config.flash_attention)
1244                .or(raw.flash_attention),
1245            model_fit,
1246            hardware,
1247            throughput,
1248            skippy: raw.skippy,
1249            speculative: raw.speculative,
1250            request_defaults: raw.request_defaults,
1251            multimodal,
1252            advanced: raw.advanced,
1253            gpu_id_from_legacy_shim,
1254        }
1255    }
1256}
1257
1258pub(crate) fn merge_model_fit(
1259    current: Option<ModelFitConfig>,
1260    ctx_size: Option<u32>,
1261    cache_type_k: Option<String>,
1262    cache_type_v: Option<String>,
1263    batch: Option<u32>,
1264    ubatch: Option<u32>,
1265    flash_attention: Option<FlashAttentionType>,
1266) -> Option<ModelFitConfig> {
1267    let mut config = current.unwrap_or_default();
1268    config.ctx_size = config.ctx_size.or(ctx_size);
1269    config.cache_type_k = config.cache_type_k.or(cache_type_k);
1270    config.cache_type_v = config.cache_type_v.or(cache_type_v);
1271    config.batch = config.batch.or(batch);
1272    config.ubatch = config.ubatch.or(ubatch);
1273    config.flash_attention = config.flash_attention.or(flash_attention);
1274    if is_model_fit_empty(&config) {
1275        None
1276    } else {
1277        Some(config)
1278    }
1279}
1280
1281pub(crate) fn merge_hardware(
1282    current: Option<HardwareConfig>,
1283    gpu_id: Option<String>,
1284    mmproj: Option<String>,
1285    mmproj_offload: Option<BoolOrAuto>,
1286) -> Option<HardwareConfig> {
1287    let mut config = current.unwrap_or_default();
1288    config.device = config.device.or(gpu_id);
1289    config.mmproj = config.mmproj.or(mmproj);
1290    config.mmproj_offload = config.mmproj_offload.or(mmproj_offload);
1291    if is_hardware_empty(&config) {
1292        None
1293    } else {
1294        Some(config)
1295    }
1296}
1297
1298pub(crate) fn merge_throughput(
1299    current: Option<ThroughputConfig>,
1300    parallel: Option<usize>,
1301) -> Option<ThroughputConfig> {
1302    let mut config = current.unwrap_or_default();
1303    config.parallel = config.parallel.or(parallel);
1304    if is_throughput_empty(&config) {
1305        None
1306    } else {
1307        Some(config)
1308    }
1309}
1310
1311pub(crate) fn merge_multimodal(
1312    current: Option<MultimodalConfig>,
1313    mmproj: Option<String>,
1314) -> Option<MultimodalConfig> {
1315    let mut config = current.unwrap_or_default();
1316    config.mmproj = config.mmproj.or(mmproj);
1317    if is_multimodal_empty(&config) {
1318        None
1319    } else {
1320        Some(config)
1321    }
1322}
1323
1324fn is_model_fit_empty(config: &ModelFitConfig) -> bool {
1325    config == &ModelFitConfig::default()
1326}
1327
1328fn is_hardware_empty(config: &HardwareConfig) -> bool {
1329    config == &HardwareConfig::default()
1330}
1331
1332fn is_throughput_empty(config: &ThroughputConfig) -> bool {
1333    config == &ThroughputConfig::default()
1334}
1335
1336fn is_multimodal_empty(config: &MultimodalConfig) -> bool {
1337    config == &MultimodalConfig::default()
1338}
1339
1340#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
1341pub struct TelemetryConfig {
1342    #[serde(default)]
1343    pub enabled: Option<bool>,
1344    #[serde(default)]
1345    pub service_name: Option<String>,
1346    #[serde(default)]
1347    pub endpoint: Option<String>,
1348    #[serde(default)]
1349    pub headers: BTreeMap<String, String>,
1350    #[serde(default)]
1351    pub export_interval_secs: Option<u64>,
1352    #[serde(default)]
1353    pub queue_size: Option<usize>,
1354    #[serde(default)]
1355    pub prompt_shape_metrics: bool,
1356    #[serde(default)]
1357    pub metrics: TelemetryMetricsConfig,
1358}
1359
1360#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
1361pub struct TelemetryMetricsConfig {
1362    #[serde(default)]
1363    pub endpoint: Option<String>,
1364}
1365
1366#[derive(Clone, Debug, Deserialize, Serialize)]
1367pub struct PluginConfigEntry {
1368    pub name: String,
1369    #[serde(default)]
1370    pub enabled: Option<bool>,
1371    #[serde(default, skip_serializing_if = "Option::is_none")]
1372    pub web_ui_enabled: Option<bool>,
1373    #[serde(default)]
1374    pub command: Option<String>,
1375    #[serde(default)]
1376    pub args: Vec<String>,
1377    /// Optional URL passed to the plugin as `MESH_LLM_PLUGIN_URL`.
1378    #[serde(default)]
1379    pub url: Option<String>,
1380    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
1381    pub settings: BTreeMap<String, toml::Value>,
1382    #[serde(default, skip_serializing_if = "PluginStartupConfig::is_default")]
1383    pub startup: PluginStartupConfig,
1384}
1385
1386#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1387pub enum PluginWebUiPreference {
1388    None,
1389    Enabled,
1390    Disabled,
1391}
1392
1393impl PluginWebUiPreference {
1394    pub const fn resolve(web_ui_enabled: Option<bool>, declares_web_ui: bool) -> Self {
1395        match (declares_web_ui, web_ui_enabled) {
1396            (false, _) => Self::None,
1397            (true, Some(false)) => Self::Disabled,
1398            (true, Some(true) | None) => Self::Enabled,
1399        }
1400    }
1401}
1402
1403impl PluginConfigEntry {
1404    pub const fn web_ui_preference(&self, declares_web_ui: bool) -> PluginWebUiPreference {
1405        PluginWebUiPreference::resolve(self.web_ui_enabled, declares_web_ui)
1406    }
1407}
1408
1409#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
1410pub struct PluginStartupConfig {
1411    #[serde(default, skip_serializing_if = "Option::is_none")]
1412    pub connect_timeout_secs: Option<u64>,
1413    #[serde(default, skip_serializing_if = "Option::is_none")]
1414    pub init_timeout_secs: Option<u64>,
1415    #[serde(default)]
1416    pub optional: bool,
1417    #[serde(default)]
1418    pub lazy_start: bool,
1419}
1420
1421impl PluginStartupConfig {
1422    pub fn is_default(&self) -> bool {
1423        self == &Self::default()
1424    }
1425}
mesh_llm_config/model.rs

mesh_llm_config/
model.rs
