use crate::{
parse_bool_env_value, parse_path_env_value, parse_usize_env_value, AttentionExecutionPolicy,
DataType, Device, ModelId, ModelInfo, ObservabilityProfileDetail, ProfileEntrypoint,
RuntimeConfigSnapshot, SamplingParams, SamplingPresets, TokenId,
};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::{collections::HashMap, path::PathBuf, time::Duration};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum SequenceFitPolicy {
FullInputMustFit,
ImmediateOnly,
}
impl SequenceFitPolicy {
pub const fn as_runtime_value(self) -> &'static str {
match self {
Self::FullInputMustFit => "full-input-must-fit",
Self::ImmediateOnly => "immediate-only",
}
}
pub fn parse_runtime_value(raw: &str) -> std::result::Result<Self, String> {
match raw.trim().to_ascii_lowercase().replace('_', "-").as_str() {
"full-input-must-fit" => Ok(Self::FullInputMustFit),
"immediate-only" => Ok(Self::ImmediateOnly),
_ => Err(format!(
"expected full-input-must-fit or immediate-only; got {raw:?}"
)),
}
}
}
impl Default for SequenceFitPolicy {
fn default() -> Self {
Self::ImmediateOnly
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum VNextDiagnosticFault {
PrefillResourceAfterSubmitOnce,
}
impl VNextDiagnosticFault {
pub const fn as_runtime_value(self) -> &'static str {
match self {
Self::PrefillResourceAfterSubmitOnce => "prefill-resource-after-submit-once",
}
}
pub fn parse_runtime_value(raw: &str) -> std::result::Result<Self, String> {
match raw.trim().to_ascii_lowercase().replace('_', "-").as_str() {
"prefill-resource-after-submit-once" => Ok(Self::PrefillResourceAfterSubmitOnce),
_ => Err(format!(
"expected prefill-resource-after-submit-once; got {raw:?}"
)),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct VNextCheckpointCaptureConfig {
pub output_dir: PathBuf,
pub value_ids: Vec<String>,
pub maximum_prefill_waves: usize,
#[serde(default)]
pub maximum_decode_waves: usize,
#[serde(default)]
pub capture_product_output: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub teacher_forcing: Option<VNextTeacherForcingConfig>,
}
pub const MAX_VNEXT_TEACHER_FORCED_TOKENS: usize = 512;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct VNextTeacherForcingConfig {
token_ids: Vec<TokenId>,
}
impl VNextTeacherForcingConfig {
pub fn new(token_ids: Vec<TokenId>) -> std::result::Result<Self, String> {
let value = Self { token_ids };
value.validate()?;
Ok(value)
}
pub fn validate(&self) -> std::result::Result<(), String> {
if self.token_ids.is_empty() || self.token_ids.len() > MAX_VNEXT_TEACHER_FORCED_TOKENS {
return Err(format!(
"vNext checkpoint teacher forcing requires 1..={MAX_VNEXT_TEACHER_FORCED_TOKENS} tokens"
));
}
Ok(())
}
pub fn token_ids(&self) -> &[TokenId] {
&self.token_ids
}
pub fn token_count(&self) -> usize {
self.token_ids.len()
}
pub fn token_ids_sha256(&self) -> String {
let mut digest = Sha256::new();
for token in &self.token_ids {
digest.update(token.get().to_le_bytes());
}
format!("{:x}", digest.finalize())
}
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct RuntimeKnobs {
pub kv_capacity: Option<usize>,
pub max_model_len: Option<usize>,
pub chunked_prefill_size: Option<usize>,
pub batch_decode_prof: bool,
pub next_batch_prof: bool,
pub rbd_prof: bool,
#[serde(default)]
pub profile_jsonl: Option<PathBuf>,
pub scheduler_trace_jsonl: Option<PathBuf>,
pub legacy_scheduler_trace_jsonl: Option<PathBuf>,
pub profile_entrypoint: Option<ProfileEntrypoint>,
pub profile_detail: ObservabilityProfileDetail,
pub unified_post_prof: bool,
pub prefix_cache_enabled: bool,
pub recurrent_state_max_slots: Option<usize>,
pub attention_execution_policy: AttentionExecutionPolicy,
pub model_path: Option<String>,
pub spec_draft: Option<String>,
pub spec_n: Option<usize>,
pub dtype: Option<String>,
pub metal_dtype: Option<String>,
pub tp: Option<usize>,
#[serde(default)]
pub vnext_checkpoint_capture: Option<VNextCheckpointCaptureConfig>,
#[serde(default)]
pub vnext_diagnostic_fault: Option<VNextDiagnosticFault>,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct EngineConfig {
pub model: EngineModelConfig,
pub scheduler: SchedulerConfig,
pub sampling: SamplingConfig,
pub backend: BackendConfig,
pub kv_cache: KvCacheConfig,
pub memory: MemoryConfig,
pub batching: BatchConfig,
pub monitoring: MonitoringConfig,
#[serde(default)]
pub runtime: RuntimeKnobs,
}
impl EngineConfig {
pub fn apply_runtime_config_snapshot(
&mut self,
snapshot: &RuntimeConfigSnapshot,
) -> std::result::Result<(), String> {
self.scheduler.apply_runtime_config_snapshot(snapshot)?;
if let Some(value) = runtime_config_value(snapshot, "FERRUM_KV_MAX_BLOCKS") {
self.kv_cache.max_blocks =
parse_required_positive_usize("FERRUM_KV_MAX_BLOCKS", value)?;
}
if let Some(value) = runtime_config_value(snapshot, "FERRUM_MAX_BATCHED_TOKENS") {
self.batching.max_num_batched_tokens =
parse_required_positive_usize("FERRUM_MAX_BATCHED_TOKENS", value)?;
}
if let Some(value) = runtime_config_value(snapshot, "FERRUM_PAGED_MAX_SEQS") {
self.scheduler.max_running_requests =
parse_required_positive_usize("FERRUM_PAGED_MAX_SEQS", value)?;
}
if let Some(value) = runtime_config_value(snapshot, "FERRUM_RUNTIME_MEMORY_BUDGET_BYTES") {
self.memory.usable_capacity_bytes = Some(parse_required_positive_usize(
"FERRUM_RUNTIME_MEMORY_BUDGET_BYTES",
value,
)?);
}
if let Some(value) = runtime_config_value(snapshot, "FERRUM_BATCHED_GRAPH") {
self.backend.enable_cuda_graphs = parse_presence_bool(value)?;
}
if let Some(value) = runtime_config_value(snapshot, "FERRUM_REUSABLE_EXECUTION") {
self.backend.enable_reusable_execution = parse_presence_bool(value)?;
}
if let Some(value) =
runtime_config_value(snapshot, "FERRUM_REUSABLE_EXECUTION_EXACT_DECODE_WIDTHS")
{
let widths =
parse_positive_usize_list("FERRUM_REUSABLE_EXECUTION_EXACT_DECODE_WIDTHS", value)?;
if widths
.iter()
.any(|width| *width > MAXIMUM_REUSABLE_EXECUTION_STARTUP_CAPTURE_WIDTH)
{
return Err(format!(
"FERRUM_REUSABLE_EXECUTION_EXACT_DECODE_WIDTHS: startup capture widths must be within 1..={MAXIMUM_REUSABLE_EXECUTION_STARTUP_CAPTURE_WIDTH}"
));
}
self.backend.reusable_execution_capture.exact_decode_widths = Some(widths);
}
if let Some(value) = runtime_config_value(
snapshot,
"FERRUM_REUSABLE_EXECUTION_MAX_AUTOMATIC_EXACT_DECODE_WIDTH",
) {
let maximum = parse_required_positive_usize(
"FERRUM_REUSABLE_EXECUTION_MAX_AUTOMATIC_EXACT_DECODE_WIDTH",
value,
)?;
if maximum > MAXIMUM_REUSABLE_EXECUTION_STARTUP_CAPTURE_WIDTH {
return Err(format!(
"FERRUM_REUSABLE_EXECUTION_MAX_AUTOMATIC_EXACT_DECODE_WIDTH: must be within 1..={MAXIMUM_REUSABLE_EXECUTION_STARTUP_CAPTURE_WIDTH}"
));
}
self.backend
.reusable_execution_capture
.maximum_automatic_exact_decode_width = maximum;
}
if let Some(value) = runtime_config_value(snapshot, "FERRUM_KV_CAPACITY") {
self.runtime.kv_capacity =
Some(parse_required_positive_usize("FERRUM_KV_CAPACITY", value)?);
}
if let Some(value) = runtime_config_value(snapshot, "FERRUM_MAX_MODEL_LEN") {
self.runtime.max_model_len = Some(parse_required_positive_usize(
"FERRUM_MAX_MODEL_LEN",
value,
)?);
}
if let Some(value) = runtime_config_value(snapshot, "FERRUM_RECURRENT_STATE_MAX_SLOTS") {
self.runtime.recurrent_state_max_slots = Some(parse_required_positive_usize(
"FERRUM_RECURRENT_STATE_MAX_SLOTS",
value,
)?);
}
if let Some(value) = runtime_config_value(snapshot, "FERRUM_ATTENTION_POLICY") {
self.runtime.attention_execution_policy =
AttentionExecutionPolicy::parse_runtime_value(value)
.map_err(|reason| format!("FERRUM_ATTENTION_POLICY: {reason}"))?;
}
if let Some(value) = runtime_config_value(snapshot, "FERRUM_CHUNKED_PREFILL") {
self.runtime.chunked_prefill_size =
parse_usize_env_value(value).ok().filter(|&v| v > 0);
}
self.runtime.batch_decode_prof |=
runtime_config_value(snapshot, "FERRUM_BATCH_DECODE_PROF").is_some();
self.runtime.next_batch_prof |=
runtime_config_value(snapshot, "FERRUM_NEXT_BATCH_PROF").is_some();
self.runtime.rbd_prof |= runtime_config_value(snapshot, "FERRUM_RBD_PROF").is_some();
if let Some(value) = runtime_config_value(snapshot, "FERRUM_PROFILE_JSONL") {
self.runtime.profile_jsonl = Some(parse_path_env_value(value)?);
}
if let Some(value) = runtime_config_value(snapshot, "FERRUM_SCHEDULER_TRACE_JSONL") {
self.runtime.scheduler_trace_jsonl = Some(parse_path_env_value(value)?);
}
if let Some(value) = runtime_config_value(snapshot, "FERRUM_LEGACY_SCHEDULER_TRACE_JSONL") {
self.runtime.legacy_scheduler_trace_jsonl = Some(parse_path_env_value(value)?);
}
if let Some(value) = runtime_config_value(snapshot, "FERRUM_PROFILE_ENTRYPOINT") {
self.runtime.profile_entrypoint = Some(parse_profile_entrypoint(
"FERRUM_PROFILE_ENTRYPOINT",
value,
)?);
}
if let Some(value) = runtime_config_value(snapshot, "FERRUM_PROFILE_DETAIL") {
self.runtime.profile_detail =
ObservabilityProfileDetail::parse(value).ok_or_else(|| {
format!(
"FERRUM_PROFILE_DETAIL: expected one of off, basic, resource, latency, kernel, debug, replay, verify, full; got {value:?}"
)
})?;
}
if let Some(value) = runtime_config_value(snapshot, "FERRUM_VNEXT_DIAGNOSTIC_FAULT") {
self.runtime.vnext_diagnostic_fault = Some(
VNextDiagnosticFault::parse_runtime_value(value)
.map_err(|reason| format!("FERRUM_VNEXT_DIAGNOSTIC_FAULT: {reason}"))?,
);
}
self.runtime.unified_post_prof |=
runtime_config_value(snapshot, "FERRUM_UNIFIED_POST_PROF").is_some();
self.runtime.prefix_cache_enabled |=
runtime_config_value(snapshot, "FERRUM_WHOLE_PROMPT_PREFIX_CACHE")
.map(|v| v == "1")
.unwrap_or(false);
if let Some(value) = runtime_config_value(snapshot, "FERRUM_MODEL_PATH") {
self.runtime.model_path = Some(value.to_string());
}
if let Some(value) = runtime_config_value(snapshot, "FERRUM_SPEC_DRAFT") {
self.runtime.spec_draft = if value.is_empty() {
None
} else {
Some(value.to_string())
};
}
if let Some(value) = runtime_config_value(snapshot, "FERRUM_SPEC_N") {
self.runtime.spec_n = value.parse::<usize>().ok();
}
if let Some(value) = runtime_config_value(snapshot, "FERRUM_DTYPE") {
self.runtime.dtype = Some(value.to_string());
}
if let Some(value) = runtime_config_value(snapshot, "FERRUM_METAL_DTYPE") {
self.runtime.metal_dtype = Some(value.to_string());
}
if let Some(value) = runtime_config_value(snapshot, "FERRUM_TP") {
self.runtime.tp = value.parse::<usize>().ok();
}
crate::install_runtime_snapshot(snapshot.clone());
Ok(())
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct EngineModelConfig {
pub model_id: ModelId,
pub model_info: Option<ModelInfo>,
pub tokenizer: TokenizerConfig,
#[serde(default)]
pub source: Option<crate::ModelSource>,
}
impl Default for EngineModelConfig {
fn default() -> Self {
Self {
model_id: ModelId::new("default"),
model_info: None,
tokenizer: TokenizerConfig::default(),
source: None,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SchedulerConfig {
pub policy: SchedulingPolicy,
pub max_waiting_requests: usize,
pub max_running_requests: usize,
pub enable_preemption: bool,
pub enable_load_balancing: bool,
pub fair_share_weights: HashMap<String, f32>,
pub enable_sla_enforcement: bool,
#[serde(default = "default_prompt_token_estimate")]
pub prompt_token_estimate: bool,
#[serde(default)]
pub prefill_first_until_active: Option<usize>,
#[serde(default)]
pub prefill_step_chunk: Option<usize>,
#[serde(default)]
pub active_decode_prefill_chunk: Option<usize>,
#[serde(default)]
pub scheduler_none_prof: bool,
#[serde(default)]
pub sequence_fit_policy: SequenceFitPolicy,
}
impl Default for SchedulerConfig {
fn default() -> Self {
Self {
policy: SchedulingPolicy::Priority,
max_waiting_requests: 1000,
max_running_requests: 32,
enable_preemption: true,
enable_load_balancing: false,
fair_share_weights: HashMap::new(),
enable_sla_enforcement: false,
prompt_token_estimate: default_prompt_token_estimate(),
prefill_first_until_active: None,
prefill_step_chunk: None,
active_decode_prefill_chunk: None,
scheduler_none_prof: false,
sequence_fit_policy: SequenceFitPolicy::default(),
}
}
}
fn default_prompt_token_estimate() -> bool {
true
}
impl SchedulerConfig {
pub fn apply_runtime_config_snapshot(
&mut self,
snapshot: &RuntimeConfigSnapshot,
) -> std::result::Result<(), String> {
if let Some(value) = runtime_config_value(snapshot, "FERRUM_SCHED_PROMPT_TOKEN_ESTIMATE") {
self.prompt_token_estimate = parse_bool_env_value(value)
.map_err(|reason| format!("FERRUM_SCHED_PROMPT_TOKEN_ESTIMATE: {reason}"))?;
}
if let Some(value) =
runtime_config_value(snapshot, "FERRUM_SCHED_PREFILL_FIRST_UNTIL_ACTIVE")
{
self.prefill_first_until_active =
parse_optional_positive_usize("FERRUM_SCHED_PREFILL_FIRST_UNTIL_ACTIVE", value)?;
}
if let Some(value) = runtime_config_value(snapshot, "FERRUM_SCHED_PREFILL_STEP_CHUNK") {
self.prefill_step_chunk =
parse_optional_positive_usize("FERRUM_SCHED_PREFILL_STEP_CHUNK", value)?;
}
if let Some(value) = runtime_config_value(snapshot, "FERRUM_ACTIVE_DECODE_PREFILL_CHUNK") {
self.active_decode_prefill_chunk =
parse_optional_positive_usize("FERRUM_ACTIVE_DECODE_PREFILL_CHUNK", value)?;
}
if let Some(value) = runtime_config_value(snapshot, "FERRUM_SCHED_NONE_PROF") {
self.scheduler_none_prof = parse_presence_bool(value)
.map_err(|reason| format!("FERRUM_SCHED_NONE_PROF: {reason}"))?;
}
if let Some(value) = runtime_config_value(snapshot, "FERRUM_SEQUENCE_FIT_POLICY") {
self.sequence_fit_policy = SequenceFitPolicy::parse_runtime_value(value)
.map_err(|reason| format!("FERRUM_SEQUENCE_FIT_POLICY: {reason}"))?;
}
Ok(())
}
}
fn runtime_config_value<'a>(snapshot: &'a RuntimeConfigSnapshot, key: &str) -> Option<&'a str> {
snapshot
.entries
.iter()
.find(|entry| entry.key == key)
.map(|entry| entry.effective_value.as_str())
}
fn parse_optional_positive_usize(
key: &str,
value: &str,
) -> std::result::Result<Option<usize>, String> {
let parsed = parse_usize_env_value(value).map_err(|reason| format!("{key}: {reason}"))?;
Ok((parsed > 0).then_some(parsed))
}
fn parse_required_positive_usize(key: &str, value: &str) -> std::result::Result<usize, String> {
let parsed = parse_usize_env_value(value).map_err(|reason| format!("{key}: {reason}"))?;
if parsed == 0 {
Err(format!("{key}: must be greater than zero"))
} else {
Ok(parsed)
}
}
fn parse_positive_usize_list(key: &str, value: &str) -> std::result::Result<Vec<usize>, String> {
let values = value
.split(',')
.map(str::trim)
.map(|value| parse_required_positive_usize(key, value))
.collect::<std::result::Result<Vec<_>, _>>()?;
if values.is_empty() {
Err(format!("{key}: must contain at least one width"))
} else {
Ok(values)
}
}
fn parse_profile_entrypoint(
key: &str,
value: &str,
) -> std::result::Result<ProfileEntrypoint, String> {
ProfileEntrypoint::parse(value).ok_or_else(|| {
format!("{key}: expected one of run, serve, bench_serve, synthetic; got {value:?}")
})
}
fn parse_presence_bool(value: &str) -> std::result::Result<bool, String> {
if value.trim().is_empty() {
Ok(true)
} else {
parse_bool_env_value(value)
}
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
pub enum SchedulingPolicy {
FCFS,
Priority,
FairShare,
SJF,
RoundRobin,
ContinuousBatch,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct KvCacheConfig {
pub cache_type: KvCacheType,
#[serde(default)]
pub dtype: KvCacheDtype,
pub block_size: usize,
pub max_blocks: usize,
pub enable_compression: bool,
pub compression_ratio: f32,
pub enable_multi_level: bool,
pub swap_threshold: f32,
pub enable_prefix_caching: bool,
pub prefix_cache_size: usize,
}
impl Default for KvCacheConfig {
fn default() -> Self {
Self {
cache_type: KvCacheType::Contiguous,
dtype: KvCacheDtype::default(),
block_size: 16,
max_blocks: 2048,
enable_compression: false,
compression_ratio: 0.5,
enable_multi_level: true,
swap_threshold: 0.8,
enable_prefix_caching: true,
prefix_cache_size: 100,
}
}
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
pub enum KvCacheType {
Contiguous,
Paged,
Tree,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum KvCacheDtype {
#[default]
Fp16,
Bf16,
Int8,
Fp8,
}
impl KvCacheDtype {
pub fn parse(s: &str) -> Option<Self> {
match s.trim().to_ascii_lowercase().as_str() {
"fp16" | "f16" | "float16" => Some(Self::Fp16),
"bf16" | "bfloat16" => Some(Self::Bf16),
"int8" | "i8" => Some(Self::Int8),
"fp8" | "f8" | "f8e4m3" | "e4m3" => Some(Self::Fp8),
_ => None,
}
}
pub fn as_str(&self) -> &'static str {
match self {
Self::Fp16 => "fp16",
Self::Bf16 => "bf16",
Self::Int8 => "int8",
Self::Fp8 => "fp8",
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MemoryConfig {
pub pool_size: Option<usize>,
#[serde(default)]
pub usable_capacity_bytes: Option<usize>,
pub enable_pooling: bool,
pub alignment: usize,
pub enable_defragmentation: bool,
pub defragmentation_threshold: f32,
pub enable_memory_stats: bool,
pub pressure_warning_threshold: f32,
pub pressure_critical_threshold: f32,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub struct MemoryCapacityBudget {
pub capacity_bytes: u64,
pub usable_capacity_bytes: u64,
pub reserve_bytes: u64,
}
impl MemoryConfig {
pub fn resolve_capacity_budget(
&self,
device_capacity_bytes: u64,
) -> std::result::Result<MemoryCapacityBudget, String> {
if device_capacity_bytes == 0 {
return Err("runtime device memory capacity must be greater than zero".to_string());
}
let capacity_bytes = self
.pool_size
.map(|bytes| bytes as u64)
.unwrap_or(device_capacity_bytes)
.min(device_capacity_bytes);
if capacity_bytes == 0 {
return Err("runtime memory capacity must be greater than zero".to_string());
}
let usable_capacity_bytes = if let Some(bytes) = self.usable_capacity_bytes {
let bytes = bytes as u64;
if bytes == 0 || bytes > capacity_bytes {
return Err(format!(
"memory.usable_capacity_bytes must be in 1..={capacity_bytes}, got {bytes}"
));
}
bytes
} else {
let critical = self.pressure_critical_threshold;
if !critical.is_finite() || critical <= 0.0 || critical > 1.0 {
return Err(format!(
"memory.pressure_critical_threshold must be in (0, 1], got {critical}"
));
}
let threshold_bytes = ((capacity_bytes as f64) * f64::from(critical)).floor() as u64;
capacity_bytes.saturating_sub(
capacity_bytes
.saturating_sub(threshold_bytes)
.min(capacity_bytes - 1),
)
};
Ok(MemoryCapacityBudget {
capacity_bytes,
usable_capacity_bytes,
reserve_bytes: capacity_bytes - usable_capacity_bytes,
})
}
}
impl Default for MemoryConfig {
fn default() -> Self {
Self {
pool_size: None,
usable_capacity_bytes: None,
enable_pooling: true,
alignment: 256,
enable_defragmentation: false,
defragmentation_threshold: 0.7,
enable_memory_stats: true,
pressure_warning_threshold: 0.8,
pressure_critical_threshold: 0.95,
}
}
}
pub const MAXIMUM_REUSABLE_EXECUTION_STARTUP_CAPTURE_WIDTH: usize = 32;
pub const DEFAULT_MAXIMUM_AUTOMATIC_EXACT_DECODE_WIDTH: usize =
MAXIMUM_REUSABLE_EXECUTION_STARTUP_CAPTURE_WIDTH;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(default)]
pub struct ReusableExecutionCaptureConfig {
pub exact_decode_widths: Option<Vec<usize>>,
pub maximum_automatic_exact_decode_width: usize,
}
impl Default for ReusableExecutionCaptureConfig {
fn default() -> Self {
Self {
exact_decode_widths: None,
maximum_automatic_exact_decode_width: DEFAULT_MAXIMUM_AUTOMATIC_EXACT_DECODE_WIDTH,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BackendConfig {
pub backend_type: BackendType,
pub device: Device,
pub dtype: DataType,
pub enable_optimizations: bool,
pub optimization_level: u8,
pub enable_cuda_graphs: bool,
#[serde(default = "default_enable_reusable_execution")]
pub enable_reusable_execution: bool,
#[serde(default)]
pub reusable_execution_capture: ReusableExecutionCaptureConfig,
pub enable_kernel_fusion: bool,
pub backend_options: HashMap<String, serde_json::Value>,
}
impl Default for BackendConfig {
fn default() -> Self {
Self {
backend_type: BackendType::Candle,
device: Device::CPU,
dtype: DataType::FP16,
enable_optimizations: true,
optimization_level: 2,
enable_cuda_graphs: false,
enable_reusable_execution: default_enable_reusable_execution(),
reusable_execution_capture: ReusableExecutionCaptureConfig::default(),
enable_kernel_fusion: true,
backend_options: HashMap::new(),
}
}
}
const fn default_enable_reusable_execution() -> bool {
true
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
pub enum BackendType {
Candle,
OnnxRuntime,
TensorRT,
Custom,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TokenizerConfig {
pub tokenizer_type: TokenizerType,
pub tokenizer_path: Option<String>,
pub enable_fast: bool,
pub add_special_tokens: bool,
pub truncation: Option<TruncationConfig>,
pub padding: Option<PaddingConfig>,
}
impl Default for TokenizerConfig {
fn default() -> Self {
Self {
tokenizer_type: TokenizerType::BPE,
tokenizer_path: None,
enable_fast: true,
add_special_tokens: true,
truncation: None,
padding: None,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum TokenizerType {
BPE,
WordPiece,
SentencePiece,
Tiktoken,
Custom,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TruncationConfig {
pub max_length: usize,
pub strategy: TruncationStrategy,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum TruncationStrategy {
TruncateStart,
TruncateEnd,
TruncateBoth,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PaddingConfig {
pub strategy: PaddingStrategy,
pub token_id: u32,
pub target_length: Option<usize>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum PaddingStrategy {
None,
MaxLength,
FixedLength,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SecurityConfig {
pub enable_auth: bool,
pub api_keys: Vec<String>,
pub enable_rate_limiting: bool,
pub rate_limit_rpm: u32,
pub enable_content_filter: bool,
pub max_prompt_length: usize,
pub enable_prompt_validation: bool,
pub allowed_extensions: Vec<String>,
}
impl Default for SecurityConfig {
fn default() -> Self {
Self {
enable_auth: false,
api_keys: vec![],
enable_rate_limiting: true,
rate_limit_rpm: 60,
enable_content_filter: false,
max_prompt_length: 32768,
enable_prompt_validation: true,
allowed_extensions: vec!["txt".to_string(), "json".to_string()],
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct SamplingConfig {
pub default_params: SamplingParams,
pub presets: SamplingPresets,
pub enable_custom_processors: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MonitoringConfig {
pub enable_metrics: bool,
pub enable_tracing: bool,
pub export_interval: Duration,
}
impl Default for MonitoringConfig {
fn default() -> Self {
Self {
enable_metrics: true,
enable_tracing: true,
export_interval: Duration::from_secs(5),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BatchConfig {
pub max_batch_size: usize,
pub max_wait_ms: u64,
pub enable_dynamic: bool,
pub enable_continuous: bool,
#[serde(default = "BatchConfig::default_max_num_batched_tokens")]
pub max_num_batched_tokens: usize,
}
impl BatchConfig {
fn default_max_num_batched_tokens() -> usize {
2048
}
}
impl Default for BatchConfig {
fn default() -> Self {
Self {
max_batch_size: 32,
max_wait_ms: 8,
enable_dynamic: true,
enable_continuous: false,
max_num_batched_tokens: Self::default_max_num_batched_tokens(),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn scheduler_keeps_immediate_fit_default_until_full_input_policy_is_gated() {
assert_eq!(
SchedulerConfig::default().sequence_fit_policy,
SequenceFitPolicy::ImmediateOnly
);
}
#[test]
fn sequence_fit_policy_uses_canonical_product_values() {
assert_eq!(
serde_json::to_string(&SequenceFitPolicy::FullInputMustFit).unwrap(),
"\"full-input-must-fit\""
);
assert_eq!(
serde_json::from_str::<SequenceFitPolicy>("\"immediate-only\"").unwrap(),
SequenceFitPolicy::ImmediateOnly
);
}
#[test]
fn diagnostic_fault_uses_one_canonical_product_value() {
assert_eq!(
VNextDiagnosticFault::parse_runtime_value("prefill_resource_after_submit_once")
.unwrap(),
VNextDiagnosticFault::PrefillResourceAfterSubmitOnce
);
assert_eq!(
VNextDiagnosticFault::PrefillResourceAfterSubmitOnce.as_runtime_value(),
"prefill-resource-after-submit-once"
);
assert!(VNextDiagnosticFault::parse_runtime_value("resource-failure").is_err());
}
#[test]
fn engine_config_applies_typed_diagnostic_fault() {
let mut config = EngineConfig::default();
let snapshot = RuntimeConfigSnapshot::from_env_vars([(
"FERRUM_VNEXT_DIAGNOSTIC_FAULT",
"prefill-resource-after-submit-once",
)]);
config
.apply_runtime_config_snapshot(&snapshot)
.expect("runtime config should apply");
assert_eq!(
config.runtime.vnext_diagnostic_fault,
Some(VNextDiagnosticFault::PrefillResourceAfterSubmitOnce)
);
}
#[test]
fn engine_config_rejects_unknown_diagnostic_fault() {
let mut config = EngineConfig::default();
let snapshot = RuntimeConfigSnapshot::from_env_vars([(
"FERRUM_VNEXT_DIAGNOSTIC_FAULT",
"resource-failure",
)]);
let error = config
.apply_runtime_config_snapshot(&snapshot)
.expect_err("unknown diagnostic fault must fail closed");
assert!(error.contains("FERRUM_VNEXT_DIAGNOSTIC_FAULT"));
}
#[test]
fn scheduler_deserialization_without_fit_policy_keeps_legacy_default() {
let mut serialized = serde_json::to_value(SchedulerConfig::default()).unwrap();
serialized
.as_object_mut()
.unwrap()
.remove("sequence_fit_policy");
let scheduler: SchedulerConfig = serde_json::from_value(serialized).unwrap();
assert_eq!(
scheduler.sequence_fit_policy,
SequenceFitPolicy::ImmediateOnly
);
}
#[test]
fn checkpoint_capture_deserialization_keeps_decode_capture_disabled_by_default() {
let capture: VNextCheckpointCaptureConfig = serde_json::from_value(serde_json::json!({
"output_dir": "capture",
"value_ids": ["value.output.logits"],
"maximum_prefill_waves": 1
}))
.unwrap();
assert_eq!(capture.maximum_decode_waves, 0);
assert!(!capture.capture_product_output);
}
#[test]
fn engine_config_applies_typed_sequence_fit_policy() {
let mut config = EngineConfig::default();
let snapshot = RuntimeConfigSnapshot::from_env_vars([(
"FERRUM_SEQUENCE_FIT_POLICY",
"full-input-must-fit",
)]);
config
.apply_runtime_config_snapshot(&snapshot)
.expect("runtime config should apply");
assert_eq!(
config.scheduler.sequence_fit_policy,
SequenceFitPolicy::FullInputMustFit
);
}
#[test]
fn engine_config_rejects_unknown_sequence_fit_policy() {
let mut config = EngineConfig::default();
let snapshot = RuntimeConfigSnapshot::from_env_vars([(
"FERRUM_SEQUENCE_FIT_POLICY",
"reserve-everything",
)]);
let error = config
.apply_runtime_config_snapshot(&snapshot)
.expect_err("unknown fit policy must fail closed");
assert!(error.contains("FERRUM_SEQUENCE_FIT_POLICY"));
}
#[test]
fn engine_config_applies_recurrent_state_max_slots_runtime_key() {
let mut config = EngineConfig::default();
let snapshot =
RuntimeConfigSnapshot::from_env_vars([("FERRUM_RECURRENT_STATE_MAX_SLOTS", "16")]);
config
.apply_runtime_config_snapshot(&snapshot)
.expect("runtime config should apply");
assert_eq!(config.runtime.recurrent_state_max_slots, Some(16));
}
#[test]
fn engine_config_does_not_apply_removed_qwen35_slot_alias() {
let mut config = EngineConfig::default();
let snapshot =
RuntimeConfigSnapshot::from_env_vars([("FERRUM_QWEN35_LINEAR_STATE_MAX_SLOTS", "16")]);
config
.apply_runtime_config_snapshot(&snapshot)
.expect("runtime config should apply");
assert_eq!(config.runtime.recurrent_state_max_slots, None);
}
#[test]
fn engine_config_uses_generic_recurrent_state_slots_when_removed_alias_is_present() {
let mut config = EngineConfig::default();
let snapshot = RuntimeConfigSnapshot::from_env_vars([
("FERRUM_RECURRENT_STATE_MAX_SLOTS", "8"),
("FERRUM_QWEN35_LINEAR_STATE_MAX_SLOTS", "16"),
]);
config
.apply_runtime_config_snapshot(&snapshot)
.expect("runtime config should apply");
assert_eq!(config.runtime.recurrent_state_max_slots, Some(8));
}
#[test]
fn engine_config_applies_profile_entrypoint_runtime_key() {
let mut config = EngineConfig::default();
let snapshot = RuntimeConfigSnapshot::from_env_vars([("FERRUM_PROFILE_ENTRYPOINT", "run")]);
config
.apply_runtime_config_snapshot(&snapshot)
.expect("runtime config should apply");
assert_eq!(
config.runtime.profile_entrypoint,
Some(ProfileEntrypoint::Run)
);
}
#[test]
fn engine_config_applies_typed_profile_detail_runtime_key() {
let mut config = EngineConfig::default();
let snapshot = RuntimeConfigSnapshot::from_env_vars([("FERRUM_PROFILE_DETAIL", "full")]);
config
.apply_runtime_config_snapshot(&snapshot)
.expect("runtime config should apply");
assert_eq!(
config.runtime.profile_detail,
ObservabilityProfileDetail::Full
);
}
#[test]
fn engine_config_applies_typed_latency_profile_detail_runtime_key() {
let mut config = EngineConfig::default();
let snapshot = RuntimeConfigSnapshot::from_env_vars([("FERRUM_PROFILE_DETAIL", "latency")]);
config
.apply_runtime_config_snapshot(&snapshot)
.expect("latency profile detail should apply");
assert_eq!(
config.runtime.profile_detail,
ObservabilityProfileDetail::Latency
);
}
#[test]
fn engine_config_applies_typed_replay_profile_detail_runtime_key() {
let mut config = EngineConfig::default();
let snapshot = RuntimeConfigSnapshot::from_env_vars([("FERRUM_PROFILE_DETAIL", "replay")]);
config
.apply_runtime_config_snapshot(&snapshot)
.expect("runtime config should apply");
assert_eq!(
config.runtime.profile_detail,
ObservabilityProfileDetail::Replay
);
}
#[test]
fn engine_config_applies_typed_verification_profile_detail_runtime_key() {
let mut config = EngineConfig::default();
let snapshot = RuntimeConfigSnapshot::from_env_vars([("FERRUM_PROFILE_DETAIL", "verify")]);
config
.apply_runtime_config_snapshot(&snapshot)
.expect("runtime config should apply");
assert_eq!(
config.runtime.profile_detail,
ObservabilityProfileDetail::Verify
);
}
#[test]
fn engine_config_applies_typed_profile_jsonl_runtime_key() {
let mut config = EngineConfig::default();
let snapshot =
RuntimeConfigSnapshot::from_env_vars([("FERRUM_PROFILE_JSONL", "/tmp/profile.jsonl")]);
config
.apply_runtime_config_snapshot(&snapshot)
.expect("runtime config should apply");
assert_eq!(
config.runtime.profile_jsonl.as_deref(),
Some(std::path::Path::new("/tmp/profile.jsonl"))
);
}
#[test]
fn engine_config_rejects_unknown_profile_detail() {
let mut config = EngineConfig::default();
let snapshot =
RuntimeConfigSnapshot::from_env_vars([("FERRUM_PROFILE_DETAIL", "everything")]);
let error = config
.apply_runtime_config_snapshot(&snapshot)
.expect_err("unknown profile detail must fail closed");
assert!(error.contains("FERRUM_PROFILE_DETAIL"));
}
}