Struct ServeCommand 

pub struct ServeCommand {

    pub model: Option<String>,
    pub model_option: Option<String>,
    pub product_sources: ProductSourceArgs,
    pub served_model_name: Vec<String>,
    pub enable_thinking: bool,
    pub disable_thinking: bool,
    pub host: Option<String>,
    pub port: Option<u16>,
    pub tts_slots: usize,
    pub backend: String,
    pub gpu_devices: Option<String>,
    pub layer_split_pipeline_mode: Option<LayerSplitPipelineModeArg>,
    pub spec_draft: Option<String>,
    pub spec_tokens: usize,
    pub gpu_memory_utilization: f32,
    pub runtime_memory_budget_bytes: Option<NonZeroUsize>,
    pub max_model_len: Option<usize>,
    pub max_num_seqs: Option<usize>,
    pub max_num_batched_tokens: Option<usize>,
    pub sequence_fit_policy: Option<SequenceFitPolicyArg>,
    pub scheduler_prefill_first_until_active: Option<usize>,
    pub scheduler_prefill_step_chunk: Option<usize>,
    pub scheduler_active_decode_prefill_chunk: Option<usize>,
    pub enable_prefix_caching: bool,
    pub no_enable_prefix_caching: bool,
    pub enable_prefix_cache: bool,
    pub disable_prefix_cache: bool,
    pub session_cache: Option<String>,
    pub session_cache_max_entries: Option<usize>,
    pub session_cache_max_tokens: Option<usize>,
    pub kv_dtype: Option<String>,
    pub kv_capacity: Option<usize>,
    pub kv_max_blocks: Option<usize>,
    pub greedy_argmax: bool,
    pub disable_greedy_argmax: bool,
    pub batched_graph: bool,
    pub disable_batched_graph: bool,
    pub reusable_execution: bool,
    pub disable_reusable_execution: bool,
    pub unified_graph: bool,
    pub disable_unified_graph: bool,
    pub unified_graph_layers_only: bool,
    pub disable_unified_graph_layers_only: bool,
    pub unified_graph_lm_head_eager: bool,
    pub disable_unified_graph_lm_head_eager: bool,
    pub runtime_preset: Option<String>,
    pub effective_config_json: Option<PathBuf>,
    pub decision_trace_jsonl: Option<PathBuf>,
    pub observability_vertical_slice_out: Option<PathBuf>,
    pub vnext_checkpoint: VNextCheckpointArgs,
    pub profile_jsonl: Option<PathBuf>,
    pub profile_detail: ProfileDetailArg,
    pub vnext_diagnostic_fault: Option<VNextDiagnosticFaultArg>,
    pub memory_profile_jsonl: Option<PathBuf>,
    pub scheduler_trace_jsonl: Option<PathBuf>,
    pub request_dump_dir: Option<PathBuf>,
    pub profile_sample_rate: f64,
    pub profile_commit_sha: Option<String>,
    pub profile_env_hash: Option<String>,
    pub profile_model: Option<String>,
    pub profile_concurrency: Option<u32>,
    pub profile_runtime_flags_json: Option<String>,
    pub lora: Vec<String>,
    pub lora_model_id_template: String,
}

Fields

model: Option<String>

Model to serve (default: from config)

model_option: Option<String>

Model to serve (default: from config)

product_sources: ProductSourceArgs

served_model_name: Vec<String>

Public OpenAI-compatible model names. The first name is primary and additional names are aliases for the same loaded model.

enable_thinking: bool

Enable model reasoning by default when a request omits chat_template_kwargs.enable_thinking.

disable_thinking: bool

Disable model reasoning by default when a request omits chat_template_kwargs.enable_thinking.

host: Option<String>

Host to bind to

port: Option<u16>

Port to listen on

tts_slots: usize

Number of TTS concurrent slots (default: 2)

backend: String

Backend: auto, cpu, metal, cuda.

gpu_devices: Option<String>

CUDA GPU ids to use, comma-separated. Multi-GPU requests select layer-split for supported Llama-family safetensors models.

layer_split_pipeline_mode: Option<LayerSplitPipelineModeArg>

Layer-split decode pipeline mode for multi-GPU CUDA serving.

spec_draft: Option<String>

Speculative decoding: draft model id (same family as target). Example: --spec-draft qwen3:0.6b when serving qwen3:4b. The draft model must share the tokenizer + vocabulary.

spec_tokens: usize

Number of speculative tokens per draft forward pass (default: 4). Only active when –spec-draft is set.

gpu_memory_utilization: f32

Fraction of GPU memory ferrum is allowed to use (mirrors vLLM’s --gpu-memory-utilization). Auto-sizes the KV pool to fit weights + scratch + KV inside total_mem * util. Default 0.9. Set 1.0 for an exclusive GPU; lower if you share the card.

runtime_memory_budget_bytes: Option<NonZeroUsize>

Exact device-wide memory budget available to runtime weights and dynamic resources. This is the same typed ceiling used by run.

max_model_len: Option<usize>

vLLM-compatible alias for FERRUM_MAX_MODEL_LEN.

max_num_seqs: Option<usize>

vLLM-compatible alias for FERRUM_PAGED_MAX_SEQS.

max_num_batched_tokens: Option<usize>

vLLM-compatible alias for FERRUM_MAX_BATCHED_TOKENS.

sequence_fit_policy: Option<SequenceFitPolicyArg>

Sequence fit gate used before prefill admission.

scheduler_prefill_first_until_active: Option<usize>

Prefer prefilling until this many requests are active before early decodes.

scheduler_prefill_step_chunk: Option<usize>

Cap per-request scheduler prefill chunks before they enter the engine.

scheduler_active_decode_prefill_chunk: Option<usize>

Cap prefill chunks while decode requests are active.

enable_prefix_caching: bool

Enable prefix caching (FERRUM_PREFIX_CACHE=1).

no_enable_prefix_caching: bool

Disable prefix caching (FERRUM_PREFIX_CACHE=0).

enable_prefix_cache: bool

Enable prefix cache (FERRUM_PREFIX_CACHE=1).

disable_prefix_cache: bool

Disable prefix cache (FERRUM_PREFIX_CACHE=0).

session_cache: Option<String>

Session cache mode (off or memory).

session_cache_max_entries: Option<usize>

Maximum in-memory session cache entries.

session_cache_max_tokens: Option<usize>

Approximate maximum tokens retained per session.

kv_dtype: Option<String>

KV cache element dtype (Dim 5 polymorphism point). Accepts fp16, bf16, int8, fp8. Default fp16. INT8 / FP8 require model wire-up; today only the kernel + type layer ships. Override via FERRUM_KV_DTYPE env var.

kv_capacity: Option<usize>

Per-sequence KV token capacity (FERRUM_KV_CAPACITY).

kv_max_blocks: Option<usize>

Global KV block budget (FERRUM_KV_MAX_BLOCKS).

greedy_argmax: bool

Use GPU argmax for greedy decoding (FERRUM_GREEDY_ARGMAX=1).

disable_greedy_argmax: bool

Disable GPU argmax for greedy decoding (FERRUM_GREEDY_ARGMAX=0).

batched_graph: bool

Enable legacy Llama/Gemma batched decode CUDA graph replay.

disable_batched_graph: bool

Disable legacy Llama/Gemma batched decode CUDA graph replay.

reusable_execution: bool

Enable vNext reusable device-program preparation.

disable_reusable_execution: bool

Disable vNext reusable device-program preparation.

unified_graph: bool

Enable Llama/Gemma unified decode CUDA graph replay.

disable_unified_graph: bool

Disable Llama/Gemma unified decode CUDA graph replay.

unified_graph_layers_only: bool

Capture only Llama/Gemma unified transformer layers in CUDA graph replay.

disable_unified_graph_layers_only: bool

Disable layers-only unified CUDA graph capture scope.

unified_graph_lm_head_eager: bool

Capture unified layers plus final packing; leave lm_head eager.

disable_unified_graph_lm_head_eager: bool

Disable lm-head-eager unified CUDA graph capture scope.

runtime_preset: Option<String>

Named startup/runtime preset, for example m3_qwen3_30b_a3b_int4.

effective_config_json: Option<PathBuf>

Write the startup effective runtime config JSON artifact.

decision_trace_jsonl: Option<PathBuf>

Write the startup auto-config decision trace JSONL artifact.

observability_vertical_slice_out: Option<PathBuf>

Generate a synthetic/no-weight observability vertical-slice artifact and exit.

vnext_checkpoint: VNextCheckpointArgs

profile_jsonl: Option<PathBuf>

Write native structured profile events to this JSONL path.

profile_detail: ProfileDetailArg

Product observability detail level.

vnext_diagnostic_fault: Option<VNextDiagnosticFaultArg>

Inject one typed vNext diagnostic fault. Requires a latency profile.

memory_profile_jsonl: Option<PathBuf>

Write product memory profile events to this JSONL path.

scheduler_trace_jsonl: Option<PathBuf>

Write scheduler iteration trace events to this JSONL path.

request_dump_dir: Option<PathBuf>

Write a sanitized request/replay bundle to this directory.

profile_sample_rate: f64

Product observability sampling rate for resource lifecycle events.

profile_commit_sha: Option<String>

Git commit stamped into native structured profile events.

profile_env_hash: Option<String>

Runtime environment hash stamped into native structured profile events.

profile_model: Option<String>

Model label stamped into native structured profile events.

profile_concurrency: Option<u32>

Concurrency stamped into native structured profile events.

profile_runtime_flags_json: Option<String>

Runtime flags/config JSON object embedded in native profile events.

lora: Vec<String>

Startup-loaded LoRA adapter, formatted as NAME=PATH. May be repeated.

lora_model_id_template: String

Public model id template for LoRA adapters. Supports and .

Trait Implementations

impl Args for ServeCommand

fn group_id() -> Option<Id>

Report the ArgGroup::id for this set of arguments

fn augment_args<'b>(__clap_app: Command) -> Command

Append to Command so it can instantiate Self via FromArgMatches::from_arg_matches_mut

fn augment_args_for_update<'b>(__clap_app: Command) -> Command

Append to Command so it can instantiate self via FromArgMatches::update_from_arg_matches_mut

impl FromArgMatches for ServeCommand

fn from_arg_matches(__clap_arg_matches: &ArgMatches) -> Result<Self, Error>

Instantiate Self from ArgMatches, parsing the arguments as needed.

fn from_arg_matches_mut( __clap_arg_matches: &mut ArgMatches, ) -> Result<Self, Error>

Instantiate Self from ArgMatches, parsing the arguments as needed.

fn update_from_arg_matches( &mut self, __clap_arg_matches: &ArgMatches, ) -> Result<(), Error>

Assign values from ArgMatches to self.

fn update_from_arg_matches_mut( &mut self, __clap_arg_matches: &mut ArgMatches, ) -> Result<(), Error>

Assign values from ArgMatches to self.