Struct Config 

pub struct Config {

    pub model_path: String,
    pub llama_bin: String,
    pub llama_host: String,
    pub llama_port: u16,
    pub ctx_size: u32,
    pub batch_size: u32,
    pub threads: u32,
    pub gpu_layers: u32,
    pub health_timeout_seconds: u64,
    pub hot_swap_grace_seconds: u64,
    pub max_concurrent_streams: u32,
    pub parallel_slots: u32,
    pub ubatch_size: u32,
    pub prometheus_port: u16,
    pub api_host: String,
    pub api_port: u16,
    pub requests_per_second: u32,
    pub generate_timeout_seconds: u64,
    pub stream_timeout_seconds: u64,
    pub health_check_timeout_seconds: u64,
    pub queue_size: usize,
    pub queue_timeout_seconds: u64,
    pub backend_url: String,
    pub openrouter_api_key: String,
    pub draft_model_path: String,
    pub speculative_draft_max: u32,
    pub speculative_draft_p_min: f32,
}

Fields

model_path: String

llama_bin: String

llama_host: String

llama_port: u16

ctx_size: u32

batch_size: u32

threads: u32

gpu_layers: u32

health_timeout_seconds: u64

hot_swap_grace_seconds: u64

max_concurrent_streams: u32

parallel_slots: u32

ubatch_size: u32

prometheus_port: u16

api_host: String

api_port: u16

requests_per_second: u32

generate_timeout_seconds: u64

stream_timeout_seconds: u64

health_check_timeout_seconds: u64

queue_size: usize

queue_timeout_seconds: u64

backend_url: String

openrouter_api_key: String

draft_model_path: String

Path to draft model for speculative decoding (empty string = disabled). Set DRAFT_MODEL_PATH in .env to enable. The draft model should be a smaller version of the main model (e.g. 0.5B for a 3B main model).

speculative_draft_max: u32

Maximum number of draft tokens the draft model generates per step. Higher values increase throughput gains but reduce acceptance rate. Maps to llama-server –draft-max. Default: 8.

speculative_draft_p_min: f32

Minimum acceptance probability for a draft token to be kept. Tokens below this threshold are rejected early. Default: 0.4.

Implementations

impl Config

pub fn from_env() -> Result<Self>

pub fn print_config(&self)

pub fn api_addr(&self) -> SocketAddr

Trait Implementations

impl Clone for Config

fn clone(&self) -> Config

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for Config

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.