Struct RealtimeOptimizer 

pub struct RealtimeOptimizer { /* private fields */ }

Real-time optimizer for LLM inference

Implementations

impl RealtimeOptimizer

pub fn new(config: RealtimeConfig) -> Self

Create a new realtime optimizer

pub fn optimize_batch_size(&self, recent_latencies: &[f32]) -> usize

Optimize batch size based on recent latency measurements

pub fn should_evict_kv_cache(&self) -> bool

Check if KV cache eviction is needed

pub fn update_kv_cache_pressure(&self, pressure: f32)

Update KV cache pressure

pub fn kv_cache_pressure(&self) -> f32

Get KV cache pressure

pub fn allocate_token_budget( &self, requests: &[Request], ) -> Vec<TokenBudgetAllocation>

Allocate token budgets for a set of requests

pub fn enable_speculative_decoding(&self, draft_model: &str)

Enable speculative decoding

pub fn disable_speculative_decoding(&self)

Disable speculative decoding

pub fn update_speculation_stats( &self, accepted_count: usize, total_drafted: usize, )

Update speculation statistics for learning/monitoring

This records the acceptance rate of speculative decoding rounds to help tune the lookahead parameter adaptively.

Arguments

• accepted_count - Number of draft tokens that were accepted
• total_drafted - Total number of draft tokens generated

pub fn is_speculative_active(&self) -> bool

Check if speculative decoding is active

pub fn draft_model(&self) -> Option<String>

Get the draft model identifier

pub fn record_latency(&self, latency_ms: f32)

Record a latency measurement

pub fn record_throughput(&self, tps: f32)

Record a throughput measurement

pub fn average_latency(&self) -> f32

Get average latency

pub fn average_throughput(&self) -> f32

Get average throughput

pub fn update_memory_usage(&self, bytes: usize)

Update memory usage

pub fn memory_pressure(&self) -> f32

Get memory pressure (0.0 - 1.0)

pub fn optimize(&self, metrics: &InferenceMetrics) -> OptimizationDecision

Make a comprehensive optimization decision

pub fn add_request(&self, request: Request)

Add a pending request

pub fn remove_request(&self, request_id: &str)

Remove a completed request

pub fn pending_request_count(&self) -> usize

Get pending request count

pub fn current_batch_size(&self) -> usize

Get current batch size

pub fn update_config(&self, config: RealtimeConfig)

Update configuration

pub fn config(&self) -> RealtimeConfig

Get current configuration

impl RealtimeOptimizer

pub fn should_use_speculative(&self, params: &GenerateParams) -> bool

Check if speculative decoding should be used for these generation parameters

Returns true when:

• Temperature is low (< 0.5) - deterministic generation benefits most
• Greedy decoding (top_k = 1)
• Speculative decoding is enabled in config

pub fn get_speculative_config(&self) -> SpeculativeConfig

Get recommended speculative decoding configuration based on current metrics

Trait Implementations

impl Default for RealtimeOptimizer

fn default() -> Self

Returns the “default value” for a type.