Struct InferenceContext 

pub struct InferenceContext {
    pub kv_cache: KVCache,
    pub backend: Arc<dyn Backend>,
    pub position: usize,
    pub recurrent_state: Option<RecurrentState>,
    pub tq_cache: Option<TurboQuantKVCache>,
}

Context for model inference

Fields

kv_cache: KVCache

KV cache for attention

backend: Arc<dyn Backend>

Backend to use for computation

position: usize

Current position in sequence

recurrent_state: Option<RecurrentState>

Recurrent state for delta-net layers (None if model has no SSM layers)

tq_cache: Option<TurboQuantKVCache>

Optional TurboQuant-compressed KV cache (replaces f32 cache for attention)

Implementations

impl InferenceContext

pub fn new(config: &ModelConfig, backend: Arc<dyn Backend>) -> Self

Create a new inference context

pub fn new_with_cache_type( config: &ModelConfig, backend: Arc<dyn Backend>, cache_type: KVCacheType, ) -> Self

Create inference context with a specific KV cache type.

pub fn new_with_recurrent( config: &ModelConfig, backend: Arc<dyn Backend>, is_recurrent: &[bool], rc: &RecurrentConfig, ) -> Self

Create inference context with recurrent state for SSM layers. is_recurrent[i] marks which layers are recurrent (DeltaNet or Mamba).

pub fn reset(&mut self)

Reset context for a new sequence

pub fn has_turboquant(&self) -> bool

Whether TurboQuant KV cache is active.