Struct KVCache 

pub struct KVCache { /* private fields */ }

Key-Value cache for efficient transformer inference

Implementations

impl KVCache

pub fn new(num_layers: usize, num_heads: usize, head_dim: usize) -> Self

Create a new KV-cache

pub fn from_config(config: KVCacheConfig) -> Result<Self>

Create KV-cache from configuration

pub fn config(&self) -> &KVCacheConfig

Get configuration

pub fn is_enabled(&self) -> bool

Check if cache is enabled

pub fn step(&self) -> usize

Get current generation step

pub fn init_layer(&mut self, layer_idx: usize, batch_size: usize) -> Result<()>

Initialize cache for a layer

pub fn update_layer( &mut self, layer_idx: usize, new_keys: Vec<f32>, new_values: Vec<f32>, ) -> Result<()>

Update cache for a layer with new keys and values

pub fn get_layer(&self, layer_idx: usize) -> Result<(&[f32], &[f32])>

Get cached keys and values for a layer

pub fn get_seq_len(&self, layer_idx: usize) -> Result<usize>

Get sequence length for a layer

pub fn clear_layer(&mut self, layer_idx: usize)

Clear cache for a specific layer

pub fn clear_all(&mut self)

Clear all cache entries

pub fn next_step(&mut self)

Increment generation step

pub fn reset(&mut self)

Reset to initial state

pub fn num_cached_layers(&self) -> usize

Get number of cached layers

pub fn current_memory_usage(&self) -> usize

Calculate current memory usage

pub fn current_memory_usage_mb(&self) -> f64

Calculate memory usage in MB

pub fn stats(&self) -> CacheStats

Get cache statistics

Trait Implementations

impl Clone for KVCache

fn clone(&self) -> KVCache

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for KVCache

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.