Struct KvCache 

pub struct KvCache {
    pub layers: Vec<LayerKvCache>,
    pub max_seq_len: usize,
    pub policy: EvictionPolicy,
}

Full KV cache for all layers.

Fields

layers: Vec<LayerKvCache>

max_seq_len: usize

policy: EvictionPolicy

Implementations

impl KvCache

pub fn new( num_layers: usize, num_kv_heads: usize, head_dim: usize, max_seq_len: usize, ) -> Self

pub fn clear(&mut self)

pub fn total_memory_bytes(&self) -> usize

pub fn seq_len(&self) -> usize

Current sequence length (max across layers — dead layers may lag).

pub fn needs_eviction(&self) -> bool

pub fn evict(&mut self, keep_last: usize)

Evict down to keep_last positions according to the policy.

Trait Implementations

impl Debug for KvCache

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.