Struct LayerKVCache 

pub struct LayerKVCache { /* private fields */ }

Per-layer KV cache storing Key and Value tensors with pre-allocated buffers

Implementations

impl LayerKVCache

pub fn new() -> Self

Create an empty layer cache with default max sequence length

pub fn with_capacity(max_seq_len: usize) -> Self

Create a layer cache with specified maximum sequence length

pub fn is_empty(&self) -> bool

Check if this layer has cached values

pub fn clear(&mut self)

Clear cached values (keeps buffers allocated for reuse)

pub fn deallocate(&mut self)

Fully deallocate buffers

pub fn seq_len(&self) -> usize

Get the current cached sequence length

pub fn max_seq_len(&self) -> usize

Get maximum sequence length (buffer capacity)

pub fn append(&mut self, new_k: &Tensor, new_v: &Tensor) -> Result<()>

Append new K,V tensors to the cache using slice_set (zero-allocation)

Arguments

• new_k - New key tensor: [batch, num_kv_heads, new_seq_len, head_dim]
• new_v - New value tensor: [batch, num_kv_heads, new_seq_len, head_dim]

Returns

Ok(()) on success, Err if buffer overflow

pub fn get_kv(&self) -> Option<(Tensor, Tensor)>

Get view of the filled portion of K,V buffers

Returns (K, V) tensors narrowed to current sequence length

pub fn k(&self) -> Option<Tensor>

👎Deprecated:

Use get_kv() instead for better performance

Get cached K tensor (legacy interface)

pub fn v(&self) -> Option<Tensor>

👎Deprecated:

Use get_kv() instead for better performance

Get cached V tensor (legacy interface)

pub fn set_k(&mut self, k: Tensor)

👎Deprecated:

Use append() instead for better performance

Set K tensor (legacy interface - converts to append)

pub fn set_v(&mut self, v: Tensor)

👎Deprecated:

Use append() instead for better performance

Set V tensor (legacy interface)

Trait Implementations

impl Clone for LayerKVCache

fn clone(&self) -> LayerKVCache

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for LayerKVCache

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for LayerKVCache

fn default() -> Self

Returns the “default value” for a type.