Struct GenericKvCacheHandle 

pub struct GenericKvCacheHandle { /* private fields */ }

Generic KV cache handle usable by any model architecture.

Implementations

impl GenericKvCacheHandle

pub fn new( num_layers: usize, num_heads: usize, head_dim: usize, device: CandleDevice, seq_len: usize, cache_id: String, ) -> Self

pub fn with_sequence_length(&self, seq_len: usize) -> Self

pub fn request_cache_id(&self) -> &str

Trait Implementations

impl Clone for GenericKvCacheHandle

fn clone(&self) -> GenericKvCacheHandle

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for GenericKvCacheHandle

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl KvCacheHandle for GenericKvCacheHandle

fn block_table(&self) -> &BlockTable

Get block table for this cache

fn block_table_mut(&mut self) -> &mut BlockTable

Get mutable block table (for extending)

fn as_any(&self) -> &dyn Any

Downcast support for backend-specific handles

fn device(&self) -> Device

Get device where cache resides

fn num_layers(&self) -> usize

Get number of layers cached

fn num_heads(&self) -> usize

Get number of attention heads

fn head_dim(&self) -> usize

Get head dimension

fn key_cache(&self, _layer: usize) -> Result<Option<TensorRef>>

Get key cache for specific layer (returns tensor reference)

fn value_cache(&self, _layer: usize) -> Result<Option<TensorRef>>

Get value cache for specific layer

fn clone_handle(&self) -> Result<Arc<dyn KvCacheHandle>>

Clone handle (creates new reference, not deep copy)

fn stats(&self) -> CacheHandleStats

Get cache statistics

fn is_valid(&self) -> bool

Check if cache is valid and accessible

fn cache_id(&self) -> String

Get unique identifier for this cache instance

fn num_tokens(&self) -> usize

Get number of tokens stored in cache

fn kv_cache( &self, layer: usize, ) -> Result<(Option<Arc<dyn TensorLike>>, Option<Arc<dyn TensorLike>>), FerrumError>

Get both key and value caches for layer