Trait KvCacheHandle 

pub trait KvCacheHandle:
    Send
    + Sync
    + Debug {
    // Required methods
    fn block_table(&self) -> &BlockTable;
    fn block_table_mut(&mut self) -> &mut BlockTable;
    fn as_any(&self) -> &dyn Any;
    fn device(&self) -> Device;
    fn num_layers(&self) -> usize;
    fn num_heads(&self) -> usize;
    fn head_dim(&self) -> usize;
    fn key_cache(&self, layer: usize) -> Result<Option<TensorRef>>;
    fn value_cache(&self, layer: usize) -> Result<Option<TensorRef>>;
    fn clone_handle(&self) -> Result<Arc<dyn KvCacheHandle>>;
    fn stats(&self) -> CacheHandleStats;
    fn is_valid(&self) -> bool;
    fn cache_id(&self) -> String;

    // Provided methods
    fn num_tokens(&self) -> usize { ... }
    fn kv_cache(
        &self,
        layer: usize,
    ) -> Result<(Option<TensorRef>, Option<TensorRef>)> { ... }
}

KV cache handle providing access to cached key-value states

Required Methods

fn block_table(&self) -> &BlockTable

Get block table for this cache

fn block_table_mut(&mut self) -> &mut BlockTable

Get mutable block table (for extending)

fn as_any(&self) -> &dyn Any

Downcast support for backend-specific handles

fn device(&self) -> Device

Get device where cache resides

fn num_layers(&self) -> usize

Get number of layers cached

fn num_heads(&self) -> usize

Get number of attention heads

fn head_dim(&self) -> usize

Get head dimension

fn key_cache(&self, layer: usize) -> Result<Option<TensorRef>>

Get key cache for specific layer (returns tensor reference)

fn value_cache(&self, layer: usize) -> Result<Option<TensorRef>>

Get value cache for specific layer

fn clone_handle(&self) -> Result<Arc<dyn KvCacheHandle>>

Clone handle (creates new reference, not deep copy)

fn stats(&self) -> CacheHandleStats

Get cache statistics

fn is_valid(&self) -> bool

Check if cache is valid and accessible

fn cache_id(&self) -> String

Get unique identifier for this cache instance

Provided Methods

fn num_tokens(&self) -> usize

Get number of tokens stored in cache

fn kv_cache( &self, layer: usize, ) -> Result<(Option<TensorRef>, Option<TensorRef>)>

Get both key and value caches for layer

Implementors