Trait KvCacheHandleInterface 

pub trait KvCacheHandleInterface:
    Send
    + Sync
    + Debug {
    // Required methods
    fn block_table(&self) -> &BlockTable;
    fn block_table_mut(&mut self) -> &mut BlockTable;
    fn as_any(&self) -> &(dyn Any + 'static);
    fn device(&self) -> Device;
    fn num_layers(&self) -> usize;
    fn num_heads(&self) -> usize;
    fn head_dim(&self) -> usize;
    fn key_cache(
        &self,
        layer: usize,
    ) -> Result<Option<Arc<dyn TensorLike>>, FerrumError>;
    fn value_cache(
        &self,
        layer: usize,
    ) -> Result<Option<Arc<dyn TensorLike>>, FerrumError>;
    fn clone_handle(&self) -> Result<Arc<dyn KvCacheHandle>, FerrumError>;
    fn stats(&self) -> CacheHandleStats;
    fn is_valid(&self) -> bool;
    fn cache_id(&self) -> String;

    // Provided methods
    fn num_tokens(&self) -> usize { ... }
    fn kv_cache(
        &self,
        layer: usize,
    ) -> Result<(Option<Arc<dyn TensorLike>>, Option<Arc<dyn TensorLike>>), FerrumError> { ... }
}

KV cache handle providing access to cached key-value states

Required Methods

fn block_table(&self) -> &BlockTable

Get block table for this cache

fn block_table_mut(&mut self) -> &mut BlockTable

Get mutable block table (for extending)

fn as_any(&self) -> &(dyn Any + 'static)

Downcast support for backend-specific handles

fn device(&self) -> Device

Get device where cache resides

fn num_layers(&self) -> usize

Get number of layers cached

fn num_heads(&self) -> usize

Get number of attention heads

fn head_dim(&self) -> usize

Get head dimension

fn key_cache( &self, layer: usize, ) -> Result<Option<Arc<dyn TensorLike>>, FerrumError>

Get key cache for specific layer (returns tensor reference)

fn value_cache( &self, layer: usize, ) -> Result<Option<Arc<dyn TensorLike>>, FerrumError>

Get value cache for specific layer

fn clone_handle(&self) -> Result<Arc<dyn KvCacheHandle>, FerrumError>

Clone handle (creates new reference, not deep copy)

fn stats(&self) -> CacheHandleStats

Get cache statistics

fn is_valid(&self) -> bool

Check if cache is valid and accessible

fn cache_id(&self) -> String

Get unique identifier for this cache instance

Provided Methods

fn num_tokens(&self) -> usize

Get number of tokens stored in cache

fn kv_cache( &self, layer: usize, ) -> Result<(Option<Arc<dyn TensorLike>>, Option<Arc<dyn TensorLike>>), FerrumError>

Get both key and value caches for layer

Dyn Compatibility

This trait is dyn compatible.

In older versions of Rust, dyn compatibility was called "object safety".

Implementors

impl KvCacheHandle for DefaultKvCacheHandle

impl KvCacheHandle for PagedKvCacheHandle