Struct DefaultKvCacheManager 

pub struct DefaultKvCacheManager { /* private fields */ }

Default KV cache manager - MVP implementation

Implementations

impl DefaultKvCacheManager

pub fn new(device: Device, block_size: usize, max_blocks: usize) -> Result<Self>

Trait Implementations

impl Debug for DefaultKvCacheManager

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl KvCacheManager for DefaultKvCacheManager

fn allocate<'life0, 'life1, 'async_trait>( &'life0 self, request: &'life1 AllocationRequest, ) -> Pin<Box<dyn Future<Output = Result<Arc<dyn KvCacheHandle>>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Allocate cache for new sequence

fn extend<'life0, 'life1, 'async_trait>( &'life0 self, _handle: &'life1 mut dyn KvCacheHandle, _additional_tokens: usize, ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Extend existing cache to accommodate more tokens

fn deallocate<'life0, 'async_trait>( &'life0 self, request_id: RequestId, ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Deallocate cache (handle becomes invalid)

fn can_allocate(&self, _request: &AllocationRequest) -> bool

Check if can allocate requested cache size

fn stats(&self) -> CacheManagerStats

Get cache statistics

fn gc<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<CacheGcStats>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Force garbage collection of unused caches

fn set_pressure_callback( &self, callback: Box<dyn Fn(MemoryPressure) + Send + Sync>, )

Set memory pressure callback

fn get_handle(&self, request_id: RequestId) -> Option<Arc<dyn KvCacheHandle>>

Get handle for existing request (if exists)

fn list_handles(&self) -> Vec<(RequestId, Arc<dyn KvCacheHandle>)>

List all active cache handles