Trait KvCacheManager 

pub trait KvCacheManager: Send + Sync {
    // Required methods
    fn allocate<'life0, 'life1, 'async_trait>(
        &'life0 self,
        request: &'life1 AllocationRequest,
    ) -> Pin<Box<dyn Future<Output = Result<Arc<dyn KvCacheHandle>, FerrumError>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             'life1: 'async_trait,
             Self: 'async_trait;
    fn extend<'life0, 'life1, 'async_trait>(
        &'life0 self,
        handle: &'life1 mut dyn KvCacheHandle,
        additional_tokens: usize,
    ) -> Pin<Box<dyn Future<Output = Result<(), FerrumError>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             'life1: 'async_trait,
             Self: 'async_trait;
    fn deallocate<'life0, 'async_trait>(
        &'life0 self,
        request_id: RequestId,
    ) -> Pin<Box<dyn Future<Output = Result<(), FerrumError>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             Self: 'async_trait;
    fn can_allocate(&self, request: &AllocationRequest) -> bool;
    fn stats(&self) -> CacheManagerStats;
    fn gc<'life0, 'async_trait>(
        &'life0 self,
    ) -> Pin<Box<dyn Future<Output = Result<CacheGcStats, FerrumError>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             Self: 'async_trait;
    fn set_pressure_callback(
        &self,
        callback: Box<dyn Fn(MemoryPressure) + Send + Sync>,
    );
    fn get_handle(
        &self,
        request_id: RequestId,
    ) -> Option<Arc<dyn KvCacheHandle>>;
    fn list_handles(&self) -> Vec<(RequestId, Arc<dyn KvCacheHandle>)>;
}

KV cache manager for allocation and lifecycle management

Required Methods

fn allocate<'life0, 'life1, 'async_trait>( &'life0 self, request: &'life1 AllocationRequest, ) -> Pin<Box<dyn Future<Output = Result<Arc<dyn KvCacheHandle>, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, Self: 'async_trait,

Allocate cache for new sequence

fn extend<'life0, 'life1, 'async_trait>( &'life0 self, handle: &'life1 mut dyn KvCacheHandle, additional_tokens: usize, ) -> Pin<Box<dyn Future<Output = Result<(), FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, Self: 'async_trait,

Extend existing cache to accommodate more tokens

fn deallocate<'life0, 'async_trait>( &'life0 self, request_id: RequestId, ) -> Pin<Box<dyn Future<Output = Result<(), FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, Self: 'async_trait,

Deallocate cache (handle becomes invalid)

fn can_allocate(&self, request: &AllocationRequest) -> bool

Check if can allocate requested cache size

fn stats(&self) -> CacheManagerStats

Get cache statistics

fn gc<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<CacheGcStats, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, Self: 'async_trait,

Force garbage collection of unused caches

fn set_pressure_callback( &self, callback: Box<dyn Fn(MemoryPressure) + Send + Sync>, )

Set memory pressure callback

fn get_handle(&self, request_id: RequestId) -> Option<Arc<dyn KvCacheHandle>>

Get handle for existing request (if exists)

fn list_handles(&self) -> Vec<(RequestId, Arc<dyn KvCacheHandle>)>

List all active cache handles

Dyn Compatibility

This trait is dyn compatible.

In older versions of Rust, dyn compatibility was called "object safety".

Implementations on Foreign Types

impl KvCacheManager for DefaultKvCacheManager

fn allocate<'life0, 'life1, 'async_trait>( &'life0 self, request: &'life1 AllocationRequest, ) -> Pin<Box<dyn Future<Output = Result<Arc<dyn KvCacheHandle>, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, DefaultKvCacheManager: 'async_trait,

fn extend<'life0, 'life1, 'async_trait>( &'life0 self, _handle: &'life1 mut dyn KvCacheHandle, _additional_tokens: usize, ) -> Pin<Box<dyn Future<Output = Result<(), FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, DefaultKvCacheManager: 'async_trait,

fn deallocate<'life0, 'async_trait>( &'life0 self, request_id: RequestId, ) -> Pin<Box<dyn Future<Output = Result<(), FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, DefaultKvCacheManager: 'async_trait,

fn can_allocate(&self, _request: &AllocationRequest) -> bool

fn stats(&self) -> CacheManagerStats

fn gc<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<CacheGcStats, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, DefaultKvCacheManager: 'async_trait,

fn set_pressure_callback( &self, callback: Box<dyn Fn(MemoryPressure) + Send + Sync>, )

fn get_handle(&self, request_id: RequestId) -> Option<Arc<dyn KvCacheHandle>>

fn list_handles(&self) -> Vec<(RequestId, Arc<dyn KvCacheHandle>)>

impl KvCacheManager for PagedKvCacheManager

fn allocate<'life0, 'life1, 'async_trait>( &'life0 self, request: &'life1 AllocationRequest, ) -> Pin<Box<dyn Future<Output = Result<Arc<dyn KvCacheHandle>, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, PagedKvCacheManager: 'async_trait,

fn extend<'life0, 'life1, 'async_trait>( &'life0 self, handle: &'life1 mut dyn KvCacheHandle, additional_tokens: usize, ) -> Pin<Box<dyn Future<Output = Result<(), FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, PagedKvCacheManager: 'async_trait,

fn deallocate<'life0, 'async_trait>( &'life0 self, request_id: RequestId, ) -> Pin<Box<dyn Future<Output = Result<(), FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, PagedKvCacheManager: 'async_trait,

fn can_allocate(&self, request: &AllocationRequest) -> bool

fn stats(&self) -> CacheManagerStats

fn gc<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<CacheGcStats, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, PagedKvCacheManager: 'async_trait,

fn set_pressure_callback( &self, callback: Box<dyn Fn(MemoryPressure) + Send + Sync>, )

fn get_handle(&self, request_id: RequestId) -> Option<Arc<dyn KvCacheHandle>>

fn list_handles(&self) -> Vec<(RequestId, Arc<dyn KvCacheHandle>)>

Implementors