Trait MultiDeviceCacheManager 

pub trait MultiDeviceCacheManager: KvCacheManager {
    // Required methods
    fn supported_devices(&self) -> Vec<Device>;
    fn set_device_preference(&self, devices: Vec<Device>);
    fn move_cache<'life0, 'async_trait>(
        &'life0 self,
        request_id: RequestId,
        target_device: Device,
    ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait;
    fn get_cache_device(&self, request_id: RequestId) -> Option<Device>;
    fn rebalance_devices<'life0, 'async_trait>(
        &'life0 self,
    ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait;
    fn device_stats(&self) -> HashMap<Device, CacheManagerStats>;
}

Multi-device cache manager supporting GPU/CPU hierarchies

Required Methods

fn supported_devices(&self) -> Vec<Device>

Get supported devices

fn set_device_preference(&self, devices: Vec<Device>)

Set device preference for new allocations

fn move_cache<'life0, 'async_trait>( &'life0 self, request_id: RequestId, target_device: Device, ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Move cache between devices

fn get_cache_device(&self, request_id: RequestId) -> Option<Device>

Get cache location

fn rebalance_devices<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Balance cache distribution across devices

fn device_stats(&self) -> HashMap<Device, CacheManagerStats>

Get per-device statistics

Implementors