pub struct PagedKvCacheManager { /* private fields */ }Expand description
Thread-safe high-level manager orchestrating KV-caches across multiple concurrent inference sequences.
Implementations§
Source§impl PagedKvCacheManager
impl PagedKvCacheManager
pub fn new(total_blocks: usize, block_size: usize) -> Self
pub fn with_defaults(total_blocks: usize) -> Self
Sourcepub fn create_sequence(&self, seq_id: u64)
pub fn create_sequence(&self, seq_id: u64)
Registers a new sequence/request with the cache manager.
Sourcepub fn append_tokens(
&self,
seq_id: u64,
tokens: &[TokenId],
) -> Result<(), CacheError>
pub fn append_tokens( &self, seq_id: u64, tokens: &[TokenId], ) -> Result<(), CacheError>
Appends tokens to an active sequence.
Sourcepub fn rollback_sequence(
&self,
seq_id: u64,
valid_token_count: usize,
) -> Result<usize, CacheError>
pub fn rollback_sequence( &self, seq_id: u64, valid_token_count: usize, ) -> Result<usize, CacheError>
Executes an O(1) rollback on the sequence when speculative tokens are rejected.
Sourcepub fn remove_sequence(&self, seq_id: u64) -> Result<(), CacheError>
pub fn remove_sequence(&self, seq_id: u64) -> Result<(), CacheError>
Releases all KV-cache memory held by a sequence when the request finishes.
Sourcepub fn evict_lru_sequence(&self) -> Result<Option<u64>, CacheError>
pub fn evict_lru_sequence(&self) -> Result<Option<u64>, CacheError>
Evicts the least recently used sequence to reclaim blocks under memory pressure. Returns the sequence ID of the evicted sequence, if any.
Sourcepub fn defragment(&self)
pub fn defragment(&self)
Defragments the free physical block list to optimize spatial memory cache locality.
Sourcepub fn utilization_ratio(&self) -> f32
pub fn utilization_ratio(&self) -> f32
Current physical block pool utilization ratio [0.0 - 1.0].
Sourcepub fn fragmentation_ratio(&self) -> f32
pub fn fragmentation_ratio(&self) -> f32
Current free memory fragmentation ratio [0.0 - 1.0].
Sourcepub fn fork_sequence(&self, src_id: u64, dest_id: u64) -> Result<(), CacheError>
pub fn fork_sequence(&self, src_id: u64, dest_id: u64) -> Result<(), CacheError>
Clones and forks a sequence’s KV-cache table (e.g. for speculative branch evaluation).
Sourcepub fn free_blocks(&self) -> usize
pub fn free_blocks(&self) -> usize
Returns the number of free physical blocks in the pool.
Sourcepub fn allocated_blocks(&self) -> usize
pub fn allocated_blocks(&self) -> usize
Returns the number of allocated physical blocks.
Sourcepub fn prefix_entries(&self) -> usize
pub fn prefix_entries(&self) -> usize
Returns the number of active prefix cache entries.
Source§impl PagedKvCacheManager
impl PagedKvCacheManager
Sourcepub fn export_snapshot(&self) -> PrefixCacheSnapshot
pub fn export_snapshot(&self) -> PrefixCacheSnapshot
Exports current warm prefix cache to a snapshot.
Sourcepub fn import_snapshot(
&self,
snapshot: &PrefixCacheSnapshot,
) -> Result<usize, CacheError>
pub fn import_snapshot( &self, snapshot: &PrefixCacheSnapshot, ) -> Result<usize, CacheError>
Imports a prefix cache snapshot into the active pool.
Sourcepub fn save_snapshot_json(&self) -> Result<String, CacheError>
pub fn save_snapshot_json(&self) -> Result<String, CacheError>
Saves the warm prefix cache to a JSON string.
Sourcepub fn load_snapshot_json(&self, json_str: &str) -> Result<usize, CacheError>
pub fn load_snapshot_json(&self, json_str: &str) -> Result<usize, CacheError>
Loads and warms the cache from a JSON snapshot string.