Struct KVCacheManager 

pub struct KVCacheManager {
    pub num_prefix_cache_hits: u64,
    pub num_prefix_cache_misses: u64,
    pub hit_size_count: u64,
    pub hit_size_sum: u64,
    /* private fields */
}

Manages KV cache blocks for all requests

Fields

num_prefix_cache_hits: u64

Metrics

num_prefix_cache_misses: u64

hit_size_count: u64

hit_size_sum: u64

Implementations

impl KVCacheManager

pub fn new( kv_cache_capacity: u64, block_size: u32, kv_cache_bytes_per_token: u64, enable_prefix_caching: bool, ) -> Self

Create a new KV cache manager

pub fn allocate_blocks( &mut self, request: &Request, num_tokens: u32, ) -> Option<Vec<BlockId>>

Try to allocate blocks for a request Returns Some(Vec) if successful, None if insufficient blocks

pub fn free_blocks(&mut self, block_ids: &[BlockId])

Free blocks from a request (due to preemption or completion)

pub fn num_free_blocks(&self) -> usize

Get number of free blocks

pub fn total_blocks(&self) -> usize

Get total number of blocks

pub fn utilization(&self) -> f64

Get cache utilization (0.0 to 1.0)

pub fn peek_prefix_cache(&mut self, request: &Request) -> u32

Check for prefix cache hits Returns the number of tokens that can be served from the cache

pub fn query_prefix_cache(&mut self, request: &Request) -> u32