Skip to main content

spec_stream_cache/
manager.rs

1use crate::allocator::BlockAllocator;
2use crate::block::DEFAULT_BLOCK_SIZE;
3use crate::error::CacheError;
4use crate::eviction::{EvictionPolicy, LruEvictionPolicy};
5use crate::table::BlockTable;
6use parking_lot::Mutex;
7use spec_stream_core::TokenId;
8use std::collections::HashMap;
9
10/// Thread-safe high-level manager orchestrating KV-caches across multiple concurrent inference sequences.
11#[derive(Debug)]
12pub struct PagedKvCacheManager {
13    pub(crate) allocator: Mutex<BlockAllocator>,
14    tables: Mutex<HashMap<u64, BlockTable>>,
15    eviction: Mutex<LruEvictionPolicy>,
16    block_size: usize,
17}
18
19impl PagedKvCacheManager {
20    pub fn new(total_blocks: usize, block_size: usize) -> Self {
21        Self {
22            allocator: Mutex::new(BlockAllocator::new(total_blocks, block_size)),
23            tables: Mutex::new(HashMap::new()),
24            eviction: Mutex::new(LruEvictionPolicy::new()),
25            block_size,
26        }
27    }
28
29    pub fn with_defaults(total_blocks: usize) -> Self {
30        Self::new(total_blocks, DEFAULT_BLOCK_SIZE)
31    }
32
33    /// Registers a new sequence/request with the cache manager.
34    pub fn create_sequence(&self, seq_id: u64) {
35        let mut tables = self.tables.lock();
36        tables.insert(seq_id, BlockTable::new(self.block_size));
37        self.eviction.lock().record_access(seq_id);
38    }
39
40    /// Appends tokens to an active sequence.
41    pub fn append_tokens(&self, seq_id: u64, tokens: &[TokenId]) -> Result<(), CacheError> {
42        let mut tables = self.tables.lock();
43        let table = tables
44            .get_mut(&seq_id)
45            .ok_or(CacheError::SequenceNotFound(seq_id))?;
46
47        let mut allocator = self.allocator.lock();
48        table.append_tokens(tokens, &mut allocator)?;
49        self.eviction.lock().record_access(seq_id);
50        Ok(())
51    }
52
53    /// Executes an O(1) rollback on the sequence when speculative tokens are rejected.
54    pub fn rollback_sequence(
55        &self,
56        seq_id: u64,
57        valid_token_count: usize,
58    ) -> Result<usize, CacheError> {
59        let mut tables = self.tables.lock();
60        let table = tables
61            .get_mut(&seq_id)
62            .ok_or(CacheError::SequenceNotFound(seq_id))?;
63
64        let mut allocator = self.allocator.lock();
65        let res = table.rollback(valid_token_count, &mut allocator);
66        self.eviction.lock().record_access(seq_id);
67        res
68    }
69
70    /// Releases all KV-cache memory held by a sequence when the request finishes.
71    pub fn remove_sequence(&self, seq_id: u64) -> Result<(), CacheError> {
72        self.eviction.lock().remove_sequence(seq_id);
73        let mut tables = self.tables.lock();
74        if let Some(mut table) = tables.remove(&seq_id) {
75            let mut allocator = self.allocator.lock();
76            table.free_all(&mut allocator)?;
77        }
78        Ok(())
79    }
80
81    /// Evicts the least recently used sequence to reclaim blocks under memory pressure.
82    /// Returns the sequence ID of the evicted sequence, if any.
83    pub fn evict_lru_sequence(&self) -> Result<Option<u64>, CacheError> {
84        let candidate = self.eviction.lock().select_eviction_candidate();
85        if let Some(seq_id) = candidate {
86            self.remove_sequence(seq_id)?;
87            Ok(Some(seq_id))
88        } else {
89            Ok(None)
90        }
91    }
92
93    /// Defragments the free physical block list to optimize spatial memory cache locality.
94    pub fn defragment(&self) {
95        self.allocator.lock().defragment_free_list();
96    }
97
98    /// Current physical block pool utilization ratio [0.0 - 1.0].
99    pub fn utilization_ratio(&self) -> f32 {
100        self.allocator.lock().utilization_ratio()
101    }
102
103    /// Current free memory fragmentation ratio [0.0 - 1.0].
104    pub fn fragmentation_ratio(&self) -> f32 {
105        self.allocator.lock().fragmentation_ratio()
106    }
107
108    /// Clones and forks a sequence's KV-cache table (e.g. for speculative branch evaluation).
109    pub fn fork_sequence(&self, src_id: u64, dest_id: u64) -> Result<(), CacheError> {
110        let mut tables = self.tables.lock();
111        let src_table = tables
112            .get(&src_id)
113            .ok_or(CacheError::SequenceNotFound(src_id))?
114            .clone();
115
116        let mut allocator = self.allocator.lock();
117        // Increment ref count for shared blocks (Copy-on-Write)
118        for &block_id in &src_table.physical_blocks {
119            allocator.inc_ref(block_id)?;
120        }
121
122        tables.insert(dest_id, src_table);
123        Ok(())
124    }
125
126    /// Returns the number of free physical blocks in the pool.
127    pub fn free_blocks(&self) -> usize {
128        self.allocator.lock().num_free_blocks()
129    }
130
131    /// Returns the number of allocated physical blocks.
132    pub fn allocated_blocks(&self) -> usize {
133        self.allocator.lock().num_allocated_blocks()
134    }
135
136    /// Returns the number of active prefix cache entries.
137    pub fn prefix_entries(&self) -> usize {
138        self.allocator.lock().num_prefix_entries()
139    }
140}