Skip to main content

patronus_ark/cache/
write_behind.rs

1// SPDX-License-Identifier: GPL-3.0-only
2use std::sync::atomic::{AtomicU64, Ordering};
3use std::sync::mpsc::{self, Receiver, SyncSender, TryRecvError, TrySendError};
4use std::sync::{Arc, Mutex};
5use std::thread::{self, JoinHandle};
6
7use super::{CacheError, CacheKey, CachedModelOutput, ExactCacheStore, SimilarityStoreRecord};
8
9#[derive(Debug, Clone, Copy)]
10pub struct WriteBehindConfig {
11    pub queue_capacity: usize,
12    pub max_batch_size: usize,
13}
14
15impl Default for WriteBehindConfig {
16    fn default() -> Self {
17        Self {
18            queue_capacity: 1_024,
19            max_batch_size: 64,
20        }
21    }
22}
23
24#[derive(Default)]
25pub(crate) struct WriteBehindStats {
26    queued: AtomicU64,
27    written: AtomicU64,
28    dropped: AtomicU64,
29    write_errors: AtomicU64,
30}
31
32#[derive(Debug, Clone, Copy, PartialEq, Eq)]
33pub(crate) struct WriteBehindStatsSnapshot {
34    pub(crate) queued: u64,
35    pub(crate) written: u64,
36    pub(crate) dropped: u64,
37    pub(crate) write_errors: u64,
38}
39
40impl WriteBehindStats {
41    pub(crate) fn snapshot(&self) -> WriteBehindStatsSnapshot {
42        WriteBehindStatsSnapshot {
43            queued: self.queued.load(Ordering::Relaxed),
44            written: self.written.load(Ordering::Relaxed),
45            dropped: self.dropped.load(Ordering::Relaxed),
46            write_errors: self.write_errors.load(Ordering::Relaxed),
47        }
48    }
49}
50
51enum Command {
52    Put(CacheKey, CachedModelOutput),
53    PutSimilarity(SimilarityStoreRecord),
54    Flush(mpsc::Sender<Result<(), CacheError>>),
55    Shutdown,
56}
57
58pub(crate) struct WriteBehindStore {
59    inner: Arc<dyn ExactCacheStore>,
60    sender: SyncSender<Command>,
61    worker: Mutex<Option<JoinHandle<()>>>,
62    stats: Arc<WriteBehindStats>,
63}
64
65impl WriteBehindStore {
66    pub(crate) fn new(inner: Arc<dyn ExactCacheStore>, config: WriteBehindConfig) -> Self {
67        let (sender, receiver) = mpsc::sync_channel(config.queue_capacity.max(1));
68        let stats = Arc::new(WriteBehindStats::default());
69        let worker_inner = Arc::clone(&inner);
70        let worker_stats = Arc::clone(&stats);
71        let max_batch_size = config.max_batch_size.max(1);
72        let worker = thread::Builder::new()
73            .name("patronus-cache-writer".to_string())
74            .spawn(move || {
75                writer_loop(worker_inner, receiver, worker_stats, max_batch_size);
76            })
77            .expect("failed to start cache writer thread");
78        Self {
79            inner,
80            sender,
81            worker: Mutex::new(Some(worker)),
82            stats,
83        }
84    }
85
86    pub(crate) fn stats(&self) -> Arc<WriteBehindStats> {
87        Arc::clone(&self.stats)
88    }
89}
90
91impl ExactCacheStore for WriteBehindStore {
92    fn get(
93        &self,
94        key: &CacheKey,
95        now_unix_ms: u64,
96    ) -> Result<Option<CachedModelOutput>, CacheError> {
97        self.inner.get(key, now_unix_ms)
98    }
99
100    fn put(&self, key: CacheKey, value: CachedModelOutput) -> Result<(), CacheError> {
101        match self.sender.try_send(Command::Put(key, value)) {
102            Ok(()) => {
103                self.stats.queued.fetch_add(1, Ordering::Relaxed);
104                Ok(())
105            }
106            Err(TrySendError::Full(_)) => {
107                self.stats.dropped.fetch_add(1, Ordering::Relaxed);
108                Ok(())
109            }
110            Err(TrySendError::Disconnected(_)) => Err(CacheError::Storage(
111                "cache writer thread is not running".to_string(),
112            )),
113        }
114    }
115
116    fn put_batch(&self, entries: Vec<(CacheKey, CachedModelOutput)>) -> Result<(), CacheError> {
117        for (key, value) in entries {
118            self.put(key, value)?;
119        }
120        Ok(())
121    }
122
123    fn remove_expired(&self, now_unix_ms: u64) -> Result<usize, CacheError> {
124        self.flush()?;
125        self.inner.remove_expired(now_unix_ms)
126    }
127
128    fn remove_created_before(&self, until_unix_ms: u64) -> Result<usize, CacheError> {
129        self.flush()?;
130        self.inner.remove_created_before(until_unix_ms)
131    }
132
133    fn similarity_candidates(
134        &self,
135        bucket_keys: &[Vec<u8>],
136        now_unix_ms: u64,
137        limit: usize,
138    ) -> Result<Vec<SimilarityStoreRecord>, CacheError> {
139        self.inner
140            .similarity_candidates(bucket_keys, now_unix_ms, limit)
141    }
142
143    fn put_similarity(&self, record: SimilarityStoreRecord) -> Result<(), CacheError> {
144        match self.sender.try_send(Command::PutSimilarity(record)) {
145            Ok(()) => {
146                self.stats.queued.fetch_add(1, Ordering::Relaxed);
147                Ok(())
148            }
149            Err(TrySendError::Full(_)) => {
150                self.stats.dropped.fetch_add(1, Ordering::Relaxed);
151                Ok(())
152            }
153            Err(TrySendError::Disconnected(_)) => Err(CacheError::Storage(
154                "cache writer thread is not running".to_string(),
155            )),
156        }
157    }
158
159    fn flush(&self) -> Result<(), CacheError> {
160        let (sender, receiver) = mpsc::channel();
161        self.sender
162            .send(Command::Flush(sender))
163            .map_err(|_| CacheError::Storage("cache writer thread is not running".to_string()))?;
164        receiver.recv().map_err(|_| {
165            CacheError::Storage("cache writer thread stopped during flush".to_string())
166        })?
167    }
168}
169
170impl Drop for WriteBehindStore {
171    fn drop(&mut self) {
172        let _ = self.sender.send(Command::Shutdown);
173        if let Some(worker) = self
174            .worker
175            .lock()
176            .expect("cache writer mutex poisoned")
177            .take()
178        {
179            let _ = worker.join();
180        }
181    }
182}
183
184fn writer_loop(
185    inner: Arc<dyn ExactCacheStore>,
186    receiver: Receiver<Command>,
187    stats: Arc<WriteBehindStats>,
188    max_batch_size: usize,
189) {
190    let mut pending_error = None;
191    while let Ok(command) = receiver.recv() {
192        match command {
193            Command::Put(key, value) => {
194                let mut batch = vec![(key, value)];
195                let mut control = None;
196                while batch.len() < max_batch_size {
197                    match receiver.try_recv() {
198                        Ok(Command::Put(key, value)) => batch.push((key, value)),
199                        Ok(command) => {
200                            control = Some(command);
201                            break;
202                        }
203                        Err(TryRecvError::Empty) => break,
204                        Err(TryRecvError::Disconnected) => {
205                            control = Some(Command::Shutdown);
206                            break;
207                        }
208                    }
209                }
210                let batch_len = batch.len() as u64;
211                match inner.put_batch(batch) {
212                    Ok(()) => {
213                        stats.written.fetch_add(batch_len, Ordering::Relaxed);
214                    }
215                    Err(error) => {
216                        stats.write_errors.fetch_add(batch_len, Ordering::Relaxed);
217                        pending_error = Some(error.to_string());
218                    }
219                }
220                if process_control(control, &inner, &stats, &mut pending_error) {
221                    break;
222                }
223            }
224            Command::PutSimilarity(record) => {
225                let mut batch = vec![record];
226                let mut control = None;
227                while batch.len() < max_batch_size {
228                    match receiver.try_recv() {
229                        Ok(Command::PutSimilarity(record)) => batch.push(record),
230                        Ok(command) => {
231                            control = Some(command);
232                            break;
233                        }
234                        Err(TryRecvError::Empty) => break,
235                        Err(TryRecvError::Disconnected) => {
236                            control = Some(Command::Shutdown);
237                            break;
238                        }
239                    }
240                }
241                let batch_len = batch.len() as u64;
242                match inner.put_similarity_batch(batch) {
243                    Ok(()) => {
244                        stats.written.fetch_add(batch_len, Ordering::Relaxed);
245                    }
246                    Err(error) => {
247                        stats.write_errors.fetch_add(batch_len, Ordering::Relaxed);
248                        pending_error = Some(error.to_string());
249                    }
250                }
251                if process_control(control, &inner, &stats, &mut pending_error) {
252                    break;
253                }
254            }
255            command => {
256                if process_control(Some(command), &inner, &stats, &mut pending_error) {
257                    break;
258                }
259            }
260        }
261    }
262}
263
264fn process_control(
265    command: Option<Command>,
266    inner: &Arc<dyn ExactCacheStore>,
267    stats: &Arc<WriteBehindStats>,
268    pending_error: &mut Option<String>,
269) -> bool {
270    match command {
271        Some(Command::Flush(sender)) => {
272            let result = match pending_error.take() {
273                Some(error) => Err(CacheError::Storage(error)),
274                None => inner.flush(),
275            };
276            let _ = sender.send(result);
277            false
278        }
279        Some(Command::Shutdown) => {
280            let _ = inner.flush();
281            true
282        }
283        Some(Command::Put(key, value)) => {
284            match inner.put(key, value) {
285                Ok(()) => {
286                    stats.written.fetch_add(1, Ordering::Relaxed);
287                }
288                Err(error) => {
289                    stats.write_errors.fetch_add(1, Ordering::Relaxed);
290                    *pending_error = Some(error.to_string());
291                }
292            }
293            false
294        }
295        Some(Command::PutSimilarity(record)) => {
296            match inner.put_similarity(record) {
297                Ok(()) => {
298                    stats.written.fetch_add(1, Ordering::Relaxed);
299                }
300                Err(error) => {
301                    stats.write_errors.fetch_add(1, Ordering::Relaxed);
302                    *pending_error = Some(error.to_string());
303                }
304            }
305            false
306        }
307        None => false,
308    }
309}
310
311#[cfg(test)]
312mod tests {
313    use std::sync::atomic::{AtomicUsize, Ordering};
314    use std::sync::{Condvar, Mutex};
315    use std::time::{SystemTime, UNIX_EPOCH};
316
317    use super::*;
318    use crate::cache::{
319        CacheNamespace, CachedHeadOutput, MemoryCacheConfig, MemoryCacheStore, RedbCacheStore,
320    };
321
322    fn key(value: &str) -> CacheKey {
323        CacheKey::for_chunk(
324            CacheNamespace::from_components(1, &[b"write-behind"]),
325            value.as_bytes(),
326        )
327    }
328
329    fn output() -> CachedModelOutput {
330        CachedModelOutput {
331            schema_version: 1,
332            heads: vec![CachedHeadOutput {
333                head: "head".to_string(),
334                logits: vec![0.1, 0.9],
335            }],
336            created_at_unix_ms: 10,
337            expires_at_unix_ms: 100,
338        }
339    }
340
341    #[test]
342    fn flush_makes_queued_redb_writes_visible_and_durable() {
343        let unique = SystemTime::now()
344            .duration_since(UNIX_EPOCH)
345            .unwrap()
346            .as_nanos();
347        let path = std::env::temp_dir().join(format!("patronus-write-behind-{unique}.redb"));
348        {
349            let redb = Arc::new(RedbCacheStore::open(&path).unwrap());
350            let store = WriteBehindStore::new(redb, WriteBehindConfig::default());
351            store.put(key("chunk"), output()).unwrap();
352            store.flush().unwrap();
353            assert_eq!(store.stats().snapshot().written, 1);
354        }
355        let reopened = RedbCacheStore::open(&path).unwrap();
356        assert!(reopened.get(&key("chunk"), 10).unwrap().is_some());
357        drop(reopened);
358        std::fs::remove_file(path).unwrap();
359    }
360
361    #[test]
362    fn writer_batches_adjacent_entries() {
363        struct BatchCountingStore {
364            memory: MemoryCacheStore,
365            largest_batch: AtomicUsize,
366            calls: AtomicUsize,
367            first_started: Arc<(Mutex<bool>, Condvar)>,
368            release_first: Arc<(Mutex<bool>, Condvar)>,
369        }
370
371        impl ExactCacheStore for BatchCountingStore {
372            fn get(
373                &self,
374                key: &CacheKey,
375                now_unix_ms: u64,
376            ) -> Result<Option<CachedModelOutput>, CacheError> {
377                self.memory.get(key, now_unix_ms)
378            }
379
380            fn put(&self, key: CacheKey, value: CachedModelOutput) -> Result<(), CacheError> {
381                self.memory.put(key, value)
382            }
383
384            fn put_batch(
385                &self,
386                entries: Vec<(CacheKey, CachedModelOutput)>,
387            ) -> Result<(), CacheError> {
388                if self.calls.fetch_add(1, Ordering::Relaxed) == 0 {
389                    let (started, started_cv) = &*self.first_started;
390                    *started.lock().unwrap() = true;
391                    started_cv.notify_one();
392                    let (release, release_cv) = &*self.release_first;
393                    let guard = release.lock().unwrap();
394                    drop(release_cv.wait_while(guard, |released| !*released).unwrap());
395                }
396                self.largest_batch
397                    .fetch_max(entries.len(), Ordering::Relaxed);
398                self.memory.put_batch(entries)
399            }
400
401            fn remove_expired(&self, now_unix_ms: u64) -> Result<usize, CacheError> {
402                self.memory.remove_expired(now_unix_ms)
403            }
404
405            fn remove_created_before(&self, until_unix_ms: u64) -> Result<usize, CacheError> {
406                self.memory.remove_created_before(until_unix_ms)
407            }
408        }
409
410        let first_started = Arc::new((Mutex::new(false), Condvar::new()));
411        let release_first = Arc::new((Mutex::new(false), Condvar::new()));
412        let inner = Arc::new(BatchCountingStore {
413            memory: MemoryCacheStore::new(MemoryCacheConfig::default()),
414            largest_batch: AtomicUsize::new(0),
415            calls: AtomicUsize::new(0),
416            first_started: Arc::clone(&first_started),
417            release_first: Arc::clone(&release_first),
418        });
419        let store = WriteBehindStore::new(
420            inner.clone(),
421            WriteBehindConfig {
422                queue_capacity: 16,
423                max_batch_size: 16,
424            },
425        );
426        store.put(key("0"), output()).unwrap();
427        let (started, started_cv) = &*first_started;
428        let guard = started.lock().unwrap();
429        drop(started_cv.wait_while(guard, |started| !*started).unwrap());
430        for index in 1..8 {
431            store.put(key(&index.to_string()), output()).unwrap();
432        }
433        let (release, release_cv) = &*release_first;
434        *release.lock().unwrap() = true;
435        release_cv.notify_one();
436        store.flush().unwrap();
437
438        assert_eq!(store.stats().snapshot().written, 8);
439        assert!(inner.largest_batch.load(Ordering::Relaxed) > 1);
440    }
441
442    #[test]
443    fn full_queue_drops_writes_without_blocking() {
444        struct BlockingStore {
445            started: Arc<(Mutex<bool>, Condvar)>,
446            release: Arc<(Mutex<bool>, Condvar)>,
447        }
448
449        impl ExactCacheStore for BlockingStore {
450            fn get(
451                &self,
452                _key: &CacheKey,
453                _now_unix_ms: u64,
454            ) -> Result<Option<CachedModelOutput>, CacheError> {
455                Ok(None)
456            }
457
458            fn put(&self, _key: CacheKey, _value: CachedModelOutput) -> Result<(), CacheError> {
459                unreachable!()
460            }
461
462            fn put_batch(
463                &self,
464                _entries: Vec<(CacheKey, CachedModelOutput)>,
465            ) -> Result<(), CacheError> {
466                let (started, started_cv) = &*self.started;
467                *started.lock().unwrap() = true;
468                started_cv.notify_one();
469                let (release, release_cv) = &*self.release;
470                let guard = release.lock().unwrap();
471                drop(release_cv.wait_while(guard, |released| !*released).unwrap());
472                Ok(())
473            }
474
475            fn remove_expired(&self, _now_unix_ms: u64) -> Result<usize, CacheError> {
476                Ok(0)
477            }
478
479            fn remove_created_before(&self, _until_unix_ms: u64) -> Result<usize, CacheError> {
480                Ok(0)
481            }
482        }
483
484        let started = Arc::new((Mutex::new(false), Condvar::new()));
485        let release = Arc::new((Mutex::new(false), Condvar::new()));
486        let inner = Arc::new(BlockingStore {
487            started: Arc::clone(&started),
488            release: Arc::clone(&release),
489        });
490        let store = WriteBehindStore::new(
491            inner,
492            WriteBehindConfig {
493                queue_capacity: 1,
494                max_batch_size: 1,
495            },
496        );
497        store.put(key("first"), output()).unwrap();
498        let (started_lock, started_cv) = &*started;
499        let guard = started_lock.lock().unwrap();
500        drop(started_cv.wait_while(guard, |started| !*started).unwrap());
501
502        store.put(key("queued"), output()).unwrap();
503        store.put(key("dropped"), output()).unwrap();
504        assert_eq!(store.stats().snapshot().dropped, 1);
505
506        let (release_lock, release_cv) = &*release;
507        *release_lock.lock().unwrap() = true;
508        release_cv.notify_one();
509        store.flush().unwrap();
510        assert_eq!(store.stats().snapshot().written, 2);
511    }
512
513    #[test]
514    fn flush_reports_background_write_errors() {
515        struct FailingStore;
516
517        impl ExactCacheStore for FailingStore {
518            fn get(
519                &self,
520                _key: &CacheKey,
521                _now_unix_ms: u64,
522            ) -> Result<Option<CachedModelOutput>, CacheError> {
523                Ok(None)
524            }
525
526            fn put(&self, _key: CacheKey, _value: CachedModelOutput) -> Result<(), CacheError> {
527                Err(CacheError::Storage("write failed".to_string()))
528            }
529
530            fn remove_expired(&self, _now_unix_ms: u64) -> Result<usize, CacheError> {
531                Ok(0)
532            }
533
534            fn remove_created_before(&self, _until_unix_ms: u64) -> Result<usize, CacheError> {
535                Ok(0)
536            }
537        }
538
539        let store = WriteBehindStore::new(Arc::new(FailingStore), WriteBehindConfig::default());
540        store.put(key("chunk"), output()).unwrap();
541
542        assert!(store.flush().is_err());
543        assert_eq!(store.stats().snapshot().write_errors, 1);
544    }
545}