1use std::sync::atomic::{AtomicU64, Ordering};
3use std::sync::mpsc::{self, Receiver, SyncSender, TryRecvError, TrySendError};
4use std::sync::{Arc, Mutex};
5use std::thread::{self, JoinHandle};
6
7use super::{CacheError, CacheKey, CachedModelOutput, ExactCacheStore, SimilarityStoreRecord};
8
9#[derive(Debug, Clone, Copy)]
10pub struct WriteBehindConfig {
11 pub queue_capacity: usize,
12 pub max_batch_size: usize,
13}
14
15impl Default for WriteBehindConfig {
16 fn default() -> Self {
17 Self {
18 queue_capacity: 1_024,
19 max_batch_size: 64,
20 }
21 }
22}
23
24#[derive(Default)]
25pub(crate) struct WriteBehindStats {
26 queued: AtomicU64,
27 written: AtomicU64,
28 dropped: AtomicU64,
29 write_errors: AtomicU64,
30}
31
32#[derive(Debug, Clone, Copy, PartialEq, Eq)]
33pub(crate) struct WriteBehindStatsSnapshot {
34 pub(crate) queued: u64,
35 pub(crate) written: u64,
36 pub(crate) dropped: u64,
37 pub(crate) write_errors: u64,
38}
39
40impl WriteBehindStats {
41 pub(crate) fn snapshot(&self) -> WriteBehindStatsSnapshot {
42 WriteBehindStatsSnapshot {
43 queued: self.queued.load(Ordering::Relaxed),
44 written: self.written.load(Ordering::Relaxed),
45 dropped: self.dropped.load(Ordering::Relaxed),
46 write_errors: self.write_errors.load(Ordering::Relaxed),
47 }
48 }
49}
50
51enum Command {
52 Put(CacheKey, CachedModelOutput),
53 PutSimilarity(SimilarityStoreRecord),
54 Flush(mpsc::Sender<Result<(), CacheError>>),
55 Shutdown,
56}
57
58pub(crate) struct WriteBehindStore {
59 inner: Arc<dyn ExactCacheStore>,
60 sender: SyncSender<Command>,
61 worker: Mutex<Option<JoinHandle<()>>>,
62 stats: Arc<WriteBehindStats>,
63}
64
65impl WriteBehindStore {
66 pub(crate) fn new(inner: Arc<dyn ExactCacheStore>, config: WriteBehindConfig) -> Self {
67 let (sender, receiver) = mpsc::sync_channel(config.queue_capacity.max(1));
68 let stats = Arc::new(WriteBehindStats::default());
69 let worker_inner = Arc::clone(&inner);
70 let worker_stats = Arc::clone(&stats);
71 let max_batch_size = config.max_batch_size.max(1);
72 let worker = thread::Builder::new()
73 .name("patronus-cache-writer".to_string())
74 .spawn(move || {
75 writer_loop(worker_inner, receiver, worker_stats, max_batch_size);
76 })
77 .expect("failed to start cache writer thread");
78 Self {
79 inner,
80 sender,
81 worker: Mutex::new(Some(worker)),
82 stats,
83 }
84 }
85
86 pub(crate) fn stats(&self) -> Arc<WriteBehindStats> {
87 Arc::clone(&self.stats)
88 }
89}
90
91impl ExactCacheStore for WriteBehindStore {
92 fn get(
93 &self,
94 key: &CacheKey,
95 now_unix_ms: u64,
96 ) -> Result<Option<CachedModelOutput>, CacheError> {
97 self.inner.get(key, now_unix_ms)
98 }
99
100 fn put(&self, key: CacheKey, value: CachedModelOutput) -> Result<(), CacheError> {
101 match self.sender.try_send(Command::Put(key, value)) {
102 Ok(()) => {
103 self.stats.queued.fetch_add(1, Ordering::Relaxed);
104 Ok(())
105 }
106 Err(TrySendError::Full(_)) => {
107 self.stats.dropped.fetch_add(1, Ordering::Relaxed);
108 Ok(())
109 }
110 Err(TrySendError::Disconnected(_)) => Err(CacheError::Storage(
111 "cache writer thread is not running".to_string(),
112 )),
113 }
114 }
115
116 fn put_batch(&self, entries: Vec<(CacheKey, CachedModelOutput)>) -> Result<(), CacheError> {
117 for (key, value) in entries {
118 self.put(key, value)?;
119 }
120 Ok(())
121 }
122
123 fn remove_expired(&self, now_unix_ms: u64) -> Result<usize, CacheError> {
124 self.flush()?;
125 self.inner.remove_expired(now_unix_ms)
126 }
127
128 fn remove_created_before(&self, until_unix_ms: u64) -> Result<usize, CacheError> {
129 self.flush()?;
130 self.inner.remove_created_before(until_unix_ms)
131 }
132
133 fn similarity_candidates(
134 &self,
135 bucket_keys: &[Vec<u8>],
136 now_unix_ms: u64,
137 limit: usize,
138 ) -> Result<Vec<SimilarityStoreRecord>, CacheError> {
139 self.inner
140 .similarity_candidates(bucket_keys, now_unix_ms, limit)
141 }
142
143 fn put_similarity(&self, record: SimilarityStoreRecord) -> Result<(), CacheError> {
144 match self.sender.try_send(Command::PutSimilarity(record)) {
145 Ok(()) => {
146 self.stats.queued.fetch_add(1, Ordering::Relaxed);
147 Ok(())
148 }
149 Err(TrySendError::Full(_)) => {
150 self.stats.dropped.fetch_add(1, Ordering::Relaxed);
151 Ok(())
152 }
153 Err(TrySendError::Disconnected(_)) => Err(CacheError::Storage(
154 "cache writer thread is not running".to_string(),
155 )),
156 }
157 }
158
159 fn flush(&self) -> Result<(), CacheError> {
160 let (sender, receiver) = mpsc::channel();
161 self.sender
162 .send(Command::Flush(sender))
163 .map_err(|_| CacheError::Storage("cache writer thread is not running".to_string()))?;
164 receiver.recv().map_err(|_| {
165 CacheError::Storage("cache writer thread stopped during flush".to_string())
166 })?
167 }
168}
169
170impl Drop for WriteBehindStore {
171 fn drop(&mut self) {
172 let _ = self.sender.send(Command::Shutdown);
173 if let Some(worker) = self
174 .worker
175 .lock()
176 .expect("cache writer mutex poisoned")
177 .take()
178 {
179 let _ = worker.join();
180 }
181 }
182}
183
184fn writer_loop(
185 inner: Arc<dyn ExactCacheStore>,
186 receiver: Receiver<Command>,
187 stats: Arc<WriteBehindStats>,
188 max_batch_size: usize,
189) {
190 let mut pending_error = None;
191 while let Ok(command) = receiver.recv() {
192 match command {
193 Command::Put(key, value) => {
194 let mut batch = vec![(key, value)];
195 let mut control = None;
196 while batch.len() < max_batch_size {
197 match receiver.try_recv() {
198 Ok(Command::Put(key, value)) => batch.push((key, value)),
199 Ok(command) => {
200 control = Some(command);
201 break;
202 }
203 Err(TryRecvError::Empty) => break,
204 Err(TryRecvError::Disconnected) => {
205 control = Some(Command::Shutdown);
206 break;
207 }
208 }
209 }
210 let batch_len = batch.len() as u64;
211 match inner.put_batch(batch) {
212 Ok(()) => {
213 stats.written.fetch_add(batch_len, Ordering::Relaxed);
214 }
215 Err(error) => {
216 stats.write_errors.fetch_add(batch_len, Ordering::Relaxed);
217 pending_error = Some(error.to_string());
218 }
219 }
220 if process_control(control, &inner, &stats, &mut pending_error) {
221 break;
222 }
223 }
224 Command::PutSimilarity(record) => {
225 let mut batch = vec![record];
226 let mut control = None;
227 while batch.len() < max_batch_size {
228 match receiver.try_recv() {
229 Ok(Command::PutSimilarity(record)) => batch.push(record),
230 Ok(command) => {
231 control = Some(command);
232 break;
233 }
234 Err(TryRecvError::Empty) => break,
235 Err(TryRecvError::Disconnected) => {
236 control = Some(Command::Shutdown);
237 break;
238 }
239 }
240 }
241 let batch_len = batch.len() as u64;
242 match inner.put_similarity_batch(batch) {
243 Ok(()) => {
244 stats.written.fetch_add(batch_len, Ordering::Relaxed);
245 }
246 Err(error) => {
247 stats.write_errors.fetch_add(batch_len, Ordering::Relaxed);
248 pending_error = Some(error.to_string());
249 }
250 }
251 if process_control(control, &inner, &stats, &mut pending_error) {
252 break;
253 }
254 }
255 command => {
256 if process_control(Some(command), &inner, &stats, &mut pending_error) {
257 break;
258 }
259 }
260 }
261 }
262}
263
264fn process_control(
265 command: Option<Command>,
266 inner: &Arc<dyn ExactCacheStore>,
267 stats: &Arc<WriteBehindStats>,
268 pending_error: &mut Option<String>,
269) -> bool {
270 match command {
271 Some(Command::Flush(sender)) => {
272 let result = match pending_error.take() {
273 Some(error) => Err(CacheError::Storage(error)),
274 None => inner.flush(),
275 };
276 let _ = sender.send(result);
277 false
278 }
279 Some(Command::Shutdown) => {
280 let _ = inner.flush();
281 true
282 }
283 Some(Command::Put(key, value)) => {
284 match inner.put(key, value) {
285 Ok(()) => {
286 stats.written.fetch_add(1, Ordering::Relaxed);
287 }
288 Err(error) => {
289 stats.write_errors.fetch_add(1, Ordering::Relaxed);
290 *pending_error = Some(error.to_string());
291 }
292 }
293 false
294 }
295 Some(Command::PutSimilarity(record)) => {
296 match inner.put_similarity(record) {
297 Ok(()) => {
298 stats.written.fetch_add(1, Ordering::Relaxed);
299 }
300 Err(error) => {
301 stats.write_errors.fetch_add(1, Ordering::Relaxed);
302 *pending_error = Some(error.to_string());
303 }
304 }
305 false
306 }
307 None => false,
308 }
309}
310
311#[cfg(test)]
312mod tests {
313 use std::sync::atomic::{AtomicUsize, Ordering};
314 use std::sync::{Condvar, Mutex};
315 use std::time::{SystemTime, UNIX_EPOCH};
316
317 use super::*;
318 use crate::cache::{
319 CacheNamespace, CachedHeadOutput, MemoryCacheConfig, MemoryCacheStore, RedbCacheStore,
320 };
321
322 fn key(value: &str) -> CacheKey {
323 CacheKey::for_chunk(
324 CacheNamespace::from_components(1, &[b"write-behind"]),
325 value.as_bytes(),
326 )
327 }
328
329 fn output() -> CachedModelOutput {
330 CachedModelOutput {
331 schema_version: 1,
332 heads: vec![CachedHeadOutput {
333 head: "head".to_string(),
334 logits: vec![0.1, 0.9],
335 }],
336 created_at_unix_ms: 10,
337 expires_at_unix_ms: 100,
338 }
339 }
340
341 #[test]
342 fn flush_makes_queued_redb_writes_visible_and_durable() {
343 let unique = SystemTime::now()
344 .duration_since(UNIX_EPOCH)
345 .unwrap()
346 .as_nanos();
347 let path = std::env::temp_dir().join(format!("patronus-write-behind-{unique}.redb"));
348 {
349 let redb = Arc::new(RedbCacheStore::open(&path).unwrap());
350 let store = WriteBehindStore::new(redb, WriteBehindConfig::default());
351 store.put(key("chunk"), output()).unwrap();
352 store.flush().unwrap();
353 assert_eq!(store.stats().snapshot().written, 1);
354 }
355 let reopened = RedbCacheStore::open(&path).unwrap();
356 assert!(reopened.get(&key("chunk"), 10).unwrap().is_some());
357 drop(reopened);
358 std::fs::remove_file(path).unwrap();
359 }
360
361 #[test]
362 fn writer_batches_adjacent_entries() {
363 struct BatchCountingStore {
364 memory: MemoryCacheStore,
365 largest_batch: AtomicUsize,
366 calls: AtomicUsize,
367 first_started: Arc<(Mutex<bool>, Condvar)>,
368 release_first: Arc<(Mutex<bool>, Condvar)>,
369 }
370
371 impl ExactCacheStore for BatchCountingStore {
372 fn get(
373 &self,
374 key: &CacheKey,
375 now_unix_ms: u64,
376 ) -> Result<Option<CachedModelOutput>, CacheError> {
377 self.memory.get(key, now_unix_ms)
378 }
379
380 fn put(&self, key: CacheKey, value: CachedModelOutput) -> Result<(), CacheError> {
381 self.memory.put(key, value)
382 }
383
384 fn put_batch(
385 &self,
386 entries: Vec<(CacheKey, CachedModelOutput)>,
387 ) -> Result<(), CacheError> {
388 if self.calls.fetch_add(1, Ordering::Relaxed) == 0 {
389 let (started, started_cv) = &*self.first_started;
390 *started.lock().unwrap() = true;
391 started_cv.notify_one();
392 let (release, release_cv) = &*self.release_first;
393 let guard = release.lock().unwrap();
394 drop(release_cv.wait_while(guard, |released| !*released).unwrap());
395 }
396 self.largest_batch
397 .fetch_max(entries.len(), Ordering::Relaxed);
398 self.memory.put_batch(entries)
399 }
400
401 fn remove_expired(&self, now_unix_ms: u64) -> Result<usize, CacheError> {
402 self.memory.remove_expired(now_unix_ms)
403 }
404
405 fn remove_created_before(&self, until_unix_ms: u64) -> Result<usize, CacheError> {
406 self.memory.remove_created_before(until_unix_ms)
407 }
408 }
409
410 let first_started = Arc::new((Mutex::new(false), Condvar::new()));
411 let release_first = Arc::new((Mutex::new(false), Condvar::new()));
412 let inner = Arc::new(BatchCountingStore {
413 memory: MemoryCacheStore::new(MemoryCacheConfig::default()),
414 largest_batch: AtomicUsize::new(0),
415 calls: AtomicUsize::new(0),
416 first_started: Arc::clone(&first_started),
417 release_first: Arc::clone(&release_first),
418 });
419 let store = WriteBehindStore::new(
420 inner.clone(),
421 WriteBehindConfig {
422 queue_capacity: 16,
423 max_batch_size: 16,
424 },
425 );
426 store.put(key("0"), output()).unwrap();
427 let (started, started_cv) = &*first_started;
428 let guard = started.lock().unwrap();
429 drop(started_cv.wait_while(guard, |started| !*started).unwrap());
430 for index in 1..8 {
431 store.put(key(&index.to_string()), output()).unwrap();
432 }
433 let (release, release_cv) = &*release_first;
434 *release.lock().unwrap() = true;
435 release_cv.notify_one();
436 store.flush().unwrap();
437
438 assert_eq!(store.stats().snapshot().written, 8);
439 assert!(inner.largest_batch.load(Ordering::Relaxed) > 1);
440 }
441
442 #[test]
443 fn full_queue_drops_writes_without_blocking() {
444 struct BlockingStore {
445 started: Arc<(Mutex<bool>, Condvar)>,
446 release: Arc<(Mutex<bool>, Condvar)>,
447 }
448
449 impl ExactCacheStore for BlockingStore {
450 fn get(
451 &self,
452 _key: &CacheKey,
453 _now_unix_ms: u64,
454 ) -> Result<Option<CachedModelOutput>, CacheError> {
455 Ok(None)
456 }
457
458 fn put(&self, _key: CacheKey, _value: CachedModelOutput) -> Result<(), CacheError> {
459 unreachable!()
460 }
461
462 fn put_batch(
463 &self,
464 _entries: Vec<(CacheKey, CachedModelOutput)>,
465 ) -> Result<(), CacheError> {
466 let (started, started_cv) = &*self.started;
467 *started.lock().unwrap() = true;
468 started_cv.notify_one();
469 let (release, release_cv) = &*self.release;
470 let guard = release.lock().unwrap();
471 drop(release_cv.wait_while(guard, |released| !*released).unwrap());
472 Ok(())
473 }
474
475 fn remove_expired(&self, _now_unix_ms: u64) -> Result<usize, CacheError> {
476 Ok(0)
477 }
478
479 fn remove_created_before(&self, _until_unix_ms: u64) -> Result<usize, CacheError> {
480 Ok(0)
481 }
482 }
483
484 let started = Arc::new((Mutex::new(false), Condvar::new()));
485 let release = Arc::new((Mutex::new(false), Condvar::new()));
486 let inner = Arc::new(BlockingStore {
487 started: Arc::clone(&started),
488 release: Arc::clone(&release),
489 });
490 let store = WriteBehindStore::new(
491 inner,
492 WriteBehindConfig {
493 queue_capacity: 1,
494 max_batch_size: 1,
495 },
496 );
497 store.put(key("first"), output()).unwrap();
498 let (started_lock, started_cv) = &*started;
499 let guard = started_lock.lock().unwrap();
500 drop(started_cv.wait_while(guard, |started| !*started).unwrap());
501
502 store.put(key("queued"), output()).unwrap();
503 store.put(key("dropped"), output()).unwrap();
504 assert_eq!(store.stats().snapshot().dropped, 1);
505
506 let (release_lock, release_cv) = &*release;
507 *release_lock.lock().unwrap() = true;
508 release_cv.notify_one();
509 store.flush().unwrap();
510 assert_eq!(store.stats().snapshot().written, 2);
511 }
512
513 #[test]
514 fn flush_reports_background_write_errors() {
515 struct FailingStore;
516
517 impl ExactCacheStore for FailingStore {
518 fn get(
519 &self,
520 _key: &CacheKey,
521 _now_unix_ms: u64,
522 ) -> Result<Option<CachedModelOutput>, CacheError> {
523 Ok(None)
524 }
525
526 fn put(&self, _key: CacheKey, _value: CachedModelOutput) -> Result<(), CacheError> {
527 Err(CacheError::Storage("write failed".to_string()))
528 }
529
530 fn remove_expired(&self, _now_unix_ms: u64) -> Result<usize, CacheError> {
531 Ok(0)
532 }
533
534 fn remove_created_before(&self, _until_unix_ms: u64) -> Result<usize, CacheError> {
535 Ok(0)
536 }
537 }
538
539 let store = WriteBehindStore::new(Arc::new(FailingStore), WriteBehindConfig::default());
540 store.put(key("chunk"), output()).unwrap();
541
542 assert!(store.flush().is_err());
543 assert_eq!(store.stats().snapshot().write_errors, 1);
544 }
545}