segment_buffer/lib.rs
1//! Durable bounded queue backed by zstd-compressed CBOR segment files.
2//!
3//! Items are accumulated in memory, flushed as zstd-compressed CBOR batches
4//! to `seg_{start:012}_{end:012}.zst` files, and deleted once the consumer
5//! acknowledges receipt via [`SegmentBuffer::delete_acked`].
6//!
7//! The buffer is generic over any `T: Serialize + DeserializeOwned + Clone + Send + 'static`.
8//! Crash recovery is filename-based: scanning the directory rebuilds `head_seq`
9//! and `next_seq` without any WAL or metadata database.
10//!
11//! # Example
12//!
13//! ```no_run
14//! use segment_buffer::{SegmentBuffer, SegmentConfig};
15//! use serde::{Serialize, Deserialize};
16//!
17//! #[derive(Serialize, Deserialize, Clone)]
18//! struct MyItem { id: u64 }
19//!
20//! let buffer = SegmentBuffer::<MyItem>::open("/tmp/my-queue", SegmentConfig::default())?;
21//! let seq = buffer.append(MyItem { id: 1 })?;
22//! let items = buffer.read_from(0, 100)?;
23//! # Ok::<(), Box<dyn std::error::Error>>(())
24//! ```
25
26#![warn(missing_docs)]
27
28mod cipher;
29mod error;
30mod segment;
31
32#[cfg(feature = "encryption")]
33pub use cipher::AesGcmCipher;
34pub use cipher::{CipherError, SegmentCipher};
35pub use error::{Result, SegmentError};
36
37use std::fs;
38use std::path::PathBuf;
39use std::time::Instant;
40
41use parking_lot::Mutex;
42use serde::de::DeserializeOwned;
43use serde::Serialize;
44use tracing::{debug, info};
45
46use segment::SegmentRange;
47
48/// Configuration knobs for [`SegmentBuffer`].
49pub struct SegmentConfig {
50 /// Max events accumulated in RAM before auto-flush (default: 256).
51 pub max_batch_events: usize,
52 /// Max seconds between flushes. An append after this interval triggers a
53 /// flush even if the batch threshold hasn't been reached (default: 5s).
54 pub flush_interval_secs: u64,
55 /// Max total disk usage before the buffer reports overload pressure (default: 10 GB).
56 pub max_size_bytes: u64,
57 /// zstd compression level (1-22; 3 is fast with a good ratio).
58 pub compression_level: i32,
59 /// Optional cipher for encrypting segment files at rest. When `None`,
60 /// segments are written as plaintext zstd+CBOR.
61 pub cipher: Option<Box<dyn SegmentCipher>>,
62}
63
64impl std::fmt::Debug for SegmentConfig {
65 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
66 f.debug_struct("SegmentConfig")
67 .field("max_batch_events", &self.max_batch_events)
68 .field("flush_interval_secs", &self.flush_interval_secs)
69 .field("max_size_bytes", &self.max_size_bytes)
70 .field("compression_level", &self.compression_level)
71 .field("cipher", &self.cipher.as_ref().map(|_| "[set]"))
72 .finish()
73 }
74}
75
76impl Default for SegmentConfig {
77 fn default() -> Self {
78 Self {
79 max_batch_events: 256,
80 flush_interval_secs: 5,
81 max_size_bytes: 10 * 1024 * 1024 * 1024,
82 compression_level: 3,
83 cipher: None,
84 }
85 }
86}
87
88/// Point-in-time snapshot of buffer state, captured atomically under a single
89/// lock acquisition so all fields are mutually consistent.
90///
91/// Returned by [`SegmentBuffer::stats`]. Useful for metrics endpoints or
92/// dashboards that need to observe multiple values without paying for several
93/// lock/unlock round-trips (and risking a torn read between calls).
94#[derive(Debug, Clone)]
95pub struct BufferStats {
96 /// Items waiting in the buffer (on-disk + in-memory pending).
97 /// Same value as [`SegmentBuffer::pending_count`].
98 pub pending_count: u64,
99 /// Highest sequence number assigned (or `0` if the buffer is empty).
100 /// Same value as [`SegmentBuffer::latest_sequence`].
101 pub latest_sequence: u64,
102 /// Oldest unacknowledged sequence number (`head_seq`).
103 pub head_sequence: u64,
104 /// Next sequence number that will be assigned by the next successful
105 /// [`SegmentBuffer::append`] (`next_seq`).
106 pub next_sequence: u64,
107 /// Approximate total bytes used by segment files on disk. Decreases when
108 /// [`SegmentBuffer::delete_acked`] removes files.
109 pub approx_disk_bytes: u64,
110 /// Configured ceiling on disk usage (`max_size_bytes`). `0` disables the
111 /// limit; in that case [`store_pressure`](Self::store_pressure) is `0.0`.
112 pub max_size_bytes: u64,
113 /// `approx_disk_bytes / max_size_bytes`, clamped to `[0.0, 1.0]`.
114 /// `0.0` when no limit is configured.
115 pub store_pressure: f32,
116}
117
118struct BufferInner<T> {
119 /// Items buffered in memory, not yet written to a segment file. Drained by
120 /// [`SegmentBuffer::flush`] and rebuilt empty on crash recovery (unflushed
121 /// items do not survive a crash by design).
122 unflushed: Vec<T>,
123 next_seq: u64,
124 head_seq: u64,
125 last_flush: Instant,
126 approx_disk_bytes: u64,
127}
128
129/// Durable bounded queue of `T` backed by compressed segment files.
130///
131/// Thread-safe via `parking_lot::Mutex`. All file I/O is synchronous. The mutex
132/// is never held across an async boundary because there are no await points.
133///
134/// Create with [`SegmentBuffer::open`], supplying the directory and config.
135pub struct SegmentBuffer<T> {
136 dir: PathBuf,
137 config: SegmentConfig,
138 inner: Mutex<BufferInner<T>>,
139}
140
141impl<T> SegmentBuffer<T>
142where
143 T: Serialize + DeserializeOwned + Clone + Send + 'static,
144{
145 /// Open (or create) a buffer at `dir`, recovering from any existing
146 /// segment files.
147 ///
148 /// Recovery is **filename-based**: it scans the directory to rebuild
149 /// `head_seq` / `next_seq` and deletes leftover `.tmp` debris. Segment
150 /// *contents* are not read until [`read_from`](Self::read_from), so a
151 /// corrupted segment does not fail here — it fails when read.
152 ///
153 /// # Example
154 ///
155 /// ```
156 /// use segment_buffer::{SegmentBuffer, SegmentConfig};
157 /// use tempfile::tempdir;
158 ///
159 /// let dir = tempdir()?;
160 /// let buf: SegmentBuffer<u64> =
161 /// SegmentBuffer::open(dir.path(), SegmentConfig::default())?;
162 /// # Ok::<(), Box<dyn std::error::Error>>(())
163 /// ```
164 ///
165 /// # Errors
166 ///
167 /// Returns [`SegmentError::Io`] if the directory cannot be created or read.
168 pub fn open(dir: impl Into<PathBuf>, config: SegmentConfig) -> Result<Self> {
169 let dir = dir.into();
170 fs::create_dir_all(&dir)?;
171
172 let buffer = Self {
173 dir,
174 config,
175 inner: Mutex::new(BufferInner {
176 unflushed: Vec::new(),
177 next_seq: 0,
178 head_seq: 0,
179 last_flush: Instant::now(),
180 approx_disk_bytes: 0,
181 }),
182 };
183
184 buffer.recover()?;
185 Ok(buffer)
186 }
187
188 // -----------------------------------------------------------------------
189 // Public API
190 // -----------------------------------------------------------------------
191
192 /// Append an item to the buffer. Assigns the next sequence number and
193 /// auto-flushes if the batch threshold or interval is reached.
194 ///
195 /// Returns the assigned sequence number. The first append returns `0`,
196 /// and the number increments by 1 for each subsequent append.
197 ///
198 /// # Example
199 ///
200 /// ```
201 /// use segment_buffer::{SegmentBuffer, SegmentConfig};
202 /// use tempfile::tempdir;
203 ///
204 /// let dir = tempdir()?;
205 /// let buf: SegmentBuffer<u64> =
206 /// SegmentBuffer::open(dir.path(), SegmentConfig::default())?;
207 ///
208 /// assert_eq!(buf.append(1)?, 0);
209 /// assert_eq!(buf.append(2)?, 1);
210 /// assert_eq!(buf.append(3)?, 2);
211 /// # Ok::<(), Box<dyn std::error::Error>>(())
212 /// ```
213 pub fn append(&self, event: T) -> Result<u64> {
214 let (should_flush, seq) = {
215 let mut inner = self.inner.lock();
216 inner.unflushed.push(event);
217 inner.next_seq += 1;
218 let seq = inner.next_seq - 1;
219
220 let batch_full = inner.unflushed.len() >= self.config.max_batch_events;
221 let interval_elapsed =
222 inner.last_flush.elapsed().as_secs() >= self.config.flush_interval_secs;
223 (batch_full || interval_elapsed, seq)
224 };
225
226 if should_flush {
227 self.flush()?;
228 }
229
230 Ok(seq)
231 }
232
233 /// Flush buffered items to a segment file. No-op if nothing is buffered.
234 ///
235 /// Flushing is also triggered automatically by [`append`](Self::append)
236 /// when the batch threshold (`max_batch_events`) or interval
237 /// (`flush_interval_secs`) is reached. Call this explicitly when you need
238 /// durability before a known threshold.
239 ///
240 /// # Example
241 ///
242 /// ```
243 /// use segment_buffer::{SegmentBuffer, SegmentConfig};
244 /// use tempfile::tempdir;
245 ///
246 /// let dir = tempdir()?;
247 /// let buf: SegmentBuffer<u64> =
248 /// SegmentBuffer::open(dir.path(), SegmentConfig::default())?;
249 /// buf.append(1)?;
250 /// buf.append(2)?;
251 ///
252 /// buf.flush()?; // items now durable on disk
253 /// assert_eq!(buf.pending_count(), 2);
254 /// # Ok::<(), Box<dyn std::error::Error>>(())
255 /// ```
256 pub fn flush(&self) -> Result<()> {
257 let (events, start_seq, end_seq) = {
258 let mut inner = self.inner.lock();
259 inner.last_flush = Instant::now();
260 if inner.unflushed.is_empty() {
261 return Ok(());
262 }
263 let events = std::mem::take(&mut inner.unflushed);
264 let count = events.len() as u64;
265 let end_seq = inner.next_seq - 1;
266 let start_seq = end_seq + 1 - count;
267 (events, start_seq, end_seq)
268 };
269
270 let compressed_len = self.write_segment(start_seq, end_seq, &events)?;
271
272 {
273 let mut inner = self.inner.lock();
274 inner.approx_disk_bytes += compressed_len;
275 }
276
277 debug!(start_seq, end_seq, count = events.len(), "Flushed segment");
278 Ok(())
279 }
280
281 /// Read up to `limit` items starting from `start_seq` (inclusive).
282 ///
283 /// Reads from both on-disk segment files and in-memory pending items.
284 /// Items are returned in ascending sequence order.
285 ///
286 /// Passing `limit = 0` returns an empty `Vec` without scanning.
287 ///
288 /// # Example
289 ///
290 /// ```
291 /// use segment_buffer::{SegmentBuffer, SegmentConfig};
292 /// use tempfile::tempdir;
293 ///
294 /// let dir = tempdir()?;
295 /// let buf: SegmentBuffer<u64> =
296 /// SegmentBuffer::open(dir.path(), SegmentConfig::default())?;
297 /// buf.append(10)?;
298 /// buf.append(20)?;
299 /// buf.append(30)?;
300 /// buf.flush()?;
301 ///
302 /// let items = buf.read_from(0, 100)?;
303 /// assert_eq!(items, vec![10, 20, 30]);
304 ///
305 /// // start_seq skips already-read items:
306 /// let tail = buf.read_from(2, 100)?;
307 /// assert_eq!(tail, vec![30]);
308 /// # Ok::<(), Box<dyn std::error::Error>>(())
309 /// ```
310 pub fn read_from(&self, start_seq: u64, limit: usize) -> Result<Vec<T>> {
311 if limit == 0 {
312 return Ok(Vec::new());
313 }
314
315 let mut result: Vec<T> = Vec::with_capacity(limit.min(1024));
316
317 // Phase 1: read from on-disk segments.
318 let segments = self.scan_segments()?;
319 for seg in &segments {
320 if result.len() >= limit {
321 break;
322 }
323 if seg.end < start_seq {
324 continue;
325 }
326
327 let events = self.read_segment(*seg)?;
328 let skip = if seg.start < start_seq {
329 (start_seq - seg.start) as usize
330 } else {
331 0
332 };
333
334 for event in events.into_iter().skip(skip) {
335 if result.len() >= limit {
336 break;
337 }
338 result.push(event);
339 }
340 }
341
342 // Phase 2: read from in-memory pending events.
343 if result.len() < limit {
344 let inner = self.inner.lock();
345 let pending_start = inner.next_seq.saturating_sub(inner.unflushed.len() as u64);
346 for (i, event) in inner.unflushed.iter().enumerate() {
347 let seq = pending_start + i as u64;
348 if seq < start_seq {
349 continue;
350 }
351 if result.len() >= limit {
352 break;
353 }
354 result.push(event.clone());
355 }
356 }
357
358 Ok(result)
359 }
360
361 /// Delete all on-disk segment files whose items are fully covered by
362 /// `acked_seq`.
363 ///
364 /// A segment is deleted when its `end_seq <= acked_seq`. Returns the number
365 /// of segment files removed.
366 ///
367 /// # Example
368 ///
369 /// ```
370 /// use segment_buffer::{SegmentBuffer, SegmentConfig};
371 /// use tempfile::tempdir;
372 ///
373 /// let dir = tempdir()?;
374 /// let buf: SegmentBuffer<u64> =
375 /// SegmentBuffer::open(dir.path(), SegmentConfig::default())?;
376 /// for i in 0..5u64 {
377 /// buf.append(i)?;
378 /// }
379 /// buf.flush()?;
380 ///
381 /// // Consumer has processed sequence 0..=4; acknowledge them:
382 /// let removed = buf.delete_acked(4)?;
383 /// assert_eq!(removed, 1); // one segment file deleted
384 /// assert_eq!(buf.pending_count(), 0);
385 /// # Ok::<(), Box<dyn std::error::Error>>(())
386 /// ```
387 ///
388 /// # Limitation
389 ///
390 /// Acknowledgement only removes **flushed** segment files. Items still held
391 /// in the in-memory pending batch have no segment file to delete, so they
392 /// remain readable (and counted by [`SegmentBuffer::pending_count`]) until
393 /// they are flushed and acknowledged in a later call. `head_seq` is clamped
394 /// so it never advances past the pending window, keeping the backlog count
395 /// honest.
396 pub fn delete_acked(&self, acked_seq: u64) -> Result<usize> {
397 let segments = self.scan_segments()?;
398 let mut deleted = 0;
399 let mut freed_bytes: u64 = 0;
400 let mut new_head = None;
401
402 for seg in &segments {
403 if seg.end <= acked_seq {
404 let path = self.segment_path(seg.start, seg.end);
405 if let Ok(meta) = fs::metadata(&path) {
406 freed_bytes += meta.len();
407 }
408 match fs::remove_file(&path) {
409 Ok(()) => {
410 deleted += 1;
411 debug!(start = seg.start, end = seg.end, "Deleted acked segment");
412 }
413 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
414 Err(e) => return Err(e.into()),
415 }
416 } else if new_head.is_none() {
417 new_head = Some(seg.start);
418 }
419 }
420
421 {
422 let mut inner = self.inner.lock();
423 inner.approx_disk_bytes = inner.approx_disk_bytes.saturating_sub(freed_bytes);
424 // `head_seq` tracks the oldest unacked sequence. Clamp it to the
425 // start of the in-memory pending window: items still waiting to be
426 // flushed cannot be acknowledged (there is no segment file to
427 // delete), so head_seq must not advance past them. Without this
428 // clamp, acknowledging past a buffer that still holds unflushed
429 // items would make `pending_count` under-report the real backlog.
430 let pending_start = inner.next_seq.saturating_sub(inner.unflushed.len() as u64);
431 inner.head_seq = new_head.unwrap_or(inner.next_seq).min(pending_start);
432 }
433
434 if deleted > 0 {
435 info!(deleted, freed_bytes, acked_seq, "Deleted acked segments");
436 }
437
438 Ok(deleted)
439 }
440
441 /// The highest sequence number assigned (or 0 if buffer is empty).
442 ///
443 /// # Example
444 ///
445 /// ```
446 /// use segment_buffer::{SegmentBuffer, SegmentConfig};
447 /// use tempfile::tempdir;
448 ///
449 /// let dir = tempdir()?;
450 /// let buf: SegmentBuffer<u64> =
451 /// SegmentBuffer::open(dir.path(), SegmentConfig::default())?;
452 ///
453 /// assert_eq!(buf.latest_sequence(), 0);
454 /// buf.append(7)?;
455 /// assert_eq!(buf.latest_sequence(), 0);
456 /// buf.append(8)?;
457 /// assert_eq!(buf.latest_sequence(), 1);
458 /// # Ok::<(), Box<dyn std::error::Error>>(())
459 /// ```
460 #[must_use = "the sequence number is meaningless if discarded"]
461 pub fn latest_sequence(&self) -> u64 {
462 let inner = self.inner.lock();
463 if inner.next_seq == 0 {
464 0
465 } else {
466 inner.next_seq - 1
467 }
468 }
469
470 /// Total items waiting in the buffer (on-disk + in-memory pending).
471 ///
472 /// Equivalent to `latest_sequence() - head_seq + 1` when non-empty, 0 when
473 /// empty. Decreases as [`delete_acked`](Self::delete_acked) removes files.
474 ///
475 /// # Example
476 ///
477 /// ```
478 /// use segment_buffer::{SegmentBuffer, SegmentConfig};
479 /// use tempfile::tempdir;
480 ///
481 /// let dir = tempdir()?;
482 /// let buf: SegmentBuffer<u64> =
483 /// SegmentBuffer::open(dir.path(), SegmentConfig::default())?;
484 ///
485 /// assert_eq!(buf.pending_count(), 0);
486 /// buf.append(1)?;
487 /// buf.append(2)?;
488 /// assert_eq!(buf.pending_count(), 2);
489 /// buf.flush()?;
490 /// assert_eq!(buf.pending_count(), 2); // still pending until acked
491 /// buf.delete_acked(1)?;
492 /// assert_eq!(buf.pending_count(), 0);
493 /// # Ok::<(), Box<dyn std::error::Error>>(())
494 /// ```
495 #[must_use = "the backlog size is meaningless if discarded"]
496 pub fn pending_count(&self) -> u64 {
497 let inner = self.inner.lock();
498 inner.next_seq.saturating_sub(inner.head_seq)
499 }
500
501 /// Standard [`len`](#method.len) alias for [`pending_count`](Self::pending_count).
502 ///
503 /// Provided so `SegmentBuffer` reads like a normal collection at the call
504 /// site (`buf.len()`, `buf.is_empty()`). Same value as `pending_count()`,
505 /// kept as `u64` because the buffer is proven beyond `usize::MAX` on
506 /// 32-bit targets (597M+ events in monitor365).
507 ///
508 /// # Example
509 ///
510 /// ```
511 /// use segment_buffer::{SegmentBuffer, SegmentConfig};
512 /// use tempfile::tempdir;
513 ///
514 /// let dir = tempdir()?;
515 /// let buf: SegmentBuffer<u64> =
516 /// SegmentBuffer::open(dir.path(), SegmentConfig::default())?;
517 /// assert!(buf.is_empty());
518 /// buf.append(7)?;
519 /// assert_eq!(buf.len(), 1);
520 /// assert!(!buf.is_empty());
521 /// # Ok::<(), Box<dyn std::error::Error>>(())
522 /// ```
523 #[must_use = "the backlog size is meaningless if discarded"]
524 pub fn len(&self) -> u64 {
525 self.pending_count()
526 }
527
528 /// `true` when there are no items waiting in the buffer (on-disk or
529 /// in-memory). Equivalent to `pending_count() == 0`.
530 ///
531 /// # Example
532 ///
533 /// ```
534 /// use segment_buffer::{SegmentBuffer, SegmentConfig};
535 /// use tempfile::tempdir;
536 ///
537 /// let dir = tempdir()?;
538 /// let buf: SegmentBuffer<u64> =
539 /// SegmentBuffer::open(dir.path(), SegmentConfig::default())?;
540 /// assert!(buf.is_empty());
541 /// # Ok::<(), Box<dyn std::error::Error>>(())
542 /// ```
543 #[must_use = "the emptiness flag is meaningless if discarded"]
544 pub fn is_empty(&self) -> bool {
545 self.pending_count() == 0
546 }
547
548 /// Disk usage pressure as a value between 0.0 and 1.0.
549 ///
550 /// Use this to implement your own admission/backpressure policy (e.g.
551 /// reject low-priority items above 0.90, reject standard items above 0.95).
552 /// Returns 0.0 when `max_size_bytes == 0` (limit disabled).
553 ///
554 /// # Example
555 ///
556 /// ```
557 /// use segment_buffer::{SegmentBuffer, SegmentConfig};
558 /// use tempfile::tempdir;
559 ///
560 /// let dir = tempdir()?;
561 /// let mut cfg = SegmentConfig::default();
562 /// cfg.max_size_bytes = 1000; // tiny limit so pressure is observable
563 /// let buf: SegmentBuffer<u64> = SegmentBuffer::open(dir.path(), cfg)?;
564 ///
565 /// assert!(buf.store_pressure() < 0.1);
566 /// # Ok::<(), Box<dyn std::error::Error>>(())
567 /// ```
568 #[must_use = "the pressure value is meaningless if discarded"]
569 pub fn store_pressure(&self) -> f32 {
570 let inner = self.inner.lock();
571 if self.config.max_size_bytes == 0 {
572 return 0.0;
573 }
574 (inner.approx_disk_bytes as f32 / self.config.max_size_bytes as f32).min(1.0)
575 }
576
577 /// True when disk usage exceeds 90% of the configured limit.
578 ///
579 /// Convenience wrapper around `store_pressure() > 0.9`.
580 ///
581 /// # Example
582 ///
583 /// ```
584 /// use segment_buffer::{SegmentBuffer, SegmentConfig};
585 /// use tempfile::tempdir;
586 ///
587 /// let dir = tempdir()?;
588 /// let buf: SegmentBuffer<u64> =
589 /// SegmentBuffer::open(dir.path(), SegmentConfig::default())?;
590 ///
591 /// assert!(!buf.is_overloaded());
592 /// # Ok::<(), Box<dyn std::error::Error>>(())
593 /// ```
594 #[must_use = "the overload flag is meaningless if discarded"]
595 pub fn is_overloaded(&self) -> bool {
596 self.store_pressure() > 0.9
597 }
598
599 /// Capture a consistent snapshot of buffer state under a single lock.
600 ///
601 /// Cheaper and more consistent than calling
602 /// [`pending_count`](Self::pending_count),
603 /// [`latest_sequence`](Self::latest_sequence),
604 /// [`store_pressure`](Self::store_pressure) etc. individually (which each
605 /// take the mutex and could observe a flush/delete between calls).
606 ///
607 /// # Example
608 ///
609 /// ```
610 /// use segment_buffer::{SegmentBuffer, SegmentConfig};
611 /// use tempfile::tempdir;
612 ///
613 /// let dir = tempdir()?;
614 /// let buf: SegmentBuffer<u64> =
615 /// SegmentBuffer::open(dir.path(), SegmentConfig::default())?;
616 /// buf.append(1)?;
617 /// buf.append(2)?;
618 ///
619 /// let snapshot = buf.stats();
620 /// assert_eq!(snapshot.pending_count, 2);
621 /// assert_eq!(snapshot.next_sequence, 2);
622 /// assert!(snapshot.store_pressure < 0.01);
623 /// # Ok::<(), Box<dyn std::error::Error>>(())
624 /// ```
625 #[must_use = "the snapshot is meaningless if discarded"]
626 pub fn stats(&self) -> BufferStats {
627 let inner = self.inner.lock();
628 let pending_count = inner.next_seq.saturating_sub(inner.head_seq);
629 let latest_sequence = if inner.next_seq == 0 {
630 0
631 } else {
632 inner.next_seq - 1
633 };
634 let store_pressure = if self.config.max_size_bytes == 0 {
635 0.0
636 } else {
637 (inner.approx_disk_bytes as f32 / self.config.max_size_bytes as f32).min(1.0)
638 };
639 BufferStats {
640 pending_count,
641 latest_sequence,
642 head_sequence: inner.head_seq,
643 next_sequence: inner.next_seq,
644 approx_disk_bytes: inner.approx_disk_bytes,
645 max_size_bytes: self.config.max_size_bytes,
646 store_pressure,
647 }
648 }
649
650 // -----------------------------------------------------------------------
651 // Internal helpers
652 // -----------------------------------------------------------------------
653
654 fn recover(&self) -> Result<()> {
655 segment::clean_tmp(&self.dir)?;
656
657 let segments = self.scan_segments()?;
658
659 // All filesystem access (stat'ing each segment for its size) happens
660 // BEFORE the mutex is taken. The lock is held only long enough to
661 // publish the rebuilt in-memory state, honouring the invariant that
662 // the mutex is never held across file I/O.
663 let total_bytes: u64 = segments
664 .iter()
665 .filter_map(|s| fs::metadata(self.segment_path(s.start, s.end)).ok())
666 .map(|m| m.len())
667 .sum();
668
669 let (head_seq, next_seq) = match (segments.first(), segments.last()) {
670 (Some(first), Some(last)) => (first.start, last.end + 1),
671 _ => (0, 0),
672 };
673
674 let segment_count = segments.len();
675 {
676 let mut inner = self.inner.lock();
677 inner.head_seq = head_seq;
678 inner.next_seq = next_seq;
679 inner.approx_disk_bytes = total_bytes;
680 }
681
682 info!(
683 segments = segment_count,
684 head_seq,
685 next_seq,
686 disk_bytes = total_bytes,
687 "Segment buffer recovered"
688 );
689
690 Ok(())
691 }
692
693 fn write_segment(&self, start: u64, end: u64, events: &[T]) -> Result<u64> {
694 segment::write(
695 &self.dir,
696 self.config.cipher.as_deref(),
697 self.config.compression_level,
698 SegmentRange::new(start, end),
699 events,
700 )
701 }
702
703 fn read_segment(&self, seg: SegmentRange) -> Result<Vec<T>> {
704 segment::read(&self.dir, self.config.cipher.as_deref(), seg)
705 }
706
707 fn scan_segments(&self) -> Result<Vec<SegmentRange>> {
708 segment::scan(&self.dir)
709 }
710
711 fn segment_path(&self, start: u64, end: u64) -> PathBuf {
712 self.dir.join(segment::filename(start, end))
713 }
714}
715
716// ---------------------------------------------------------------------------
717// Static thread-safety assertion
718// ---------------------------------------------------------------------------
719
720// `SegmentBuffer<T>` is documented as MPMC-safe via `parking_lot::Mutex`. This
721// fails to compile if anyone ever introduces a non-`Send`/`Sync` field on
722// `SegmentBuffer` or `BufferInner` (e.g. an `Rc`), turning the documented
723// thread-safety guarantee into a compile-time contract instead of a comment.
724const _: () = {
725 const fn assert_send_sync<T: Send + Sync>() {}
726 assert_send_sync::<SegmentBuffer<()>>();
727};
728
729#[cfg(test)]
730mod tests;
731
732#[cfg(test)]
733mod property_tests;