use std::fs::{self, File, OpenOptions};
use std::io::{self, Read, Seek, SeekFrom, Write};
use std::path::{Path, PathBuf};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
const MAGIC: &[u8; 4] = b"SBF1";
const HEADER_LEN: u64 = 28;
const DEFAULT_SEGMENT_BYTES: u64 = 64 * 1024 * 1024;
const DEFAULT_CAP_BYTES: u64 = 64 * 1024 * 1024;
#[derive(Debug)]
pub enum SpillError {
Io(io::Error),
CapFull { depth_bytes: u64, cap_bytes: u64 },
Corrupt { path: PathBuf, reason: String },
}
impl std::fmt::Display for SpillError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Io(error) => write!(f, "spillbuf io: {error}"),
Self::CapFull {
depth_bytes,
cap_bytes,
} => write!(f, "spillbuf cap full: {depth_bytes} >= {cap_bytes}"),
Self::Corrupt { path, reason } => {
write!(f, "spillbuf corrupt {}: {reason}", path.display())
}
}
}
}
impl std::error::Error for SpillError {}
impl From<io::Error> for SpillError {
fn from(error: io::Error) -> Self {
Self::Io(error)
}
}
pub type Result<T> = std::result::Result<T, SpillError>;
#[derive(Clone, Debug)]
pub struct Options {
pub segment_bytes: u64,
pub cap_bytes: u64,
}
impl Default for Options {
fn default() -> Self {
Self {
segment_bytes: DEFAULT_SEGMENT_BYTES,
cap_bytes: DEFAULT_CAP_BYTES,
}
}
}
impl Options {
pub fn from_env() -> Self {
Self {
segment_bytes: env_u64("SPILLBUF_SEGMENT_BYTES", DEFAULT_SEGMENT_BYTES).max(64),
cap_bytes: env_u64("SPILLBUF_CAP_BYTES", DEFAULT_CAP_BYTES).max(64),
}
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Frame {
pub seq: u64,
pub unix_ms: u64,
pub bytes: Vec<u8>,
}
#[derive(Clone, Debug)]
struct IndexEntry {
seq: u64,
unix_ms: u64,
segment: u64,
offset: u64,
payload_len: u32,
}
pub struct SpillBuf {
dir: PathBuf,
opts: Options,
pending: Vec<IndexEntry>,
next_seq: u64,
ack_seq: u64,
current_segment: u64,
writer: File,
}
impl SpillBuf {
pub fn open(dir: impl AsRef<Path>, opts: Options) -> Result<Self> {
let dir = dir.as_ref().to_path_buf();
fs::create_dir_all(&dir)?;
let ack_seq = read_ack(&dir)?;
let (pending, next_seq, current_segment) = scan_segments(&dir, ack_seq)?;
let writer = open_segment(&dir, current_segment)?;
Ok(Self {
dir,
opts,
pending,
next_seq,
ack_seq,
current_segment,
writer,
})
}
pub fn append(&mut self, bytes: &[u8]) -> Result<u64> {
let frame_bytes = HEADER_LEN + bytes.len() as u64;
let depth_bytes = self.depth_bytes();
if depth_bytes.saturating_add(frame_bytes) > self.opts.cap_bytes {
return Err(SpillError::CapFull {
depth_bytes,
cap_bytes: self.opts.cap_bytes,
});
}
let seq = self.next_seq;
let unix_ms = now_unix_ms();
let offset = self.writer.seek(SeekFrom::End(0))?;
write_frame(&mut self.writer, seq, unix_ms, bytes)?;
self.writer.sync_data()?;
self.pending.push(IndexEntry {
seq,
unix_ms,
segment: self.current_segment,
offset,
payload_len: bytes.len() as u32,
});
self.next_seq = seq.saturating_add(1);
let len = self.writer.metadata()?.len();
if len >= self.opts.segment_bytes {
self.rotate()?;
}
Ok(seq)
}
pub fn drain_batch(&self, max: usize) -> Result<Vec<Frame>> {
let mut out = Vec::new();
for entry in self.pending.iter().take(max) {
out.push(self.read_frame(entry)?);
}
Ok(out)
}
pub fn ack_through(&mut self, watermark: u64) -> Result<()> {
if watermark < self.ack_seq {
return Ok(());
}
self.ack_seq = watermark;
persist_ack(&self.dir, self.ack_seq)?;
let keep_from = self
.pending
.iter()
.position(|entry| entry.seq > watermark)
.unwrap_or(self.pending.len());
let retired: Vec<IndexEntry> = self.pending.drain(..keep_from).collect();
let live_segments: std::collections::BTreeSet<u64> =
self.pending.iter().map(|entry| entry.segment).collect();
let retired_segments: std::collections::BTreeSet<u64> = retired
.iter()
.map(|entry| entry.segment)
.filter(|seg| !live_segments.contains(seg) && *seg != self.current_segment)
.collect();
for segment in retired_segments {
let path = segment_path(&self.dir, segment);
if path.exists() {
fs::remove_file(path)?;
}
}
Ok(())
}
pub fn depth(&self) -> u64 {
self.pending.len() as u64
}
pub fn depth_bytes(&self) -> u64 {
self.pending
.iter()
.map(|entry| HEADER_LEN + u64::from(entry.payload_len))
.sum()
}
pub fn oldest_age(&self) -> Option<Duration> {
let oldest = self.pending.first()?.unix_ms;
let now = now_unix_ms();
Some(Duration::from_millis(now.saturating_sub(oldest)))
}
pub fn oldest_unix_ms(&self) -> Option<u64> {
self.pending.first().map(|entry| entry.unix_ms)
}
pub fn next_seq(&self) -> u64 {
self.next_seq
}
pub fn ack_seq(&self) -> u64 {
self.ack_seq
}
pub fn cap_bytes(&self) -> u64 {
self.opts.cap_bytes
}
fn rotate(&mut self) -> Result<()> {
self.writer.sync_all()?;
self.current_segment = self.current_segment.saturating_add(1);
self.writer = open_segment(&self.dir, self.current_segment)?;
Ok(())
}
fn read_frame(&self, entry: &IndexEntry) -> Result<Frame> {
let path = segment_path(&self.dir, entry.segment);
let mut file = File::open(&path)?;
file.seek(SeekFrom::Start(entry.offset))?;
let mut header = [0u8; HEADER_LEN as usize];
file.read_exact(&mut header)?;
let parsed = parse_header(&header).ok_or_else(|| SpillError::Corrupt {
path: path.clone(),
reason: format!("bad header at {}", entry.offset),
})?;
let mut bytes = vec![0u8; parsed.len as usize];
file.read_exact(&mut bytes)?;
if crc32(&bytes) != parsed.crc {
return Err(SpillError::Corrupt {
path,
reason: format!("crc mismatch seq={}", parsed.seq),
});
}
Ok(Frame {
seq: parsed.seq,
unix_ms: parsed.unix_ms,
bytes,
})
}
}
struct ParsedHeader {
seq: u64,
unix_ms: u64,
len: u32,
crc: u32,
}
fn write_frame(file: &mut File, seq: u64, unix_ms: u64, bytes: &[u8]) -> io::Result<()> {
let len = bytes.len() as u32;
let crc = crc32(bytes);
file.write_all(MAGIC)?;
file.write_all(&seq.to_le_bytes())?;
file.write_all(&unix_ms.to_le_bytes())?;
file.write_all(&len.to_le_bytes())?;
file.write_all(&crc.to_le_bytes())?;
file.write_all(bytes)?;
Ok(())
}
fn parse_header(buf: &[u8]) -> Option<ParsedHeader> {
if buf.len() < HEADER_LEN as usize || &buf[0..4] != MAGIC {
return None;
}
Some(ParsedHeader {
seq: u64::from_le_bytes(buf[4..12].try_into().ok()?),
unix_ms: u64::from_le_bytes(buf[12..20].try_into().ok()?),
len: u32::from_le_bytes(buf[20..24].try_into().ok()?),
crc: u32::from_le_bytes(buf[24..28].try_into().ok()?),
})
}
fn scan_segments(dir: &Path, ack_seq: u64) -> Result<(Vec<IndexEntry>, u64, u64)> {
let mut ids = list_segments(dir)?;
if ids.is_empty() {
ids.push(0);
}
let mut pending = Vec::new();
let mut next_seq = 1.max(ack_seq.saturating_add(1));
let last_id = *ids.last().unwrap_or(&0);
for id in &ids {
let path = segment_path(dir, *id);
if !path.exists() {
continue;
}
let mut file = OpenOptions::new().read(true).write(true).open(&path)?;
let file_len = file.metadata()?.len();
let mut offset = 0u64;
loop {
if offset + HEADER_LEN > file_len {
if offset < file_len {
file.set_len(offset)?;
}
break;
}
file.seek(SeekFrom::Start(offset))?;
let mut header = [0u8; HEADER_LEN as usize];
match file.read_exact(&mut header) {
Ok(()) => {}
Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => {
file.set_len(offset)?;
break;
}
Err(error) => return Err(error.into()),
}
let Some(parsed) = parse_header(&header) else {
file.set_len(offset)?;
break;
};
let payload_end = offset + HEADER_LEN + u64::from(parsed.len);
if payload_end > file_len {
file.set_len(offset)?;
break;
}
let mut payload = vec![0u8; parsed.len as usize];
file.read_exact(&mut payload)?;
if crc32(&payload) != parsed.crc {
file.set_len(offset)?;
break;
}
if parsed.seq > ack_seq {
pending.push(IndexEntry {
seq: parsed.seq,
unix_ms: parsed.unix_ms,
segment: *id,
offset,
payload_len: parsed.len,
});
}
next_seq = next_seq.max(parsed.seq.saturating_add(1));
offset = payload_end;
}
}
pending.sort_by_key(|entry| entry.seq);
Ok((pending, next_seq, last_id))
}
fn list_segments(dir: &Path) -> Result<Vec<u64>> {
let mut ids = Vec::new();
if !dir.exists() {
return Ok(ids);
}
for entry in fs::read_dir(dir)? {
let entry = entry?;
let name = entry.file_name();
let Some(name) = name.to_str() else {
continue;
};
if let Some(rest) = name.strip_prefix("seg-") {
if let Some(stem) = rest.strip_suffix(".log") {
if let Ok(id) = stem.parse::<u64>() {
ids.push(id);
}
}
}
}
ids.sort_unstable();
Ok(ids)
}
fn open_segment(dir: &Path, id: u64) -> io::Result<File> {
OpenOptions::new()
.create(true)
.read(true)
.append(true)
.open(segment_path(dir, id))
}
fn segment_path(dir: &Path, id: u64) -> PathBuf {
dir.join(format!("seg-{id:08}.log"))
}
fn ack_path(dir: &Path) -> PathBuf {
dir.join("ack.seq")
}
fn read_ack(dir: &Path) -> Result<u64> {
let path = ack_path(dir);
if !path.exists() {
return Ok(0);
}
let raw = fs::read(&path)?;
if raw.len() < 8 {
return Err(SpillError::Corrupt {
path,
reason: "short ack.seq".into(),
});
}
Ok(u64::from_le_bytes(raw[0..8].try_into().unwrap()))
}
fn persist_ack(dir: &Path, seq: u64) -> Result<()> {
let path = ack_path(dir);
let tmp = dir.join("ack.seq.tmp");
let mut file = OpenOptions::new()
.create(true)
.write(true)
.truncate(true)
.open(&tmp)?;
file.write_all(&seq.to_le_bytes())?;
file.sync_all()?;
fs::rename(tmp, path)?;
Ok(())
}
fn now_unix_ms() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_millis() as u64)
.unwrap_or(0)
}
fn env_u64(name: &str, default: u64) -> u64 {
std::env::var(name)
.ok()
.and_then(|value| value.parse().ok())
.unwrap_or(default)
}
fn crc32(data: &[u8]) -> u32 {
let mut crc = 0xFFFF_FFFFu32;
for &byte in data {
crc ^= u32::from(byte);
for _ in 0..8 {
crc = if crc & 1 == 1 {
(crc >> 1) ^ 0xEDB8_8320
} else {
crc >> 1
};
}
}
!crc
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Write;
fn tmp() -> tempfile::TempDir {
tempfile::tempdir().expect("tempdir")
}
fn open_small(dir: &Path, segment: u64, cap: u64) -> SpillBuf {
SpillBuf::open(
dir,
Options {
segment_bytes: segment,
cap_bytes: cap,
},
)
.expect("open")
}
#[test]
fn crc32_known_vector() {
assert_eq!(crc32(b"123456789"), 0xCBF4_3926);
}
#[test]
fn append_drain_ack_round_trip() {
let dir = tmp();
let mut buf = open_small(dir.path(), 64 * 1024, 64 * 1024);
let a = buf.append(b"one").unwrap();
let b = buf.append(b"two").unwrap();
assert_eq!(a, 1);
assert_eq!(b, 2);
assert_eq!(buf.depth(), 2);
let batch = buf.drain_batch(10).unwrap();
assert_eq!(batch.len(), 2);
assert_eq!(batch[0].bytes, b"one");
assert_eq!(batch[1].bytes, b"two");
buf.ack_through(a).unwrap();
assert_eq!(buf.depth(), 1);
let rest = buf.drain_batch(10).unwrap();
assert_eq!(rest[0].bytes, b"two");
}
#[test]
fn crash_mid_append_discards_partial() {
let dir = tmp();
let mut buf = open_small(dir.path(), 64 * 1024, 64 * 1024);
buf.append(b"durable").unwrap();
drop(buf);
let seg = segment_path(dir.path(), 0);
let mut file = OpenOptions::new().append(true).open(&seg).unwrap();
file.write_all(&[1, 2, 3, 4, 5]).unwrap();
drop(file);
let buf = open_small(dir.path(), 64 * 1024, 64 * 1024);
let batch = buf.drain_batch(10).unwrap();
assert_eq!(batch.len(), 1);
assert_eq!(batch[0].bytes, b"durable");
assert_eq!(
fs::metadata(&seg).unwrap().len(),
HEADER_LEN + b"durable".len() as u64
);
}
#[test]
fn replay_preserves_oldest_first_order() {
let dir = tmp();
let mut buf = open_small(dir.path(), 64 * 1024, 64 * 1024);
buf.append(b"a").unwrap();
buf.append(b"b").unwrap();
buf.append(b"c").unwrap();
drop(buf);
let buf = open_small(dir.path(), 64 * 1024, 64 * 1024);
let batch = buf.drain_batch(10).unwrap();
let bodies: Vec<&[u8]> = batch.iter().map(|f| f.bytes.as_slice()).collect();
assert_eq!(bodies, [b"a".as_slice(), b"b", b"c"]);
assert_eq!(buf.depth(), 3);
}
#[test]
fn rotation_splits_segments_and_acks_delete_old() {
let dir = tmp();
let payload = vec![b'x'; 40];
let mut buf = open_small(dir.path(), 80, 64 * 1024);
let s1 = buf.append(&payload).unwrap();
let s2 = buf.append(&payload).unwrap();
let s3 = buf.append(&payload).unwrap();
let segs = list_segments(dir.path()).unwrap();
assert!(
segs.len() >= 2,
"expected rotation, segs={segs:?} s={s1},{s2},{s3}"
);
buf.ack_through(s2).unwrap();
let after = list_segments(dir.path()).unwrap();
assert!(
!after.contains(&0) || buf.depth() > 0,
"acked prefix should drop idle segments: {after:?}"
);
let rest = buf.drain_batch(10).unwrap();
assert_eq!(rest.len(), 1);
assert_eq!(rest[0].seq, s3);
}
#[test]
fn cap_rejects_when_full() {
let dir = tmp();
let mut buf = open_small(dir.path(), 64 * 1024, HEADER_LEN + 8);
buf.append(b"abcd").unwrap();
let err = buf.append(b"efgh").unwrap_err();
match err {
SpillError::CapFull { .. } => {}
other => panic!("expected CapFull, got {other}"),
}
assert_eq!(buf.depth(), 1);
}
#[test]
fn ack_survives_reboot_and_does_not_redeliver() {
let dir = tmp();
let mut buf = open_small(dir.path(), 64 * 1024, 64 * 1024);
let a = buf.append(b"keep-me-acked").unwrap();
buf.append(b"still-pending").unwrap();
buf.ack_through(a).unwrap();
drop(buf);
let buf = open_small(dir.path(), 64 * 1024, 64 * 1024);
let batch = buf.drain_batch(10).unwrap();
assert_eq!(batch.len(), 1);
assert_eq!(batch[0].bytes, b"still-pending");
}
#[test]
fn oldest_age_is_some_when_pending() {
let dir = tmp();
let mut buf = open_small(dir.path(), 64 * 1024, 64 * 1024);
assert!(buf.oldest_age().is_none());
buf.append(b"x").unwrap();
let age = buf.oldest_age().expect("age");
assert!(age < Duration::from_secs(5));
}
}