use std::fs::{File, OpenOptions};
use std::io::{BufReader, Read, Seek, SeekFrom, Write};
use std::path::{Path, PathBuf};
use memfork_core::{journal, Record};
pub const MAGIC: &[u8; 8] = b"MEMFWAL\0";
pub const VERSION: u32 = 1;
pub const HEADER_LEN: u64 = 16;
const FRAME_LEN: usize = 4 + 8;
pub const MAX_RECORD_LEN: u32 = 16 * 1024 * 1024;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum FsyncPolicy {
#[default]
Always,
Interval,
Never,
}
impl FsyncPolicy {
pub fn parse(s: &str) -> Option<Self> {
match s.to_ascii_lowercase().as_str() {
"always" => Some(FsyncPolicy::Always),
"interval" => Some(FsyncPolicy::Interval),
"never" => Some(FsyncPolicy::Never),
_ => None,
}
}
pub fn as_str(self) -> &'static str {
match self {
FsyncPolicy::Always => "always",
FsyncPolicy::Interval => "interval",
FsyncPolicy::Never => "never",
}
}
}
#[derive(Debug, thiserror::Error)]
pub enum WalError {
#[error("{context}: {source}")]
Io {
context: String,
#[source]
source: std::io::Error,
},
#[error("{path} is not a MemFork write-ahead log (wrong magic bytes)")]
NotAWal {
path: PathBuf,
},
#[error(
"{path} is a MemFork write-ahead log of format version {found}; \
this build reads version {VERSION}"
)]
UnknownVersion {
path: PathBuf,
found: u32,
},
#[error("the record at offset {at} is not a valid change: {source}")]
BadRecord {
at: u64,
#[source]
source: journal::DecodeError,
},
}
fn io(context: impl Into<String>) -> impl FnOnce(std::io::Error) -> WalError {
let context = context.into();
move |source| WalError::Io { context, source }
}
#[derive(Debug)]
pub struct Replay {
pub records: Vec<Record>,
pub valid_len: u64,
pub torn_tail: Option<String>,
}
#[derive(Debug)]
pub struct Wal {
path: PathBuf,
file: File,
policy: FsyncPolicy,
appended: u64,
dirty: bool,
}
impl Wal {
pub fn open(path: &Path, policy: FsyncPolicy) -> Result<Self, WalError> {
let mut file = OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(false)
.open(path)
.map_err(io(format!("cannot open {}", path.display())))?;
let len = file
.metadata()
.map_err(io(format!("cannot inspect {}", path.display())))?
.len();
if len == 0 {
write_header(&mut file, path)?;
} else {
read_header(&mut file, path)?;
}
file.seek(SeekFrom::End(0))
.map_err(io(format!("cannot seek to the end of {}", path.display())))?;
Ok(Wal {
path: path.to_path_buf(),
file,
policy,
appended: 0,
dirty: false,
})
}
pub fn path(&self) -> &Path {
&self.path
}
pub fn appended(&self) -> u64 {
self.appended
}
pub fn append(&mut self, record: &Record) -> Result<(), WalError> {
let payload = journal::encode(record);
let len = u32::try_from(payload.len()).unwrap_or(u32::MAX);
if len > MAX_RECORD_LEN {
return Err(WalError::Io {
context: format!(
"a single change came to {} bytes, over the {MAX_RECORD_LEN}-byte limit",
payload.len()
),
source: std::io::Error::new(std::io::ErrorKind::InvalidData, "record too large"),
});
}
let mut framed = Vec::with_capacity(FRAME_LEN + payload.len());
framed.extend_from_slice(&len.to_le_bytes());
framed.extend_from_slice(&checksum(len, &payload).to_le_bytes());
framed.extend_from_slice(&payload);
self.file
.write_all(&framed)
.map_err(io(format!("cannot append to {}", self.path.display())))?;
self.appended += 1;
self.dirty = true;
if self.policy == FsyncPolicy::Always {
self.flush()?;
}
Ok(())
}
pub fn flush(&mut self) -> Result<(), WalError> {
if !self.dirty {
return Ok(());
}
self.file
.sync_data()
.map_err(io(format!("cannot flush {}", self.path.display())))?;
self.dirty = false;
Ok(())
}
pub fn is_dirty(&self) -> bool {
self.dirty
}
}
impl Drop for Wal {
fn drop(&mut self) {
let _ = self.flush();
}
}
fn checksum(len: u32, payload: &[u8]) -> u64 {
let mut hasher = blake3::Hasher::new();
hasher.update(&len.to_le_bytes());
hasher.update(payload);
let hash = hasher.finalize();
let mut eight = [0u8; 8];
eight.copy_from_slice(&hash.as_bytes()[..8]);
u64::from_le_bytes(eight)
}
fn write_header(file: &mut File, path: &Path) -> Result<(), WalError> {
let mut header = Vec::with_capacity(HEADER_LEN as usize);
header.extend_from_slice(MAGIC);
header.extend_from_slice(&VERSION.to_le_bytes());
header.extend_from_slice(&0u32.to_le_bytes());
file.write_all(&header)
.map_err(io(format!("cannot write the header of {}", path.display())))?;
file.sync_data()
.map_err(io(format!("cannot flush the header of {}", path.display())))
}
fn read_header(file: &mut File, path: &Path) -> Result<(), WalError> {
file.seek(SeekFrom::Start(0))
.map_err(io(format!("cannot seek in {}", path.display())))?;
let mut header = [0u8; HEADER_LEN as usize];
file.read_exact(&mut header).map_err(|source| {
if source.kind() == std::io::ErrorKind::UnexpectedEof {
WalError::NotAWal {
path: path.to_path_buf(),
}
} else {
WalError::Io {
context: format!("cannot read the header of {}", path.display()),
source,
}
}
})?;
if &header[..8] != MAGIC {
return Err(WalError::NotAWal {
path: path.to_path_buf(),
});
}
let found = u32::from_le_bytes([header[8], header[9], header[10], header[11]]);
if found != VERSION {
return Err(WalError::UnknownVersion {
path: path.to_path_buf(),
found,
});
}
Ok(())
}
pub fn read(path: &Path) -> Result<Replay, WalError> {
let mut file = match File::open(path) {
Ok(f) => f,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
return Ok(Replay {
records: Vec::new(),
valid_len: 0,
torn_tail: None,
})
}
Err(e) => return Err(io(format!("cannot open {}", path.display()))(e)),
};
read_header(&mut file, path)?;
let mut reader = BufReader::new(file);
let mut records = Vec::new();
let mut offset = HEADER_LEN;
loop {
let mut frame = [0u8; FRAME_LEN];
match read_full(&mut reader, &mut frame) {
Ok(FRAME_LEN) => {}
Ok(0) => break, Ok(short) => {
return Ok(Replay {
records,
valid_len: offset,
torn_tail: Some(format!(
"the file ends {short} byte(s) into a record header at offset {offset}"
)),
})
}
Err(e) => return Err(io(format!("cannot read {}", path.display()))(e)),
}
let len = u32::from_le_bytes([frame[0], frame[1], frame[2], frame[3]]);
let stored = u64::from_le_bytes([
frame[4], frame[5], frame[6], frame[7], frame[8], frame[9], frame[10], frame[11],
]);
if len > MAX_RECORD_LEN {
return Ok(Replay {
records,
valid_len: offset,
torn_tail: Some(format!(
"the record at offset {offset} claims {len} bytes, over the \
{MAX_RECORD_LEN}-byte limit"
)),
});
}
let mut payload = vec![0u8; len as usize];
match read_full(&mut reader, &mut payload) {
Ok(n) if n as u32 == len => {}
Ok(n) => {
return Ok(Replay {
records,
valid_len: offset,
torn_tail: Some(format!(
"the record at offset {offset} wanted {len} bytes and the file has {n}"
)),
})
}
Err(e) => return Err(io(format!("cannot read {}", path.display()))(e)),
}
if checksum(len, &payload) != stored {
return Ok(Replay {
records,
valid_len: offset,
torn_tail: Some(format!(
"the record at offset {offset} does not match its checksum"
)),
});
}
match journal::decode(&payload) {
Ok(record) => records.push(record),
Err(source) => return Err(WalError::BadRecord { at: offset, source }),
}
offset += FRAME_LEN as u64 + u64::from(len);
}
Ok(Replay {
records,
valid_len: offset,
torn_tail: None,
})
}
fn read_full(reader: &mut impl Read, buf: &mut [u8]) -> std::io::Result<usize> {
let mut filled = 0;
while filled < buf.len() {
match reader.read(&mut buf[filled..]) {
Ok(0) => break,
Ok(n) => filled += n,
Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {}
Err(e) => return Err(e),
}
}
Ok(filled)
}
pub fn truncate(path: &Path, valid_len: u64) -> Result<(), WalError> {
let file = OpenOptions::new()
.write(true)
.open(path)
.map_err(io(format!("cannot open {} to truncate it", path.display())))?;
file.set_len(valid_len)
.map_err(io(format!("cannot truncate {}", path.display())))?;
file.sync_all().map_err(io(format!(
"cannot flush the truncation of {}",
path.display()
)))
}
pub fn reset(path: &Path) -> Result<(), WalError> {
let mut file = OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(true)
.open(path)
.map_err(io(format!("cannot create {}", path.display())))?;
write_header(&mut file, path)
}