#![doc = include_str!("../README.md")]
use crc32fast::hash;
use std::fs::{self, File};
use std::io::{IoSlice, Read, Seek, SeekFrom, Write};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex, RwLock};
mod error;
pub use error::Error;
pub type Result<T> = std::result::Result<T, Error>;
const DATA_SUFFIX: &'static str = ".data";
const INDEX_SUFFIX: &'static str = ".index";
const LOCK_FILE: &'static str = "LOCK";
const INDEX_INTERVAL: u64 = 1024;
const READBUF_SIZE: usize = 128 * 1024;
const LEN_SIZE: usize = std::mem::size_of::<u16>();
const CHSUM_SIZE: usize = 2; const HEADER_SIZE: usize = LEN_SIZE + CHSUM_SIZE;
const INDEX_SIZE: usize = std::mem::size_of::<u64>();
const ENTRY_MAX_LEN: usize = u16::MAX as usize;
struct SharedState {
dir: PathBuf,
file_seqs: RwLock<Vec<Arc<u64>>>,
current_dup: Mutex<Option<File>>,
sync_seq: AtomicU64,
next_seq: AtomicU64,
}
pub struct SeqLog {
_lock: File,
state: Arc<SharedState>,
rotate_size: usize,
rotate_count: usize,
current_data: File,
current_index: File,
data_file_size: usize,
file_seq: u64,
}
impl SeqLog {
pub fn create<P: AsRef<Path>>(path: P, start_seq: u64) -> Result<Self> {
let dir = path.as_ref();
fs::create_dir(dir)?;
File::create(dir.join(LOCK_FILE))?;
File::create(data_file_name(dir, start_seq))?;
File::create(index_file_name(dir, start_seq))?;
Self::open(dir)
}
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
let dir = path.as_ref();
let lock = File::open(dir.join(LOCK_FILE))?;
lock.try_lock()?;
let file_seqs = list_files(dir)?;
let Some(file_seq) = file_seqs.last() else {
return Err(Error::NoDataFile);
};
let file_seq = **file_seq;
let fname = data_file_name(dir, file_seq);
let mut current_data = File::options().append(true).read(true).open(fname)?;
let fname = index_file_name(dir, file_seq);
let mut current_index = File::options().append(true).read(true).open(fname)?;
let (count, file_size) =
check_current_file(&mut current_data, &mut current_index, file_seq)?;
let next_seq = file_seq + count;
let data_file_size = file_size as usize;
dbg!(next_seq);
let state = SharedState {
dir: dir.to_path_buf(),
file_seqs: RwLock::new(file_seqs),
current_dup: Mutex::new(None),
sync_seq: AtomicU64::new(next_seq),
next_seq: AtomicU64::new(next_seq),
};
Ok(Self {
_lock: lock,
state: Arc::new(state),
rotate_size: 1024 * 1024 * 1024, rotate_count: 10,
current_data,
current_index,
data_file_size,
file_seq,
})
}
pub fn append<T>(&mut self, entries: &[T]) -> Result<()>
where
T: AsRef<[u8]>,
{
if self.data_file_size >= self.rotate_size {
self.rotate()?;
}
let mut headers = Vec::with_capacity(entries.len());
let mut bufs = Vec::with_capacity(entries.len() * 2);
let mut total_len = 0;
let mut next_seq = self.next_seq();
for entry in entries.iter() {
let entry = entry.as_ref();
let len = entry.len();
if len > ENTRY_MAX_LEN {
return Err(Error::EntryTooLarge(len as usize));
}
let mut header_buf = [0u8; HEADER_SIZE];
header_buf[..2].copy_from_slice(&(len as u16).to_le_bytes());
header_buf[2..].copy_from_slice(&(hash(entry) as u16).to_le_bytes());
headers.push(header_buf);
bufs.push(IoSlice::new(&[]));
bufs.push(IoSlice::new(entry));
total_len += HEADER_SIZE + len;
next_seq += 1;
if (next_seq - self.file_seq) % INDEX_INTERVAL == 0 {
let offset = self.data_file_size + total_len;
append_index(&mut self.current_index, offset as u64)?;
}
}
for (i, header) in headers.iter().enumerate() {
bufs[i * 2] = IoSlice::new(&header[..]);
}
let writen_len = self.current_data.write_vectored(&bufs)?;
if writen_len == 0 {
return Err(std::io::Error::from(std::io::ErrorKind::WriteZero).into());
}
if writen_len < total_len {
self.current_data.set_len(self.data_file_size as u64)?;
return Err(std::io::Error::new(std::io::ErrorKind::WriteZero, "partitial").into());
}
self.data_file_size += total_len;
self.state.next_seq.store(next_seq, Ordering::Release);
Ok(())
}
pub fn purge(&mut self, seq: u64) -> Result<()> {
let mut file_seqs = self.state.file_seqs.write().unwrap();
let mut until = file_seqs.len() - 1;
for (i, file_seq) in file_seqs.iter().enumerate() {
if **file_seq > seq {
until = i.saturating_sub(1);
break;
}
if Arc::strong_count(file_seq) > 1 {
until = i;
break;
}
}
for file_seq in file_seqs.drain(..until) {
fs::remove_file(data_file_name(&self.state.dir, *file_seq))?;
fs::remove_file(index_file_name(&self.state.dir, *file_seq))?;
}
Ok(())
}
pub fn truncate(&mut self, seq: u64) -> Result<()> {
if seq >= self.next_seq() {
return Err(Error::SeqNotReached(seq, self.next_seq()));
}
if seq < self.file_seq {
let mut file_seqs = self.state.file_seqs.write().unwrap();
let Some(i) = file_seqs.iter().rposition(|f| **f <= seq) else {
return Err(Error::SeqPurged(seq, *file_seqs[0]));
};
for file_seq in file_seqs.drain(i + 1..) {
fs::remove_file(data_file_name(&self.state.dir, *file_seq))?;
fs::remove_file(index_file_name(&self.state.dir, *file_seq))?;
}
let file_seq = *file_seqs[i];
let fname = data_file_name(&self.state.dir, file_seq);
self.current_data = File::options().append(true).read(true).open(fname)?;
let fname = index_file_name(&self.state.dir, file_seq);
self.current_index = File::options().append(true).read(true).open(fname)?;
self.data_file_size = 0; self.file_seq = file_seq;
*self.state.current_dup.lock().unwrap() = Some(self.current_data.try_clone()?);
}
let mut data_buf = Vec::new();
let (mut file, data_pos) =
seek_seq_in_file(&self.state.dir, self.file_seq, seq, &mut data_buf)?;
self.data_file_size = file.stream_position()? as usize - (data_buf.len() - data_pos);
self.current_data.set_len(self.data_file_size as u64)?;
self.current_data.seek(SeekFrom::End(0))?;
let index_file_size = (seq - self.file_seq) / INDEX_INTERVAL * INDEX_SIZE as u64;
self.current_index.set_len(index_file_size)?;
self.current_index.seek(SeekFrom::End(0))?;
self.state.next_seq.store(seq, Ordering::Relaxed);
self.state.sync_seq.store(seq, Ordering::Relaxed);
Ok(())
}
pub fn next_seq(&self) -> u64 {
self.state.next_seq()
}
pub fn sync_seq(&self) -> u64 {
self.state.sync_seq()
}
pub fn sync(&self) -> Result<()> {
let seq = self.next_seq();
self.current_data.sync_data()?;
self.state.sync_seq.store(seq, Ordering::Release);
Ok(())
}
pub fn syncer(&self) -> Result<SeqLogSyncer> {
Ok(SeqLogSyncer {
state: self.state.clone(),
current: self.current_data.try_clone()?,
})
}
pub fn reader(&self, next_seq: u64, synced_only: bool) -> Result<SeqLogReader> {
let mut data_buf = Vec::new();
let (current, file_seq, data_pos) =
self.state.seek_seq(next_seq, &mut data_buf, synced_only)?;
Ok(SeqLogReader {
state: self.state.clone(),
data_buf,
synced_only,
current,
file_seq,
data_pos,
next_seq,
})
}
pub fn set_rotate(&mut self, size: usize, count: usize) {
self.rotate_size = if size == 0 { usize::MAX } else { size };
self.rotate_count = if count == 0 { usize::MAX } else { count };
}
pub fn rotate(&mut self) -> Result<()> {
if self.data_file_size == 0 {
return Ok(());
}
let next_seq = self.next_seq();
self.current_data = File::create_new(data_file_name(&self.state.dir, next_seq))?;
self.current_index = File::create_new(index_file_name(&self.state.dir, next_seq))?;
self.data_file_size = 0;
self.file_seq = next_seq;
let mut file_seqs = self.state.file_seqs.write().unwrap();
file_seqs.push(Arc::new(next_seq));
while file_seqs.len() > self.rotate_count {
if Arc::strong_count(&file_seqs[0]) > 0 {
break;
}
let file_seq = file_seqs.remove(0);
fs::remove_file(data_file_name(&self.state.dir, *file_seq))?;
fs::remove_file(index_file_name(&self.state.dir, *file_seq))?;
}
*self.state.current_dup.lock().unwrap() = Some(self.current_data.try_clone()?);
Ok(())
}
}
pub struct SeqLogReader {
state: Arc<SharedState>,
synced_only: bool,
current: File,
file_seq: Arc<u64>,
data_buf: Vec<u8>,
data_pos: usize,
next_seq: u64,
}
impl SeqLogReader {
pub fn next(&mut self) -> Result<Option<&[u8]>> {
let max_seq = if self.synced_only {
self.state.sync_seq()
} else {
self.state.next_seq()
};
if self.next_seq >= max_seq {
return Ok(None);
}
let len = match parse_entry_len(&self.data_buf[self.data_pos..]) {
Some(len) => len,
None => {
self.read_more_data()?;
parse_entry_len(&self.data_buf)
.ok_or(Error::DataFileTruncated(*self.file_seq, self.next_seq))?
}
};
let entry = &self.data_buf[self.data_pos..];
let payload = &entry[HEADER_SIZE..HEADER_SIZE + len];
let chsum1 = parse_checksum(&entry[LEN_SIZE..]);
let chsum2 = hash(&payload) as u16;
if chsum1 != chsum2 {
return Err(Error::ChecksumMismatch(*self.file_seq, self.next_seq));
}
self.next_seq += 1;
self.data_pos += HEADER_SIZE + len;
return Ok(Some(payload));
}
fn read_more_data(&mut self) -> Result<()> {
let remain_len = self.data_buf.len() - self.data_pos;
self.data_buf.copy_within(self.data_pos.., 0);
self.data_buf.resize(READBUF_SIZE, 0);
let mut len = self.current.read(&mut self.data_buf[remain_len..])?;
if len == 0 {
if remain_len != 0 {
return Err(Error::DataFileTruncated(*self.file_seq, self.next_seq));
}
let file_seqs = self.state.file_seqs.read().unwrap();
let Some(file_seq) = file_seqs.iter().rev().find(|&f| **f == self.next_seq) else {
return Err(Error::DataFileTruncated(self.next_seq, self.next_seq));
};
self.file_seq = file_seq.clone();
self.current = File::open(data_file_name(&self.state.dir, **file_seq))?;
len = self.current.read(&mut self.data_buf)?;
if len == 0 {
return Err(Error::DataFileTruncated(*self.file_seq, self.next_seq));
}
}
self.data_buf.truncate(remain_len + len);
self.data_pos = 0;
Ok(())
}
pub fn next_seq(&self) -> u64 {
self.next_seq
}
pub fn reset(&mut self, seq: u64) -> Result<()> {
let (file, file_seq, data_pos) =
self.state
.seek_seq(seq, &mut self.data_buf, self.synced_only)?;
self.data_pos = data_pos;
self.file_seq = file_seq;
self.current = file;
self.next_seq = seq;
Ok(())
}
}
pub struct SeqLogSyncer {
state: Arc<SharedState>,
current: File,
}
impl SeqLogSyncer {
pub fn sync(&mut self) -> Result<()> {
let mut next_seq = self.state.next_seq();
self.current.sync_data()?;
if let Some(new_file) = self.state.current_dup.lock().unwrap().take() {
next_seq = self.state.next_seq();
self.current = new_file;
self.current.sync_data()?;
}
self.state.sync_seq.store(next_seq, Ordering::Release);
Ok(())
}
}
impl SharedState {
fn seek_seq(
&self,
seq: u64,
data_buf: &mut Vec<u8>,
synced_only: bool,
) -> Result<(File, Arc<u64>, usize)> {
let max_seq = if synced_only {
self.sync_seq()
} else {
self.next_seq()
};
if seq > max_seq {
return Err(Error::SeqNotReached(seq, max_seq));
}
let file_seqs = self.file_seqs.read().unwrap();
let Some(file_seq) = file_seqs.iter().rev().find(|&f| **f <= seq) else {
return Err(Error::SeqPurged(seq, *file_seqs[0]));
};
let file_seq = file_seq.clone();
drop(file_seqs);
let (file, data_pos) = seek_seq_in_file(&self.dir, *file_seq, seq, data_buf)?;
Ok((file, file_seq, data_pos))
}
fn next_seq(&self) -> u64 {
self.next_seq.load(Ordering::Acquire)
}
fn sync_seq(&self) -> u64 {
self.sync_seq.load(Ordering::Acquire)
}
}
fn seek_seq_in_file(
dir: &Path,
file_seq: u64,
seq: u64,
data_buf: &mut Vec<u8>,
) -> Result<(File, usize)> {
let mut data_file = File::open(data_file_name(dir, file_seq))?;
let diff_seq = seq - file_seq;
if diff_seq >= INDEX_INTERVAL {
let mut index_file = File::open(index_file_name(dir, file_seq))?;
index_file.seek(SeekFrom::Start(
(diff_seq / INDEX_INTERVAL - 1) * INDEX_SIZE as u64,
))?;
let mut index_buf = [0; INDEX_SIZE];
index_file.read_exact(&mut index_buf)?;
let offset = parse_index(&index_buf);
data_file.seek(SeekFrom::Start(offset))?;
}
data_buf.resize(READBUF_SIZE, 0);
let len = data_file.read(data_buf)?;
data_buf.truncate(len);
let mut count = diff_seq % INDEX_INTERVAL;
if count == 0 {
return Ok((data_file, 0));
}
loop {
let mut data_pos = 0;
while let Some(len) = parse_entry_len(&data_buf[data_pos..]) {
data_pos += HEADER_SIZE + len;
count -= 1;
if count == 0 {
return Ok((data_file, data_pos));
}
}
if data_buf.len() < READBUF_SIZE {
return Err(Error::DataFileTruncated(file_seq, seq));
}
data_buf.copy_within(data_pos.., 0);
let len = data_file.read(&mut data_buf[READBUF_SIZE - data_pos..])?;
if len == 0 {
return Err(Error::DataFileTruncated(file_seq, seq));
}
data_buf.truncate(READBUF_SIZE - data_pos + len);
}
}
fn list_files(dir: &Path) -> Result<Vec<Arc<u64>>> {
let mut files = Vec::new();
for entry in fs::read_dir(dir)? {
let path = entry?.path().to_path_buf();
let fname = path
.file_name()
.expect("invalid filename")
.to_str()
.expect("fail to parse filename");
if !fname.ends_with(DATA_SUFFIX) {
continue;
}
let Ok(seq) = fname[..fname.len() - DATA_SUFFIX.len()].parse() else {
return Err(Error::InvalidDataFilename(fname.to_string()));
};
files.push(Arc::new(seq));
}
files.sort();
Ok(files)
}
fn check_current_file(
data_file: &mut File,
index_file: &mut File,
file_seq: u64,
) -> Result<(u64, u64)> {
let index_file_len = index_file.metadata()?.len() as usize;
if index_file_len % INDEX_SIZE != 0 {
return Err(Error::InvalidIndexFile(file_seq));
}
let mut count = 0;
let mut offset = 0;
if index_file_len != 0 {
let mut index_buf = [0; INDEX_SIZE];
index_file.seek(SeekFrom::End(-(INDEX_SIZE as i64)))?;
index_file.read_exact(&mut index_buf)?;
offset = parse_index(&index_buf);
data_file.seek(SeekFrom::Start(offset))?;
count = (index_file_len / INDEX_SIZE) as u64 * INDEX_INTERVAL;
}
let mut data_buf = Vec::new();
data_file.read_to_end(&mut data_buf)?;
let mut entry = &data_buf[..];
while let Some(len) = parse_entry_len(entry) {
entry = &entry[HEADER_SIZE + len..];
count += 1;
offset += (HEADER_SIZE + len) as u64;
if count % INDEX_INTERVAL == 0 {
append_index(index_file, offset)?;
}
}
Ok((count, offset))
}
fn append_index(file: &mut File, offset: u64) -> Result<()> {
Ok(file.write_all(&offset.to_le_bytes())?)
}
fn data_file_name(dir: &Path, seq: u64) -> PathBuf {
dir.join(format!("{:020}{}", seq, DATA_SUFFIX))
}
fn index_file_name(dir: &Path, seq: u64) -> PathBuf {
dir.join(format!("{:020}{}", seq, INDEX_SUFFIX))
}
fn parse_entry_len(buf: &[u8]) -> Option<usize> {
if buf.len() < HEADER_SIZE {
return None;
}
let len = u16::from_le_bytes(buf[0..LEN_SIZE].try_into().unwrap()) as usize;
if buf.len() < HEADER_SIZE + len {
return None;
}
Some(len)
}
fn parse_checksum(buf: &[u8]) -> u16 {
u16::from_le_bytes(buf[0..CHSUM_SIZE].try_into().unwrap())
}
fn parse_index(buf: &[u8]) -> u64 {
u64::from_le_bytes(buf[..8].try_into().unwrap())
}