1#![doc = include_str!("../README.md")]
2
3use crc32fast::hash;
4use std::fs::{self, File};
5use std::io::{IoSlice, Read, Seek, SeekFrom, Write};
6use std::path::{Path, PathBuf};
7use std::sync::atomic::{AtomicU64, Ordering};
8use std::sync::{Arc, Mutex, RwLock};
9
10mod error;
11
12pub use error::Error;
13
14pub type Result<T> = std::result::Result<T, Error>;
15
16const DATA_SUFFIX: &'static str = ".data";
17const INDEX_SUFFIX: &'static str = ".index";
18const LOCK_FILE: &'static str = "LOCK";
19
20const INDEX_INTERVAL: u64 = 1024;
21
22const READBUF_SIZE: usize = 128 * 1024; const LEN_SIZE: usize = std::mem::size_of::<u16>();
25const CHSUM_SIZE: usize = 2; const HEADER_SIZE: usize = LEN_SIZE + CHSUM_SIZE;
27
28const INDEX_SIZE: usize = std::mem::size_of::<u64>();
29
30const ENTRY_MAX_LEN: usize = u16::MAX as usize;
31
32struct SharedState {
34 dir: PathBuf,
36
37 file_seqs: RwLock<Vec<Arc<u64>>>,
40
41 current_dup: Mutex<Option<File>>,
44
45 sync_seq: AtomicU64,
47
48 next_seq: AtomicU64,
50}
51
52pub struct SeqLog {
54 _lock: File,
55
56 state: Arc<SharedState>,
57
58 rotate_size: usize,
60 rotate_count: usize,
61
62 current_data: File,
64 current_index: File,
65 data_file_size: usize,
66 file_seq: u64,
67}
68
69impl SeqLog {
70 pub fn create<P: AsRef<Path>>(path: P, start_seq: u64) -> Result<Self> {
85 let dir = path.as_ref();
86
87 fs::create_dir(dir)?;
88
89 File::create(dir.join(LOCK_FILE))?;
91
92 File::create(data_file_name(dir, start_seq))?;
94 File::create(index_file_name(dir, start_seq))?;
95
96 Self::open(dir)
97 }
98
99 pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
101 let dir = path.as_ref();
102
103 let lock = File::open(dir.join(LOCK_FILE))?;
104 lock.try_lock()?;
105
106 let file_seqs = list_files(dir)?;
107
108 let Some(file_seq) = file_seqs.last() else {
110 return Err(Error::NoDataFile);
111 };
112
113 let file_seq = **file_seq; let fname = data_file_name(dir, file_seq);
116 let mut current_data = File::options().append(true).read(true).open(fname)?;
117
118 let fname = index_file_name(dir, file_seq);
119 let mut current_index = File::options().append(true).read(true).open(fname)?;
120
121 let (count, file_size) =
122 check_current_file(&mut current_data, &mut current_index, file_seq)?;
123
124 let next_seq = file_seq + count;
125 let data_file_size = file_size as usize;
126 dbg!(next_seq);
127
128 let state = SharedState {
129 dir: dir.to_path_buf(),
130 file_seqs: RwLock::new(file_seqs),
131 current_dup: Mutex::new(None),
132 sync_seq: AtomicU64::new(next_seq),
133 next_seq: AtomicU64::new(next_seq),
134 };
135
136 Ok(Self {
137 _lock: lock,
138 state: Arc::new(state),
139
140 rotate_size: 1024 * 1024 * 1024, rotate_count: 10, current_data,
144 current_index,
145 data_file_size,
146 file_seq,
147 })
148 }
149
150 pub fn append<T>(&mut self, entries: &[T]) -> Result<()>
158 where
159 T: AsRef<[u8]>,
160 {
161 if self.data_file_size >= self.rotate_size {
166 self.rotate()?;
167 }
168
169 let mut headers = Vec::with_capacity(entries.len());
170 let mut bufs = Vec::with_capacity(entries.len() * 2); let mut total_len = 0;
173 let mut next_seq = self.next_seq();
174
175 for entry in entries.iter() {
176 let entry = entry.as_ref();
177 let len = entry.len();
178
179 if len > ENTRY_MAX_LEN {
180 return Err(Error::EntryTooLarge(len as usize));
181 }
182
183 let mut header_buf = [0u8; HEADER_SIZE];
185 header_buf[..2].copy_from_slice(&(len as u16).to_le_bytes());
186 header_buf[2..].copy_from_slice(&(hash(entry) as u16).to_le_bytes());
187 headers.push(header_buf);
188
189 bufs.push(IoSlice::new(&[])); bufs.push(IoSlice::new(entry));
193
194 total_len += HEADER_SIZE + len;
195 next_seq += 1;
196
197 if (next_seq - self.file_seq) % INDEX_INTERVAL == 0 {
199 let offset = self.data_file_size + total_len;
200 append_index(&mut self.current_index, offset as u64)?;
201 }
202 }
203
204 for (i, header) in headers.iter().enumerate() {
206 bufs[i * 2] = IoSlice::new(&header[..]);
207 }
208
209 let writen_len = self.current_data.write_vectored(&bufs)?;
211 if writen_len == 0 {
212 return Err(std::io::Error::from(std::io::ErrorKind::WriteZero).into());
213 }
214 if writen_len < total_len {
215 self.current_data.set_len(self.data_file_size as u64)?;
217 return Err(std::io::Error::new(std::io::ErrorKind::WriteZero, "partitial").into());
218 }
219
220 self.data_file_size += total_len;
222 self.state.next_seq.store(next_seq, Ordering::Release);
223
224 Ok(())
225 }
226
227 pub fn purge(&mut self, seq: u64) -> Result<()> {
236 let mut file_seqs = self.state.file_seqs.write().unwrap();
237
238 let mut until = file_seqs.len() - 1;
239 for (i, file_seq) in file_seqs.iter().enumerate() {
240 if **file_seq > seq {
241 until = i.saturating_sub(1);
242 break;
243 }
244 if Arc::strong_count(file_seq) > 1 {
245 until = i;
246 break;
247 }
248 }
249
250 for file_seq in file_seqs.drain(..until) {
251 fs::remove_file(data_file_name(&self.state.dir, *file_seq))?;
252 fs::remove_file(index_file_name(&self.state.dir, *file_seq))?;
253 }
254 Ok(())
255 }
256
257 pub fn truncate(&mut self, seq: u64) -> Result<()> {
261 if seq >= self.next_seq() {
262 return Err(Error::SeqNotReached(seq, self.next_seq()));
263 }
264
265 if seq < self.file_seq {
267 let mut file_seqs = self.state.file_seqs.write().unwrap();
268
269 let Some(i) = file_seqs.iter().rposition(|f| **f <= seq) else {
272 return Err(Error::SeqPurged(seq, *file_seqs[0]));
273 };
274
275 for file_seq in file_seqs.drain(i + 1..) {
277 fs::remove_file(data_file_name(&self.state.dir, *file_seq))?;
278 fs::remove_file(index_file_name(&self.state.dir, *file_seq))?;
279 }
280
281 let file_seq = *file_seqs[i];
282
283 let fname = data_file_name(&self.state.dir, file_seq);
285 self.current_data = File::options().append(true).read(true).open(fname)?;
286
287 let fname = index_file_name(&self.state.dir, file_seq);
288 self.current_index = File::options().append(true).read(true).open(fname)?;
289
290 self.data_file_size = 0; self.file_seq = file_seq;
292
293 *self.state.current_dup.lock().unwrap() = Some(self.current_data.try_clone()?);
294 }
295
296 let mut data_buf = Vec::new();
298 let (mut file, data_pos) =
299 seek_seq_in_file(&self.state.dir, self.file_seq, seq, &mut data_buf)?;
300 self.data_file_size = file.stream_position()? as usize - (data_buf.len() - data_pos);
301 self.current_data.set_len(self.data_file_size as u64)?;
302 self.current_data.seek(SeekFrom::End(0))?;
303
304 let index_file_size = (seq - self.file_seq) / INDEX_INTERVAL * INDEX_SIZE as u64;
307 self.current_index.set_len(index_file_size)?;
308 self.current_index.seek(SeekFrom::End(0))?;
309
310 self.state.next_seq.store(seq, Ordering::Relaxed);
312 self.state.sync_seq.store(seq, Ordering::Relaxed);
313
314 Ok(())
315 }
316
317 pub fn next_seq(&self) -> u64 {
319 self.state.next_seq()
320 }
321
322 pub fn sync_seq(&self) -> u64 {
326 self.state.sync_seq()
327 }
328
329 pub fn sync(&self) -> Result<()> {
336 let seq = self.next_seq();
337
338 self.current_data.sync_data()?;
339
340 self.state.sync_seq.store(seq, Ordering::Release);
341 Ok(())
342 }
343
344 pub fn syncer(&self) -> Result<SeqLogSyncer> {
346 Ok(SeqLogSyncer {
347 state: self.state.clone(),
348 current: self.current_data.try_clone()?,
349 })
350 }
351
352 pub fn reader(&self, next_seq: u64, synced_only: bool) -> Result<SeqLogReader> {
357 let mut data_buf = Vec::new();
358
359 let (current, file_seq, data_pos) =
360 self.state.seek_seq(next_seq, &mut data_buf, synced_only)?;
361
362 Ok(SeqLogReader {
363 state: self.state.clone(),
364 data_buf,
365
366 synced_only,
367
368 current,
369 file_seq,
370 data_pos,
371 next_seq,
372 })
373 }
374
375 pub fn set_rotate(&mut self, size: usize, count: usize) {
385 self.rotate_size = if size == 0 { usize::MAX } else { size };
386 self.rotate_count = if count == 0 { usize::MAX } else { count };
387 }
388
389 pub fn rotate(&mut self) -> Result<()> {
396 if self.data_file_size == 0 {
397 return Ok(());
399 }
400
401 let next_seq = self.next_seq();
402
403 self.current_data = File::create_new(data_file_name(&self.state.dir, next_seq))?;
405 self.current_index = File::create_new(index_file_name(&self.state.dir, next_seq))?;
406 self.data_file_size = 0;
407 self.file_seq = next_seq;
408
409 let mut file_seqs = self.state.file_seqs.write().unwrap();
411 file_seqs.push(Arc::new(next_seq));
412
413 while file_seqs.len() > self.rotate_count {
415 if Arc::strong_count(&file_seqs[0]) > 0 {
416 break;
418 }
419 let file_seq = file_seqs.remove(0);
420 fs::remove_file(data_file_name(&self.state.dir, *file_seq))?;
421 fs::remove_file(index_file_name(&self.state.dir, *file_seq))?;
422 }
423
424 *self.state.current_dup.lock().unwrap() = Some(self.current_data.try_clone()?);
426
427 Ok(())
428 }
429}
430
431pub struct SeqLogReader {
451 state: Arc<SharedState>,
452
453 synced_only: bool,
454
455 current: File,
456 file_seq: Arc<u64>,
457 data_buf: Vec<u8>,
458 data_pos: usize,
459 next_seq: u64,
460}
461
462impl SeqLogReader {
463 pub fn next(&mut self) -> Result<Option<&[u8]>> {
465 let max_seq = if self.synced_only {
467 self.state.sync_seq()
468 } else {
469 self.state.next_seq()
470 };
471 if self.next_seq >= max_seq {
472 return Ok(None);
473 }
474
475 let len = match parse_entry_len(&self.data_buf[self.data_pos..]) {
479 Some(len) => len,
480 None => {
481 self.read_more_data()?;
483 parse_entry_len(&self.data_buf)
484 .ok_or(Error::DataFileTruncated(*self.file_seq, self.next_seq))?
485 }
486 };
487
488 let entry = &self.data_buf[self.data_pos..];
489 let payload = &entry[HEADER_SIZE..HEADER_SIZE + len];
490
491 let chsum1 = parse_checksum(&entry[LEN_SIZE..]);
493 let chsum2 = hash(&payload) as u16;
494 if chsum1 != chsum2 {
495 return Err(Error::ChecksumMismatch(*self.file_seq, self.next_seq));
496 }
497
498 self.next_seq += 1;
499 self.data_pos += HEADER_SIZE + len;
500 return Ok(Some(payload));
501 }
502
503 fn read_more_data(&mut self) -> Result<()> {
506 let remain_len = self.data_buf.len() - self.data_pos;
507 self.data_buf.copy_within(self.data_pos.., 0);
508
509 self.data_buf.resize(READBUF_SIZE, 0);
510
511 let mut len = self.current.read(&mut self.data_buf[remain_len..])?;
512
513 if len == 0 {
514 if remain_len != 0 {
517 return Err(Error::DataFileTruncated(*self.file_seq, self.next_seq));
518 }
519
520 let file_seqs = self.state.file_seqs.read().unwrap();
521 let Some(file_seq) = file_seqs.iter().rev().find(|&f| **f == self.next_seq) else {
522 return Err(Error::DataFileTruncated(self.next_seq, self.next_seq));
523 };
524
525 self.file_seq = file_seq.clone();
526 self.current = File::open(data_file_name(&self.state.dir, **file_seq))?;
527
528 len = self.current.read(&mut self.data_buf)?;
529 if len == 0 {
530 return Err(Error::DataFileTruncated(*self.file_seq, self.next_seq));
531 }
532 }
533
534 self.data_buf.truncate(remain_len + len);
535 self.data_pos = 0;
536 Ok(())
537 }
538
539 pub fn next_seq(&self) -> u64 {
541 self.next_seq
542 }
543
544 pub fn reset(&mut self, seq: u64) -> Result<()> {
550 let (file, file_seq, data_pos) =
551 self.state
552 .seek_seq(seq, &mut self.data_buf, self.synced_only)?;
553
554 self.data_pos = data_pos;
556 self.file_seq = file_seq;
557 self.current = file;
558 self.next_seq = seq;
559 Ok(())
560 }
561}
562
563pub struct SeqLogSyncer {
570 state: Arc<SharedState>,
571
572 current: File,
573}
574
575impl SeqLogSyncer {
576 pub fn sync(&mut self) -> Result<()> {
578 let mut next_seq = self.state.next_seq();
580
581 self.current.sync_data()?;
582
583 if let Some(new_file) = self.state.current_dup.lock().unwrap().take() {
585 next_seq = self.state.next_seq();
586 self.current = new_file;
587 self.current.sync_data()?;
588 }
589
590 self.state.sync_seq.store(next_seq, Ordering::Release);
592 Ok(())
593 }
594}
595
596impl SharedState {
597 fn seek_seq(
598 &self,
599 seq: u64,
600 data_buf: &mut Vec<u8>,
601 synced_only: bool,
602 ) -> Result<(File, Arc<u64>, usize)> {
603 let max_seq = if synced_only {
604 self.sync_seq()
605 } else {
606 self.next_seq()
607 };
608 if seq > max_seq {
609 return Err(Error::SeqNotReached(seq, max_seq));
610 }
611
612 let file_seqs = self.file_seqs.read().unwrap();
614 let Some(file_seq) = file_seqs.iter().rev().find(|&f| **f <= seq) else {
615 return Err(Error::SeqPurged(seq, *file_seqs[0]));
616 };
617
618 let file_seq = file_seq.clone();
620
621 drop(file_seqs);
623
624 let (file, data_pos) = seek_seq_in_file(&self.dir, *file_seq, seq, data_buf)?;
626
627 Ok((file, file_seq, data_pos))
628 }
629
630 fn next_seq(&self) -> u64 {
631 self.next_seq.load(Ordering::Acquire)
632 }
633
634 fn sync_seq(&self) -> u64 {
635 self.sync_seq.load(Ordering::Acquire)
636 }
637}
638
639fn seek_seq_in_file(
640 dir: &Path,
641 file_seq: u64,
642 seq: u64,
643 data_buf: &mut Vec<u8>,
644) -> Result<(File, usize)> {
645 let mut data_file = File::open(data_file_name(dir, file_seq))?;
646
647 let diff_seq = seq - file_seq;
649 if diff_seq >= INDEX_INTERVAL {
650 let mut index_file = File::open(index_file_name(dir, file_seq))?;
651 index_file.seek(SeekFrom::Start(
652 (diff_seq / INDEX_INTERVAL - 1) * INDEX_SIZE as u64,
653 ))?;
654
655 let mut index_buf = [0; INDEX_SIZE];
656 index_file.read_exact(&mut index_buf)?;
657
658 let offset = parse_index(&index_buf);
659
660 data_file.seek(SeekFrom::Start(offset))?;
662 }
663
664 data_buf.resize(READBUF_SIZE, 0);
666 let len = data_file.read(data_buf)?;
667 data_buf.truncate(len);
668
669 let mut count = diff_seq % INDEX_INTERVAL;
670 if count == 0 {
671 return Ok((data_file, 0));
672 }
673
674 loop {
676 let mut data_pos = 0;
677 while let Some(len) = parse_entry_len(&data_buf[data_pos..]) {
678 data_pos += HEADER_SIZE + len;
679 count -= 1;
680 if count == 0 {
681 return Ok((data_file, data_pos));
682 }
683 }
684
685 if data_buf.len() < READBUF_SIZE {
686 return Err(Error::DataFileTruncated(file_seq, seq));
687 }
688
689 data_buf.copy_within(data_pos.., 0);
690
691 let len = data_file.read(&mut data_buf[READBUF_SIZE - data_pos..])?;
692 if len == 0 {
693 return Err(Error::DataFileTruncated(file_seq, seq));
694 }
695 data_buf.truncate(READBUF_SIZE - data_pos + len);
696 }
697}
698
699fn list_files(dir: &Path) -> Result<Vec<Arc<u64>>> {
701 let mut files = Vec::new();
702
703 for entry in fs::read_dir(dir)? {
704 let path = entry?.path().to_path_buf();
705 let fname = path
706 .file_name()
707 .expect("invalid filename")
708 .to_str()
709 .expect("fail to parse filename");
710 if !fname.ends_with(DATA_SUFFIX) {
711 continue;
712 }
713 let Ok(seq) = fname[..fname.len() - DATA_SUFFIX.len()].parse() else {
714 return Err(Error::InvalidDataFilename(fname.to_string()));
715 };
716
717 files.push(Arc::new(seq));
718 }
719
720 files.sort();
721
722 Ok(files)
723}
724
725fn check_current_file(
726 data_file: &mut File,
727 index_file: &mut File,
728 file_seq: u64,
729) -> Result<(u64, u64)> {
730 let index_file_len = index_file.metadata()?.len() as usize;
731 if index_file_len % INDEX_SIZE != 0 {
732 return Err(Error::InvalidIndexFile(file_seq));
733 }
734
735 let mut count = 0;
737 let mut offset = 0;
738 if index_file_len != 0 {
739 let mut index_buf = [0; INDEX_SIZE];
741 index_file.seek(SeekFrom::End(-(INDEX_SIZE as i64)))?;
742 index_file.read_exact(&mut index_buf)?;
743 offset = parse_index(&index_buf);
744
745 data_file.seek(SeekFrom::Start(offset))?;
747
748 count = (index_file_len / INDEX_SIZE) as u64 * INDEX_INTERVAL;
749 }
750
751 let mut data_buf = Vec::new();
753 data_file.read_to_end(&mut data_buf)?;
754
755 let mut entry = &data_buf[..];
757 while let Some(len) = parse_entry_len(entry) {
758 entry = &entry[HEADER_SIZE + len..];
759
760 count += 1;
761
762 offset += (HEADER_SIZE + len) as u64;
763 if count % INDEX_INTERVAL == 0 {
764 append_index(index_file, offset)?;
765 }
766 }
767
768 Ok((count, offset))
769}
770
771fn append_index(file: &mut File, offset: u64) -> Result<()> {
773 Ok(file.write_all(&offset.to_le_bytes())?)
774}
775
776fn data_file_name(dir: &Path, seq: u64) -> PathBuf {
778 dir.join(format!("{:020}{}", seq, DATA_SUFFIX))
779}
780
781fn index_file_name(dir: &Path, seq: u64) -> PathBuf {
783 dir.join(format!("{:020}{}", seq, INDEX_SUFFIX))
784}
785
786fn parse_entry_len(buf: &[u8]) -> Option<usize> {
788 if buf.len() < HEADER_SIZE {
789 return None;
790 }
791 let len = u16::from_le_bytes(buf[0..LEN_SIZE].try_into().unwrap()) as usize;
792 if buf.len() < HEADER_SIZE + len {
793 return None;
794 }
795 Some(len)
796}
797
798fn parse_checksum(buf: &[u8]) -> u16 {
799 u16::from_le_bytes(buf[0..CHSUM_SIZE].try_into().unwrap())
800}
801
802fn parse_index(buf: &[u8]) -> u64 {
803 u64::from_le_bytes(buf[..8].try_into().unwrap())
804}