Skip to main content

seqlog/
lib.rs

1#![doc = include_str!("../README.md")]
2
3use crc32fast::hash;
4use std::fs::{self, File};
5use std::io::{IoSlice, Read, Seek, SeekFrom, Write};
6use std::path::{Path, PathBuf};
7use std::sync::atomic::{AtomicU64, Ordering};
8use std::sync::{Arc, Mutex, RwLock};
9
10mod error;
11
12pub use error::Error;
13
14pub type Result<T> = std::result::Result<T, Error>;
15
16const DATA_SUFFIX: &'static str = ".data";
17const INDEX_SUFFIX: &'static str = ".index";
18const LOCK_FILE: &'static str = "LOCK";
19
20const INDEX_INTERVAL: u64 = 1024;
21
22const READBUF_SIZE: usize = 128 * 1024; // bigger that ENTRY_MAX_LEN
23
24const LEN_SIZE: usize = std::mem::size_of::<u16>();
25const CHSUM_SIZE: usize = 2; // use 2 bytes of CRC32
26const HEADER_SIZE: usize = LEN_SIZE + CHSUM_SIZE;
27
28const INDEX_SIZE: usize = std::mem::size_of::<u64>();
29
30const ENTRY_MAX_LEN: usize = u16::MAX as usize;
31
32// The state that is shared amount Writer, Syncer, and multiple Readers.
33struct SharedState {
34    // read-only
35    dir: PathBuf,
36
37    // Writer updates this when rotating.
38    // Readers use this to scan files.
39    file_seqs: RwLock<Vec<Arc<u64>>>,
40
41    // Writer inserts new file's clone into this when rotating.
42    // Syncer takes this.
43    current_dup: Mutex<Option<File>>,
44
45    // Syncer updates this after syncing.
46    sync_seq: AtomicU64,
47
48    // Writer updates this after appending.
49    next_seq: AtomicU64,
50}
51
52/// SeqLog instance. It is also the Writer.
53pub struct SeqLog {
54    _lock: File,
55
56    state: Arc<SharedState>,
57
58    // config
59    rotate_size: usize,
60    rotate_count: usize,
61
62    // Writer's current file status
63    current_data: File,
64    current_index: File,
65    data_file_size: usize,
66    file_seq: u64,
67}
68
69impl SeqLog {
70    /// Create a SeqLog instance with start sequence number, and open it.
71    ///
72    /// This requires that the parent directory of the given path already
73    /// exists and the path itself does not exist.
74    ///
75    /// This method creates the directory and some required files. If you
76    /// want to create it out of your program, use this script:
77    ///
78    /// ```bash
79    /// mkdir $path
80    /// touch $path/LOCK
81    /// touch $path/`printf "%020d.data" $start_seq`   # first data file
82    /// touch $path/`printf "%020d.index" $start_seq`  # first index file
83    /// ```
84    pub fn create<P: AsRef<Path>>(path: P, start_seq: u64) -> Result<Self> {
85        let dir = path.as_ref();
86
87        fs::create_dir(dir)?;
88
89        // LOCK file
90        File::create(dir.join(LOCK_FILE))?;
91
92        // first empty data file and index file
93        File::create(data_file_name(dir, start_seq))?;
94        File::create(index_file_name(dir, start_seq))?;
95
96        Self::open(dir)
97    }
98
99    /// Open an existing SeqLog instance.
100    pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
101        let dir = path.as_ref();
102
103        let lock = File::open(dir.join(LOCK_FILE))?;
104        lock.try_lock()?;
105
106        let file_seqs = list_files(dir)?;
107
108        // at least one data file
109        let Some(file_seq) = file_seqs.last() else {
110            return Err(Error::NoDataFile);
111        };
112
113        let file_seq = **file_seq; // deref "&Arc<u64>" to u64
114
115        let fname = data_file_name(dir, file_seq);
116        let mut current_data = File::options().append(true).read(true).open(fname)?;
117
118        let fname = index_file_name(dir, file_seq);
119        let mut current_index = File::options().append(true).read(true).open(fname)?;
120
121        let (count, file_size) =
122            check_current_file(&mut current_data, &mut current_index, file_seq)?;
123
124        let next_seq = file_seq + count;
125        let data_file_size = file_size as usize;
126        dbg!(next_seq);
127
128        let state = SharedState {
129            dir: dir.to_path_buf(),
130            file_seqs: RwLock::new(file_seqs),
131            current_dup: Mutex::new(None),
132            sync_seq: AtomicU64::new(next_seq),
133            next_seq: AtomicU64::new(next_seq),
134        };
135
136        Ok(Self {
137            _lock: lock,
138            state: Arc::new(state),
139
140            rotate_size: 1024 * 1024 * 1024, // default value: 1G
141            rotate_count: 10,                // default value: 10
142
143            current_data,
144            current_index,
145            data_file_size,
146            file_seq,
147        })
148    }
149
150    /// Append a batch of entries.
151    ///
152    /// This issues one `write(2)` syscall. Therefore, appending in batches
153    /// reduces the number of system calls.
154    ///
155    /// This writes data into file (page cache), but does not synchronize
156    /// disk. Use [`Self::sync`] or [`SeqLogSyncer`] if needed.
157    pub fn append<T>(&mut self, entries: &[T]) -> Result<()>
158    where
159        T: AsRef<[u8]>,
160    {
161        // check rotation
162        //
163        // We must rotate at the beginning of function, but not at the tail.
164        // So uses can always sync() the current file after the function.
165        if self.data_file_size >= self.rotate_size {
166            self.rotate()?;
167        }
168
169        let mut headers = Vec::with_capacity(entries.len());
170        let mut bufs = Vec::with_capacity(entries.len() * 2); // header + payload
171
172        let mut total_len = 0;
173        let mut next_seq = self.next_seq();
174
175        for entry in entries.iter() {
176            let entry = entry.as_ref();
177            let len = entry.len();
178
179            if len > ENTRY_MAX_LEN {
180                return Err(Error::EntryTooLarge(len as usize));
181            }
182
183            // encode the header: length + checksum
184            let mut header_buf = [0u8; HEADER_SIZE];
185            header_buf[..2].copy_from_slice(&(len as u16).to_le_bytes());
186            header_buf[2..].copy_from_slice(&(hash(entry) as u16).to_le_bytes());
187            headers.push(header_buf);
188
189            bufs.push(IoSlice::new(&[])); // hold the place
190
191            // the payload
192            bufs.push(IoSlice::new(entry));
193
194            total_len += HEADER_SIZE + len;
195            next_seq += 1;
196
197            // append index
198            if (next_seq - self.file_seq) % INDEX_INTERVAL == 0 {
199                let offset = self.data_file_size + total_len;
200                append_index(&mut self.current_index, offset as u64)?;
201            }
202        }
203
204        // fix headers in buf
205        for (i, header) in headers.iter().enumerate() {
206            bufs[i * 2] = IoSlice::new(&header[..]);
207        }
208
209        // finally, write into file
210        let writen_len = self.current_data.write_vectored(&bufs)?;
211        if writen_len == 0 {
212            return Err(std::io::Error::from(std::io::ErrorKind::WriteZero).into());
213        }
214        if writen_len < total_len {
215            // truncate the new data
216            self.current_data.set_len(self.data_file_size as u64)?;
217            return Err(std::io::Error::new(std::io::ErrorKind::WriteZero, "partitial").into());
218        }
219
220        // update status
221        self.data_file_size += total_len;
222        self.state.next_seq.store(next_seq, Ordering::Release);
223
224        Ok(())
225    }
226
227    /// Remove files containing only entries before the sequence number.
228    ///
229    /// If a data file contains entries both before and after the sequence
230    /// number, it will not be removed.
231    ///
232    /// If a data file is in read by any reader, it will not be removed.
233    ///
234    /// The current file is never removed.
235    pub fn purge(&mut self, seq: u64) -> Result<()> {
236        let mut file_seqs = self.state.file_seqs.write().unwrap();
237
238        let mut until = file_seqs.len() - 1;
239        for (i, file_seq) in file_seqs.iter().enumerate() {
240            if **file_seq > seq {
241                until = i.saturating_sub(1);
242                break;
243            }
244            if Arc::strong_count(file_seq) > 1 {
245                until = i;
246                break;
247            }
248        }
249
250        for file_seq in file_seqs.drain(..until) {
251            fs::remove_file(data_file_name(&self.state.dir, *file_seq))?;
252            fs::remove_file(index_file_name(&self.state.dir, *file_seq))?;
253        }
254        Ok(())
255    }
256
257    /// Truncate entries from the sequence number, inclusive.
258    ///
259    /// This does not check if any reader are reading these entries.
260    pub fn truncate(&mut self, seq: u64) -> Result<()> {
261        if seq >= self.next_seq() {
262            return Err(Error::SeqNotReached(seq, self.next_seq()));
263        }
264
265        // is the seq in current file?
266        if seq < self.file_seq {
267            let mut file_seqs = self.state.file_seqs.write().unwrap();
268
269            // we have to keep one file at least, so return error if
270            // it would remove all files
271            let Some(i) = file_seqs.iter().rposition(|f| **f <= seq) else {
272                return Err(Error::SeqPurged(seq, *file_seqs[0]));
273            };
274
275            // remove some files
276            for file_seq in file_seqs.drain(i + 1..) {
277                fs::remove_file(data_file_name(&self.state.dir, *file_seq))?;
278                fs::remove_file(index_file_name(&self.state.dir, *file_seq))?;
279            }
280
281            let file_seq = *file_seqs[i];
282
283            // open new data file and index file
284            let fname = data_file_name(&self.state.dir, file_seq);
285            self.current_data = File::options().append(true).read(true).open(fname)?;
286
287            let fname = index_file_name(&self.state.dir, file_seq);
288            self.current_index = File::options().append(true).read(true).open(fname)?;
289
290            self.data_file_size = 0; // reset later
291            self.file_seq = file_seq;
292
293            *self.state.current_dup.lock().unwrap() = Some(self.current_data.try_clone()?);
294        }
295
296        // remove entries in current data file
297        let mut data_buf = Vec::new();
298        let (mut file, data_pos) =
299            seek_seq_in_file(&self.state.dir, self.file_seq, seq, &mut data_buf)?;
300        self.data_file_size = file.stream_position()? as usize - (data_buf.len() - data_pos);
301        self.current_data.set_len(self.data_file_size as u64)?;
302        self.current_data.seek(SeekFrom::End(0))?;
303
304        // remove indexes in current index file
305        // TODO check index
306        let index_file_size = (seq - self.file_seq) / INDEX_INTERVAL * INDEX_SIZE as u64;
307        self.current_index.set_len(index_file_size)?;
308        self.current_index.seek(SeekFrom::End(0))?;
309
310        // update shared state
311        self.state.next_seq.store(seq, Ordering::Relaxed);
312        self.state.sync_seq.store(seq, Ordering::Relaxed);
313
314        Ok(())
315    }
316
317    /// Return the next sequence number.
318    pub fn next_seq(&self) -> u64 {
319        self.state.next_seq()
320    }
321
322    /// Return the next sequence number to synchronize to disk.
323    ///
324    /// This is the synchronization version of [`Self::next_seq`].
325    pub fn sync_seq(&self) -> u64 {
326        self.state.sync_seq()
327    }
328
329    /// Synchronizes entries to disk.
330    ///
331    /// Syncing data to disk is a relatively slow and blocking operation.
332    /// If you do not want to block the current thread, create a
333    /// [`SeqLogSyncer`] via [`Self::syncer`] and send it to another thread
334    /// to perform synchronization there.
335    pub fn sync(&self) -> Result<()> {
336        let seq = self.next_seq();
337
338        self.current_data.sync_data()?;
339
340        self.state.sync_seq.store(seq, Ordering::Release);
341        Ok(())
342    }
343
344    /// Create a [`SeqLogSyncer`].
345    pub fn syncer(&self) -> Result<SeqLogSyncer> {
346        Ok(SeqLogSyncer {
347            state: self.state.clone(),
348            current: self.current_data.try_clone()?,
349        })
350    }
351
352    /// Create a new [`SeqLogReader`] with the sequence number.
353    ///
354    /// If `synced_only` is set to `true`, the reader will only read entries
355    /// that have been synchronized to disk via [`Self::sync`] or [`SeqLogSyncer::sync`].
356    pub fn reader(&self, next_seq: u64, synced_only: bool) -> Result<SeqLogReader> {
357        let mut data_buf = Vec::new();
358
359        let (current, file_seq, data_pos) =
360            self.state.seek_seq(next_seq, &mut data_buf, synced_only)?;
361
362        Ok(SeqLogReader {
363            state: self.state.clone(),
364            data_buf,
365
366            synced_only,
367
368            current,
369            file_seq,
370            data_pos,
371            next_seq,
372        })
373    }
374
375    /// Configrate rotation.
376    ///
377    /// `size`: Rotate a data file when it exceeds this size. The default value
378    /// is 1G. Setting to 0 means never rotating, and you can call [`Self::rotate`]
379    /// to rotate manaully.
380    ///
381    /// `count`: Number of files to retain. The default value is 10. Setting to
382    /// 0 means keeping all files, and you can call [`Self::purge`] to delete
383    /// manually.
384    pub fn set_rotate(&mut self, size: usize, count: usize) {
385        self.rotate_size = if size == 0 { usize::MAX } else { size };
386        self.rotate_count = if count == 0 { usize::MAX } else { count };
387    }
388
389    /// Rotate the data file manually, e.g. by time or sequence number.
390    ///
391    /// It does not rotate if the current file is empty.
392    ///
393    /// You should have called [`Self::set_rotate`] with `size=0` to disable
394    /// the automatic rotation.
395    pub fn rotate(&mut self) -> Result<()> {
396        if self.data_file_size == 0 {
397            // do not rotate if current file is empty
398            return Ok(());
399        }
400
401        let next_seq = self.next_seq();
402
403        // open new file
404        self.current_data = File::create_new(data_file_name(&self.state.dir, next_seq))?;
405        self.current_index = File::create_new(index_file_name(&self.state.dir, next_seq))?;
406        self.data_file_size = 0;
407        self.file_seq = next_seq;
408
409        // save new file info
410        let mut file_seqs = self.state.file_seqs.write().unwrap();
411        file_seqs.push(Arc::new(next_seq));
412
413        // expire
414        while file_seqs.len() > self.rotate_count {
415            if Arc::strong_count(&file_seqs[0]) > 0 {
416                // this file is in used
417                break;
418            }
419            let file_seq = file_seqs.remove(0);
420            fs::remove_file(data_file_name(&self.state.dir, *file_seq))?;
421            fs::remove_file(index_file_name(&self.state.dir, *file_seq))?;
422        }
423
424        // notify the syncer
425        *self.state.current_dup.lock().unwrap() = Some(self.current_data.try_clone()?);
426
427        Ok(())
428    }
429}
430
431/// A sequential reader for scanning entries.
432///
433/// Compared with key-value stores, SeqLog has two characteristics for reading:
434///
435/// 1. Seeking to an arbitrary sequence number is not that fast bacause
436///    it uses sparse index;
437/// 2. Once positioned, sequential reads are very efficient because entries
438///    are stored in sequence order.
439///
440/// This access pattern is similar to a hard disk (HDD).
441/// A `SeqLogReader` acts like the disk read head. It is positioned at the
442/// specific sequence number when created (by [`SeqLog::reader`]).
443/// Subsequent reads (by [`Self::next`]) continue sequentially from that
444/// position.
445///
446/// Each reader maintains its own read position. Multiple readers can
447/// coexist and scan the log independently.
448///
449/// The reader can be sent between threads.
450pub struct SeqLogReader {
451    state: Arc<SharedState>,
452
453    synced_only: bool,
454
455    current: File,
456    file_seq: Arc<u64>,
457    data_buf: Vec<u8>,
458    data_pos: usize,
459    next_seq: u64,
460}
461
462impl SeqLogReader {
463    /// Return the next entry.
464    pub fn next(&mut self) -> Result<Option<&[u8]>> {
465        // check if end
466        let max_seq = if self.synced_only {
467            self.state.sync_seq()
468        } else {
469            self.state.next_seq()
470        };
471        if self.next_seq >= max_seq {
472            return Ok(None);
473        }
474
475        // now there must be an entry
476
477        // if any in the current buffer
478        let len = match parse_entry_len(&self.data_buf[self.data_pos..]) {
479            Some(len) => len,
480            None => {
481                // or read more data
482                self.read_more_data()?;
483                parse_entry_len(&self.data_buf)
484                    .ok_or(Error::DataFileTruncated(*self.file_seq, self.next_seq))?
485            }
486        };
487
488        let entry = &self.data_buf[self.data_pos..];
489        let payload = &entry[HEADER_SIZE..HEADER_SIZE + len];
490
491        // check checksum
492        let chsum1 = parse_checksum(&entry[LEN_SIZE..]);
493        let chsum2 = hash(&payload) as u16;
494        if chsum1 != chsum2 {
495            return Err(Error::ChecksumMismatch(*self.file_seq, self.next_seq));
496        }
497
498        self.next_seq += 1;
499        self.data_pos += HEADER_SIZE + len;
500        return Ok(Some(payload));
501    }
502
503    // We have checked the seq, so there must be one entry at least.
504    // Return Error if can not read more data.
505    fn read_more_data(&mut self) -> Result<()> {
506        let remain_len = self.data_buf.len() - self.data_pos;
507        self.data_buf.copy_within(self.data_pos.., 0);
508
509        self.data_buf.resize(READBUF_SIZE, 0);
510
511        let mut len = self.current.read(&mut self.data_buf[remain_len..])?;
512
513        if len == 0 {
514            // end of current file, open new file
515
516            if remain_len != 0 {
517                return Err(Error::DataFileTruncated(*self.file_seq, self.next_seq));
518            }
519
520            let file_seqs = self.state.file_seqs.read().unwrap();
521            let Some(file_seq) = file_seqs.iter().rev().find(|&f| **f == self.next_seq) else {
522                return Err(Error::DataFileTruncated(self.next_seq, self.next_seq));
523            };
524
525            self.file_seq = file_seq.clone();
526            self.current = File::open(data_file_name(&self.state.dir, **file_seq))?;
527
528            len = self.current.read(&mut self.data_buf)?;
529            if len == 0 {
530                return Err(Error::DataFileTruncated(*self.file_seq, self.next_seq));
531            }
532        }
533
534        self.data_buf.truncate(remain_len + len);
535        self.data_pos = 0;
536        Ok(())
537    }
538
539    /// Return the next sequence number.
540    pub fn next_seq(&self) -> u64 {
541        self.next_seq
542    }
543
544    /// Reset the sequence number. Then [`Self::next`] will return
545    /// the entry with this sequence number later.
546    ///
547    /// This is equvalent to [`SeqLog::reader`] to create a new reader.
548    /// This is useful only if you can not access the SeqLog instance.
549    pub fn reset(&mut self, seq: u64) -> Result<()> {
550        let (file, file_seq, data_pos) =
551            self.state
552                .seek_seq(seq, &mut self.data_buf, self.synced_only)?;
553
554        // update
555        self.data_pos = data_pos;
556        self.file_seq = file_seq;
557        self.current = file;
558        self.next_seq = seq;
559        Ok(())
560    }
561}
562
563/// A cross-thread synchronizing handler.
564///
565/// Unlike [`SeqLog::sync`], a `SeqLogSyncer` can be sent to a different thread
566/// and used to perform synchronization there. This allows disk sync operations
567/// to be offloaded from the writer thread, avoiding stalls caused by slow and
568/// blocking storage operations.
569pub struct SeqLogSyncer {
570    state: Arc<SharedState>,
571
572    current: File,
573}
574
575impl SeqLogSyncer {
576    /// Synchronize data to disk.
577    pub fn sync(&mut self) -> Result<()> {
578        // load next_seq before sync()
579        let mut next_seq = self.state.next_seq();
580
581        self.current.sync_data()?;
582
583        // check new file rotated
584        if let Some(new_file) = self.state.current_dup.lock().unwrap().take() {
585            next_seq = self.state.next_seq();
586            self.current = new_file;
587            self.current.sync_data()?;
588        }
589
590        // store sync_seq after sync()
591        self.state.sync_seq.store(next_seq, Ordering::Release);
592        Ok(())
593    }
594}
595
596impl SharedState {
597    fn seek_seq(
598        &self,
599        seq: u64,
600        data_buf: &mut Vec<u8>,
601        synced_only: bool,
602    ) -> Result<(File, Arc<u64>, usize)> {
603        let max_seq = if synced_only {
604            self.sync_seq()
605        } else {
606            self.next_seq()
607        };
608        if seq > max_seq {
609            return Err(Error::SeqNotReached(seq, max_seq));
610        }
611
612        // locate the file
613        let file_seqs = self.file_seqs.read().unwrap();
614        let Some(file_seq) = file_seqs.iter().rev().find(|&f| **f <= seq) else {
615            return Err(Error::SeqPurged(seq, *file_seqs[0]));
616        };
617
618        // lock the file, so the rotate() will not remove this
619        let file_seq = file_seq.clone();
620
621        // unlock ASAP
622        drop(file_seqs);
623
624        // locate the entry in the file
625        let (file, data_pos) = seek_seq_in_file(&self.dir, *file_seq, seq, data_buf)?;
626
627        Ok((file, file_seq, data_pos))
628    }
629
630    fn next_seq(&self) -> u64 {
631        self.next_seq.load(Ordering::Acquire)
632    }
633
634    fn sync_seq(&self) -> u64 {
635        self.sync_seq.load(Ordering::Acquire)
636    }
637}
638
639fn seek_seq_in_file(
640    dir: &Path,
641    file_seq: u64,
642    seq: u64,
643    data_buf: &mut Vec<u8>,
644) -> Result<(File, usize)> {
645    let mut data_file = File::open(data_file_name(dir, file_seq))?;
646
647    // seek by index
648    let diff_seq = seq - file_seq;
649    if diff_seq >= INDEX_INTERVAL {
650        let mut index_file = File::open(index_file_name(dir, file_seq))?;
651        index_file.seek(SeekFrom::Start(
652            (diff_seq / INDEX_INTERVAL - 1) * INDEX_SIZE as u64,
653        ))?;
654
655        let mut index_buf = [0; INDEX_SIZE];
656        index_file.read_exact(&mut index_buf)?;
657
658        let offset = parse_index(&index_buf);
659
660        // seek the data file to the offset
661        data_file.seek(SeekFrom::Start(offset))?;
662    }
663
664    // read data file
665    data_buf.resize(READBUF_SIZE, 0);
666    let len = data_file.read(data_buf)?;
667    data_buf.truncate(len);
668
669    let mut count = diff_seq % INDEX_INTERVAL;
670    if count == 0 {
671        return Ok((data_file, 0));
672    }
673
674    // walk the entries
675    loop {
676        let mut data_pos = 0;
677        while let Some(len) = parse_entry_len(&data_buf[data_pos..]) {
678            data_pos += HEADER_SIZE + len;
679            count -= 1;
680            if count == 0 {
681                return Ok((data_file, data_pos));
682            }
683        }
684
685        if data_buf.len() < READBUF_SIZE {
686            return Err(Error::DataFileTruncated(file_seq, seq));
687        }
688
689        data_buf.copy_within(data_pos.., 0);
690
691        let len = data_file.read(&mut data_buf[READBUF_SIZE - data_pos..])?;
692        if len == 0 {
693            return Err(Error::DataFileTruncated(file_seq, seq));
694        }
695        data_buf.truncate(READBUF_SIZE - data_pos + len);
696    }
697}
698
699// list all files and return the seqs
700fn list_files(dir: &Path) -> Result<Vec<Arc<u64>>> {
701    let mut files = Vec::new();
702
703    for entry in fs::read_dir(dir)? {
704        let path = entry?.path().to_path_buf();
705        let fname = path
706            .file_name()
707            .expect("invalid filename")
708            .to_str()
709            .expect("fail to parse filename");
710        if !fname.ends_with(DATA_SUFFIX) {
711            continue;
712        }
713        let Ok(seq) = fname[..fname.len() - DATA_SUFFIX.len()].parse() else {
714            return Err(Error::InvalidDataFilename(fname.to_string()));
715        };
716
717        files.push(Arc::new(seq));
718    }
719
720    files.sort();
721
722    Ok(files)
723}
724
725fn check_current_file(
726    data_file: &mut File,
727    index_file: &mut File,
728    file_seq: u64,
729) -> Result<(u64, u64)> {
730    let index_file_len = index_file.metadata()?.len() as usize;
731    if index_file_len % INDEX_SIZE != 0 {
732        return Err(Error::InvalidIndexFile(file_seq));
733    }
734
735    // read the last index, if any
736    let mut count = 0;
737    let mut offset = 0;
738    if index_file_len != 0 {
739        // read the index, get the offset
740        let mut index_buf = [0; INDEX_SIZE];
741        index_file.seek(SeekFrom::End(-(INDEX_SIZE as i64)))?;
742        index_file.read_exact(&mut index_buf)?;
743        offset = parse_index(&index_buf);
744
745        // seek the data file to the offset
746        data_file.seek(SeekFrom::Start(offset))?;
747
748        count = (index_file_len / INDEX_SIZE) as u64 * INDEX_INTERVAL;
749    }
750
751    // read data
752    let mut data_buf = Vec::new();
753    data_file.read_to_end(&mut data_buf)?;
754
755    // walk entries to the end
756    let mut entry = &data_buf[..];
757    while let Some(len) = parse_entry_len(entry) {
758        entry = &entry[HEADER_SIZE + len..];
759
760        count += 1;
761
762        offset += (HEADER_SIZE + len) as u64;
763        if count % INDEX_INTERVAL == 0 {
764            append_index(index_file, offset)?;
765        }
766    }
767
768    Ok((count, offset))
769}
770
771// append index to index file
772fn append_index(file: &mut File, offset: u64) -> Result<()> {
773    Ok(file.write_all(&offset.to_le_bytes())?)
774}
775
776// build data file name
777fn data_file_name(dir: &Path, seq: u64) -> PathBuf {
778    dir.join(format!("{:020}{}", seq, DATA_SUFFIX))
779}
780
781// build index file name
782fn index_file_name(dir: &Path, seq: u64) -> PathBuf {
783    dir.join(format!("{:020}{}", seq, INDEX_SUFFIX))
784}
785
786// return None if not complete
787fn parse_entry_len(buf: &[u8]) -> Option<usize> {
788    if buf.len() < HEADER_SIZE {
789        return None;
790    }
791    let len = u16::from_le_bytes(buf[0..LEN_SIZE].try_into().unwrap()) as usize;
792    if buf.len() < HEADER_SIZE + len {
793        return None;
794    }
795    Some(len)
796}
797
798fn parse_checksum(buf: &[u8]) -> u16 {
799    u16::from_le_bytes(buf[0..CHSUM_SIZE].try_into().unwrap())
800}
801
802fn parse_index(buf: &[u8]) -> u64 {
803    u64::from_le_bytes(buf[..8].try_into().unwrap())
804}