use std::fs::File;
use std::io;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use memmap2::{Mmap, MmapOptions};
use parking_lot::RwLock;
use super::format::{PayloadIndexRecord, SegmentMeta, SEGMENT_MAGIC};
use crate::store::TombstoneSet;
pub struct SegmentReader {
pub id: u64,
dir: PathBuf,
payload_mmap: Mmap,
index: Vec<PayloadIndexRecord>,
tombstones: Arc<RwLock<TombstoneSet>>,
}
impl SegmentReader {
pub fn open(dir: PathBuf) -> io::Result<Self> {
let id = Self::parse_segment_id(&dir)?;
let payload_path = dir.join("payload.bin");
let payload_file = File::open(&payload_path)?;
let payload_mmap = unsafe { MmapOptions::new().map(&payload_file)? };
let index_path = dir.join("payload.idx");
let index = Self::load_index(&index_path)?;
let tombstone_path = dir.join("tombstone.bin");
let tombstones = TombstoneSet::load_or_create(tombstone_path)
.map_err(|e| io::Error::other(e.to_string()))?;
Ok(Self {
id,
dir,
payload_mmap,
index,
tombstones: Arc::new(RwLock::new(tombstones)),
})
}
fn parse_segment_id(dir: &Path) -> io::Result<u64> {
let name = dir
.file_name()
.and_then(|n| n.to_str())
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "invalid segment dir"))?;
if !name.starts_with("seg_") {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
format!("invalid segment name: {}", name),
));
}
name[4..].parse().map_err(|_| {
io::Error::new(
io::ErrorKind::InvalidInput,
format!("invalid segment id: {}", name),
)
})
}
fn load_index(path: &Path) -> io::Result<Vec<PayloadIndexRecord>> {
let bytes = std::fs::read(path)?;
if bytes.len() < SEGMENT_MAGIC.len() {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"index file too small",
));
}
if &bytes[..SEGMENT_MAGIC.len()] != SEGMENT_MAGIC {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"invalid index magic",
));
}
PayloadIndexRecord::parse_all(&bytes[SEGMENT_MAGIC.len()..])
}
pub fn chunk_count(&self) -> u32 {
self.index.len() as u32
}
pub fn active_count(&self) -> u32 {
self.chunk_count() - self.tombstones.read().deleted_count() as u32
}
pub fn read_chunk(&self, ordinal: u32) -> io::Result<Option<Vec<u8>>> {
if ordinal as usize >= self.index.len() {
return Ok(None);
}
if self.tombstones.read().is_deleted(ordinal) {
return Ok(None);
}
let record = &self.index[ordinal as usize];
let start = record.offset as usize;
let end = start + record.length as usize;
if end > self.payload_mmap.len() {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"segment {} ordinal {} offset out of bounds",
self.id, ordinal
),
));
}
let data = &self.payload_mmap[start..end];
let computed_crc = crc32fast::hash(data);
if computed_crc != record.crc32 {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"segment {} ordinal {} CRC mismatch: expected {:08x}, got {:08x}",
self.id, ordinal, record.crc32, computed_crc
),
));
}
Ok(Some(data.to_vec()))
}
pub fn mark_deleted(&self, ordinal: u32) -> io::Result<()> {
let mut tombstones = self.tombstones.write();
tombstones.mark_deleted(ordinal);
tombstones
.persist()
.map_err(|e| io::Error::other(e.to_string()))
}
pub fn dir(&self) -> &Path {
&self.dir
}
pub fn metadata(&self) -> io::Result<SegmentMeta> {
let meta_path = self.dir.join("meta");
SegmentMeta::load(&meta_path)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::store::segment::SegmentWriter;
use tempfile::tempdir;
#[test]
fn read_written_segment() {
let temp_dir = tempdir().unwrap();
let base_dir = temp_dir.path();
let mut writer = SegmentWriter::create(base_dir, 1).unwrap();
writer.append_chunk(b"hello").unwrap();
writer.append_chunk(b"world").unwrap();
writer.append_chunk(b"test data").unwrap();
writer.finalize().unwrap();
let seg_dir = base_dir.join("seg_000001");
let reader = SegmentReader::open(seg_dir).unwrap();
assert_eq!(reader.id, 1);
assert_eq!(reader.chunk_count(), 3);
assert_eq!(reader.active_count(), 3);
assert_eq!(reader.read_chunk(0).unwrap(), Some(b"hello".to_vec()));
assert_eq!(reader.read_chunk(1).unwrap(), Some(b"world".to_vec()));
assert_eq!(reader.read_chunk(2).unwrap(), Some(b"test data".to_vec()));
assert_eq!(reader.read_chunk(3).unwrap(), None);
assert_eq!(reader.read_chunk(100).unwrap(), None);
}
#[test]
fn tombstone_filtering() {
let temp_dir = tempdir().unwrap();
let base_dir = temp_dir.path();
let mut writer = SegmentWriter::create(base_dir, 2).unwrap();
writer.append_chunk(b"keep me").unwrap();
writer.append_chunk(b"delete me").unwrap();
writer.append_chunk(b"keep me too").unwrap();
writer.finalize().unwrap();
let seg_dir = base_dir.join("seg_000002");
let reader = SegmentReader::open(seg_dir.clone()).unwrap();
assert_eq!(reader.active_count(), 3);
reader.mark_deleted(1).unwrap();
assert_eq!(reader.active_count(), 2);
assert_eq!(reader.read_chunk(0).unwrap(), Some(b"keep me".to_vec()));
assert_eq!(reader.read_chunk(1).unwrap(), None); assert_eq!(reader.read_chunk(2).unwrap(), Some(b"keep me too".to_vec()));
let reader2 = SegmentReader::open(seg_dir).unwrap();
assert_eq!(reader2.active_count(), 2);
assert_eq!(reader2.read_chunk(1).unwrap(), None);
}
#[test]
fn crc_verification() {
let temp_dir = tempdir().unwrap();
let base_dir = temp_dir.path();
let mut writer = SegmentWriter::create(base_dir, 3).unwrap();
writer.append_chunk(b"verified data").unwrap();
writer.finalize().unwrap();
let seg_dir = base_dir.join("seg_000003");
let payload_path = seg_dir.join("payload.bin");
let mut payload = std::fs::read(&payload_path).unwrap();
payload[0] ^= 0xFF; std::fs::write(&payload_path, payload).unwrap();
let reader = SegmentReader::open(seg_dir).unwrap();
let result = reader.read_chunk(0);
assert!(result.is_err());
assert!(result.unwrap_err().to_string().contains("CRC mismatch"));
}
#[test]
fn parse_segment_id_variants() {
let dir = PathBuf::from("/tmp/seg_000001");
assert_eq!(SegmentReader::parse_segment_id(&dir).unwrap(), 1);
let dir = PathBuf::from("/tmp/seg_000042");
assert_eq!(SegmentReader::parse_segment_id(&dir).unwrap(), 42);
let dir = PathBuf::from("/tmp/seg_999999");
assert_eq!(SegmentReader::parse_segment_id(&dir).unwrap(), 999999);
let dir = PathBuf::from("/tmp/segment_001");
assert!(SegmentReader::parse_segment_id(&dir).is_err());
let dir = PathBuf::from("/tmp/seg_abc");
assert!(SegmentReader::parse_segment_id(&dir).is_err());
}
#[test]
fn metadata_access() {
let temp_dir = tempdir().unwrap();
let base_dir = temp_dir.path();
let mut writer = SegmentWriter::create(base_dir, 5).unwrap();
writer.append_chunk(b"data").unwrap();
writer.finalize().unwrap();
let seg_dir = base_dir.join("seg_000005");
let reader = SegmentReader::open(seg_dir).unwrap();
let meta = reader.metadata().unwrap();
assert_eq!(meta.id, 5);
assert_eq!(meta.chunk_count, 1);
assert!(meta.finalized);
}
#[test]
fn mark_deleted_is_visible_to_concurrent_reader() {
use std::sync::Arc;
let temp_dir = tempdir().unwrap();
let base_dir = temp_dir.path();
let mut writer = SegmentWriter::create(base_dir, 7).unwrap();
for i in 0..8u32 {
writer
.append_chunk(format!("chunk-{}", i).as_bytes())
.unwrap();
}
writer.finalize().unwrap();
let seg_dir = base_dir.join("seg_000007");
let reader = Arc::new(SegmentReader::open(seg_dir).unwrap());
let reader_a = Arc::clone(&reader);
let t_a = std::thread::spawn(move || {
reader_a.mark_deleted(3).unwrap();
});
t_a.join().unwrap();
let read_result = reader.read_chunk(3).unwrap();
assert!(
read_result.is_none(),
"concurrent reader must observe the tombstone after mark_deleted returned"
);
let ok = reader.read_chunk(5).unwrap();
assert_eq!(ok.as_deref(), Some(b"chunk-5".as_slice()));
}
}