use std::fs::{self, File, OpenOptions};
use std::io::{BufWriter, Write};
use std::path::{Path, PathBuf};
use crate::error::{MemdError, Result};
use super::format::{PayloadIndexRecord, SegmentMeta, SEGMENT_MAGIC};
pub struct SegmentWriter {
id: u64,
dir: PathBuf,
payload_writer: BufWriter<File>,
payload_len: u64,
index_records: Vec<PayloadIndexRecord>,
meta: SegmentMeta,
}
impl SegmentWriter {
pub fn create(base_dir: &Path, segment_id: u64) -> Result<Self> {
let dir_name = format!("seg_{:06}", segment_id);
let dir = base_dir.join(&dir_name);
fs::create_dir_all(&dir).map_err(|e| {
MemdError::StorageError(format!(
"failed to create segment directory {}: {}",
dir.display(),
e
))
})?;
let payload_path = dir.join("payload.bin");
let payload_file = OpenOptions::new()
.create(true)
.write(true)
.truncate(true)
.open(&payload_path)
.map_err(|e| {
MemdError::StorageError(format!(
"failed to create payload file {}: {}",
payload_path.display(),
e
))
})?;
let meta = SegmentMeta::new(segment_id);
Ok(Self {
id: segment_id,
dir,
payload_writer: BufWriter::new(payload_file),
payload_len: 0,
index_records: Vec::new(),
meta,
})
}
pub fn append_chunk(&mut self, data: &[u8]) -> Result<u32> {
let offset = self.payload_len;
let length = data.len() as u32;
let crc32 = crc32fast::hash(data);
self.payload_writer
.write_all(data)
.map_err(|e| MemdError::StorageError(format!("failed to write payload: {}", e)))?;
self.payload_len += length as u64;
let record = PayloadIndexRecord::new(offset, length, crc32);
self.index_records.push(record);
Ok((self.index_records.len() - 1) as u32)
}
pub fn flush_payload(&mut self) -> Result<()> {
self.payload_writer
.flush()
.map_err(|e| MemdError::StorageError(format!("failed to flush payload: {}", e)))?;
self.payload_writer
.get_ref()
.sync_data()
.map_err(|e| MemdError::StorageError(format!("failed to fsync payload: {}", e)))?;
Ok(())
}
pub fn finalize(mut self) -> Result<SegmentMeta> {
self.payload_writer
.flush()
.map_err(|e| MemdError::StorageError(format!("failed to flush payload: {}", e)))?;
self.payload_writer
.get_ref()
.sync_all()
.map_err(|e| MemdError::StorageError(format!("failed to sync payload: {}", e)))?;
self.write_index_file()?;
self.meta.chunk_count = self.index_records.len() as u32;
self.meta.finalized = true;
self.write_meta_file()?;
Self::sync_directory_path(&self.dir)?;
Ok(self.meta)
}
fn write_index_file(&self) -> Result<()> {
let index_path = self.dir.join("payload.idx");
let mut file = File::create(&index_path).map_err(|e| {
MemdError::StorageError(format!(
"failed to create index file {}: {}",
index_path.display(),
e
))
})?;
file.write_all(SEGMENT_MAGIC)
.map_err(|e| MemdError::StorageError(format!("failed to write magic: {}", e)))?;
for record in &self.index_records {
record.write_to(&mut file).map_err(|e| {
MemdError::StorageError(format!("failed to write index record: {}", e))
})?;
}
file.sync_all()
.map_err(|e| MemdError::StorageError(format!("failed to sync index: {}", e)))?;
Ok(())
}
fn write_meta_file(&self) -> Result<()> {
let meta_path = self.dir.join("meta");
let encoded = bincode::serde::encode_to_vec(&self.meta, bincode::config::standard())
.map_err(|e| MemdError::StorageError(format!("failed to encode meta: {}", e)))?;
let mut file = File::create(&meta_path).map_err(|e| {
MemdError::StorageError(format!(
"failed to create meta file {}: {}",
meta_path.display(),
e
))
})?;
file.write_all(&encoded)
.map_err(|e| MemdError::StorageError(format!("failed to write meta: {}", e)))?;
file.sync_all()
.map_err(|e| MemdError::StorageError(format!("failed to sync meta: {}", e)))?;
Ok(())
}
fn sync_directory_path(dir: &Path) -> Result<()> {
let dir_file = File::open(dir).map_err(|e| {
MemdError::StorageError(format!(
"failed to open directory for sync {}: {}",
dir.display(),
e
))
})?;
dir_file
.sync_all()
.map_err(|e| MemdError::StorageError(format!("failed to sync directory: {}", e)))?;
Ok(())
}
pub fn id(&self) -> u64 {
self.id
}
pub fn dir(&self) -> &Path {
&self.dir
}
pub fn chunk_count(&self) -> usize {
self.index_records.len()
}
pub fn payload_size(&self) -> u64 {
self.payload_len
}
pub fn read_chunk(&mut self, ordinal: u32) -> Result<Option<Vec<u8>>> {
let idx = ordinal as usize;
if idx >= self.index_records.len() {
return Ok(None);
}
self.payload_writer
.flush()
.map_err(|e| MemdError::StorageError(format!("flush for read: {}", e)))?;
let record = &self.index_records[idx];
let payload_path = self.dir.join("payload.bin");
let payload_data = std::fs::read(&payload_path).map_err(|e| {
MemdError::StorageError(format!("read payload for chunk {}: {}", ordinal, e))
})?;
let start = record.offset as usize;
let end = start + record.length as usize;
if end > payload_data.len() {
return Err(MemdError::StorageError(format!(
"chunk {} offset out of bounds: {} > {}",
ordinal,
end,
payload_data.len()
)));
}
let data = &payload_data[start..end];
let computed_crc = crc32fast::hash(data);
if computed_crc != record.crc32 {
return Err(MemdError::StorageError(format!(
"chunk {} CRC mismatch: expected {:08x}, got {:08x}",
ordinal, record.crc32, computed_crc
)));
}
Ok(Some(data.to_vec()))
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::fs;
#[test]
fn write_single_chunk() {
let temp_dir = tempfile::tempdir().unwrap();
let base_dir = temp_dir.path();
let mut writer = SegmentWriter::create(base_dir, 1).unwrap();
let ordinal = writer.append_chunk(b"hello world").unwrap();
assert_eq!(ordinal, 0);
let meta = writer.finalize().unwrap();
assert_eq!(meta.id, 1);
assert_eq!(meta.chunk_count, 1);
assert!(meta.finalized);
let seg_dir = base_dir.join("seg_000001");
assert!(seg_dir.join("payload.bin").exists());
assert!(seg_dir.join("payload.idx").exists());
assert!(seg_dir.join("meta").exists());
let payload = fs::read(seg_dir.join("payload.bin")).unwrap();
assert_eq!(payload, b"hello world");
let index = fs::read(seg_dir.join("payload.idx")).unwrap();
assert_eq!(index.len(), 20);
assert_eq!(&index[0..4], SEGMENT_MAGIC);
}
#[test]
fn write_multiple_chunks() {
let temp_dir = tempfile::tempdir().unwrap();
let base_dir = temp_dir.path();
let mut writer = SegmentWriter::create(base_dir, 42).unwrap();
let ord1 = writer.append_chunk(b"first").unwrap();
let ord2 = writer.append_chunk(b"second").unwrap();
let ord3 = writer.append_chunk(b"third").unwrap();
assert_eq!(ord1, 0);
assert_eq!(ord2, 1);
assert_eq!(ord3, 2);
assert_eq!(writer.chunk_count(), 3);
let meta = writer.finalize().unwrap();
assert_eq!(meta.chunk_count, 3);
let seg_dir = base_dir.join("seg_000042");
let index = fs::read(seg_dir.join("payload.idx")).unwrap();
assert_eq!(index.len(), 52);
let payload = fs::read(seg_dir.join("payload.bin")).unwrap();
assert_eq!(payload, b"firstsecondthird");
}
#[test]
fn crc_integrity() {
let temp_dir = tempfile::tempdir().unwrap();
let base_dir = temp_dir.path();
let test_data = b"test data for CRC verification";
let expected_crc = crc32fast::hash(test_data);
let mut writer = SegmentWriter::create(base_dir, 1).unwrap();
writer.append_chunk(test_data).unwrap();
writer.finalize().unwrap();
let seg_dir = base_dir.join("seg_000001");
let index_bytes = fs::read(seg_dir.join("payload.idx")).unwrap();
let record = PayloadIndexRecord::from_bytes(&index_bytes[4..]).unwrap();
assert_eq!(record.offset, 0);
assert_eq!(record.length, test_data.len() as u32);
assert_eq!(record.crc32, expected_crc);
let payload = fs::read(seg_dir.join("payload.bin")).unwrap();
let recomputed_crc = crc32fast::hash(&payload);
assert_eq!(record.crc32, recomputed_crc);
}
#[test]
fn segment_directory_naming() {
let temp_dir = tempfile::tempdir().unwrap();
let base_dir = temp_dir.path();
let _ = SegmentWriter::create(base_dir, 1).unwrap();
let _ = SegmentWriter::create(base_dir, 999999).unwrap();
let _ = SegmentWriter::create(base_dir, 1000000).unwrap();
assert!(base_dir.join("seg_000001").exists());
assert!(base_dir.join("seg_999999").exists());
assert!(base_dir.join("seg_1000000").exists());
}
#[test]
fn meta_file_readable() {
let temp_dir = tempfile::tempdir().unwrap();
let base_dir = temp_dir.path();
let mut writer = SegmentWriter::create(base_dir, 7).unwrap();
writer.append_chunk(b"data").unwrap();
let original_meta = writer.finalize().unwrap();
let seg_dir = base_dir.join("seg_000007");
let meta_bytes = fs::read(seg_dir.join("meta")).unwrap();
let (decoded_meta, _): (SegmentMeta, _) =
bincode::serde::decode_from_slice(&meta_bytes, bincode::config::standard()).unwrap();
assert_eq!(original_meta.id, decoded_meta.id);
assert_eq!(original_meta.chunk_count, decoded_meta.chunk_count);
assert_eq!(original_meta.finalized, decoded_meta.finalized);
}
}