use crate::compress::{validate_compress_type, COMPRESS_TYPE_ZSTD};
use crate::error::{Result, TmslError};
use crate::util::*;
use memmap2::MmapMut;
pub const DATA_HEADER_SIZE: u64 = 124;
pub const INDEX_HEADER_SIZE: u64 = 128;
pub const MAGIC: [u8; 4] = *b"TMSL";
pub const VERSION: u16 = 1;
pub const INDEX_VERSION: u16 = 2;
pub const FILE_TYPE_INDEX: u8 = 1;
pub const FILE_TYPE_DATA: u8 = 2;
const META_TYPE_CREATED_AT: u8 = 0x01; const META_TYPE_FILE_OFFSET: u8 = 0x02; const META_TYPE_FILE_SIZE: u8 = 0x03; const META_TYPE_COMPRESS_LEVEL: u8 = 0x04; const META_TYPE_COMPRESS_TYPE: u8 = 0x05; const META_TYPE_BASE_BLOCK_OFFSET: u8 = 0x06; const META_TYPE_INDEX_DATA_SIZE: u8 = 0x07;
const META_LENGTH_V1: u16 = 41;
const META_LENGTH_INDEX: u16 = META_LENGTH_V1 + 11 + 7;
const DATA_STATE_LENGTH_V1: u16 = 72;
const INDEX_STATE_LENGTH_V1: u16 = 8;
const OFF_MAGIC: usize = 0;
const OFF_VERSION: usize = 4;
const OFF_FILE_TYPE: usize = 6;
const OFF_META_LENGTH: usize = 7;
const META_START: usize = 9;
pub const FIXED_PREFIX_SIZE: usize = META_START;
const STATE_LENGTH_SIZE: usize = 2;
#[cfg(test)]
const STATE_START: usize = 52;
const DS_MIN_TIMESTAMP: usize = 0;
const DS_MAX_TIMESTAMP: usize = 8;
const DS_WROTE_POSITION: usize = 16;
const DS_RECORD_COUNT: usize = 24;
const DS_TOTAL_UNCOMP_SIZE: usize = 32;
const DS_PENDING_BLOCK_OFFSET: usize = 40;
const DS_PENDING_WROTE_POSITION: usize = 48;
const DS_PENDING_RECORD_COUNT: usize = 56;
const DS_INVALID_RECORD_COUNT: usize = 64;
const IS_WROTE_POSITION: usize = 0;
pub const PENDING_NONE: u64 = u64::MAX;
pub const TIMESTAMP_MIN_SENTINEL: i64 = i64::MAX;
pub const TIMESTAMP_MAX_SENTINEL: i64 = i64::MIN;
fn state_length_offset(meta_length: u16) -> usize {
META_START + meta_length as usize
}
fn state_start(meta_length: u16) -> usize {
state_length_offset(meta_length) + STATE_LENGTH_SIZE
}
fn header_size(meta_length: u16, state_length: u16) -> u64 {
(state_start(meta_length) + state_length as usize) as u64
}
fn ensure_len(mmap: &[u8], end: usize, what: &str) -> Result<()> {
if mmap.len() < end {
return Err(TmslError::InvalidData(format!(
"truncated {}: need {} bytes, got {}",
what,
end,
mmap.len()
)));
}
Ok(())
}
fn validate_state_field(mmap: &[u8], expected_file_type: u8, field_offset: usize) -> Result<usize> {
let (_, _, file_type, meta_length) = read_shared_prefix(mmap)?;
if file_type != expected_file_type {
return Err(TmslError::InvalidData(format!(
"unexpected file_type {}",
file_type
)));
}
let state_len_off = state_length_offset(meta_length);
ensure_len(mmap, state_len_off + STATE_LENGTH_SIZE, "state_length")?;
let state_length = read_u16_from_mmap(mmap, state_len_off);
let state_start = state_start(meta_length);
let end = state_start + state_length as usize;
ensure_len(mmap, end, "state")?;
if field_offset + 8 > state_length as usize {
return Err(TmslError::InvalidData(format!(
"state field offset {} exceeds state_length {}",
field_offset, state_length
)));
}
Ok(state_start)
}
pub fn write_data_core_state_to_mmap(
mmap: &mut [u8],
min_timestamp: i64,
max_timestamp: i64,
wrote_position: u64,
record_count: u64,
total_uncompressed_size: u64,
) -> Result<()> {
let state_start = validate_state_field(mmap, FILE_TYPE_DATA, DS_TOTAL_UNCOMP_SIZE)?;
mmap[state_start + DS_MIN_TIMESTAMP..state_start + DS_MIN_TIMESTAMP + 8]
.copy_from_slice(&min_timestamp.to_le_bytes());
mmap[state_start + DS_MAX_TIMESTAMP..state_start + DS_MAX_TIMESTAMP + 8]
.copy_from_slice(&max_timestamp.to_le_bytes());
mmap[state_start + DS_WROTE_POSITION..state_start + DS_WROTE_POSITION + 8]
.copy_from_slice(&wrote_position.to_le_bytes());
mmap[state_start + DS_RECORD_COUNT..state_start + DS_RECORD_COUNT + 8]
.copy_from_slice(&record_count.to_le_bytes());
mmap[state_start + DS_TOTAL_UNCOMP_SIZE..state_start + DS_TOTAL_UNCOMP_SIZE + 8]
.copy_from_slice(&total_uncompressed_size.to_le_bytes());
Ok(())
}
pub fn write_data_pending_state_to_mmap(
mmap: &mut [u8],
pending_block_offset: u64,
pending_wrote_position: u64,
pending_record_count: u64,
) -> Result<()> {
let state_start = validate_state_field(mmap, FILE_TYPE_DATA, DS_PENDING_RECORD_COUNT)?;
mmap[state_start + DS_PENDING_BLOCK_OFFSET..state_start + DS_PENDING_BLOCK_OFFSET + 8]
.copy_from_slice(&pending_block_offset.to_le_bytes());
mmap[state_start + DS_PENDING_WROTE_POSITION..state_start + DS_PENDING_WROTE_POSITION + 8]
.copy_from_slice(&pending_wrote_position.to_le_bytes());
mmap[state_start + DS_PENDING_RECORD_COUNT..state_start + DS_PENDING_RECORD_COUNT + 8]
.copy_from_slice(&pending_record_count.to_le_bytes());
Ok(())
}
pub fn write_data_invalid_record_count_to_mmap(
mmap: &mut [u8],
invalid_record_count: u64,
) -> Result<()> {
let state_start = validate_state_field(mmap, FILE_TYPE_DATA, DS_INVALID_RECORD_COUNT)?;
let off = state_start + DS_INVALID_RECORD_COUNT;
mmap[off..off + 8].copy_from_slice(&invalid_record_count.to_le_bytes());
Ok(())
}
pub fn write_index_wrote_position_to_mmap(mmap: &mut [u8], wrote_position: u64) -> Result<()> {
let state_start = validate_state_field(mmap, FILE_TYPE_INDEX, IS_WROTE_POSITION)?;
let off = state_start + IS_WROTE_POSITION;
mmap[off..off + 8].copy_from_slice(&wrote_position.to_le_bytes());
Ok(())
}
#[derive(Debug, Clone)]
pub struct DataFileMetadata {
pub magic: [u8; 4],
pub version: u16,
pub file_type: u8,
pub created_at: i64,
pub file_offset: i64,
pub file_size: u64,
pub compress_level: u8,
pub compress_type: u8,
pub meta_length: u16,
pub header_size: u64,
pub min_timestamp: i64,
pub max_timestamp: i64,
pub wrote_position: u64,
pub record_count: u64,
pub total_uncompressed_size: u64,
pub pending_block_offset: u64, pub pending_wrote_position: u64,
pub pending_record_count: u64,
pub invalid_record_count: u64,
pub state_length: u16,
}
impl DataFileMetadata {
pub fn create_default(
file_offset: i64,
file_size: u64,
compress_level: u8,
compress_type: u8,
) -> Self {
Self {
magic: MAGIC,
version: VERSION,
file_type: FILE_TYPE_DATA,
created_at: std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_millis() as i64)
.unwrap_or(0),
file_offset,
file_size,
compress_level,
compress_type,
meta_length: META_LENGTH_V1,
header_size: DATA_HEADER_SIZE,
min_timestamp: TIMESTAMP_MIN_SENTINEL,
max_timestamp: TIMESTAMP_MAX_SENTINEL,
wrote_position: DATA_HEADER_SIZE,
record_count: 0,
total_uncompressed_size: 0,
pending_block_offset: PENDING_NONE,
pending_wrote_position: 0,
pending_record_count: 0,
invalid_record_count: 0,
state_length: DATA_STATE_LENGTH_V1,
}
}
fn state_start(&self) -> usize {
state_start(self.meta_length)
}
pub fn write_to(&self, mmap: &mut MmapMut) {
write_shared_prefix(
mmap,
self.magic,
self.version,
self.file_type,
self.meta_length,
);
write_meta_tlv(
mmap,
self.created_at,
self.file_offset,
self.file_size,
self.compress_level,
self.compress_type,
);
let state_length_offset = state_length_offset(self.meta_length);
let state_start = self.state_start();
write_u16_to_mmap(mmap, state_length_offset, self.state_length);
write_i64_to_mmap(mmap, state_start + DS_MIN_TIMESTAMP, self.min_timestamp);
write_i64_to_mmap(mmap, state_start + DS_MAX_TIMESTAMP, self.max_timestamp);
write_u64_to_mmap(mmap, state_start + DS_WROTE_POSITION, self.wrote_position);
write_u64_to_mmap(mmap, state_start + DS_RECORD_COUNT, self.record_count);
write_u64_to_mmap(
mmap,
state_start + DS_TOTAL_UNCOMP_SIZE,
self.total_uncompressed_size,
);
write_u64_to_mmap(
mmap,
state_start + DS_PENDING_BLOCK_OFFSET,
self.pending_block_offset,
);
write_u64_to_mmap(
mmap,
state_start + DS_PENDING_WROTE_POSITION,
self.pending_wrote_position,
);
write_u64_to_mmap(
mmap,
state_start + DS_PENDING_RECORD_COUNT,
self.pending_record_count,
);
write_u64_to_mmap(
mmap,
state_start + DS_INVALID_RECORD_COUNT,
self.invalid_record_count,
);
}
pub fn read_from(mmap: &[u8]) -> Result<Self> {
let (magic, version, file_type, meta_length) = read_shared_prefix(mmap)?;
let (created_at, file_offset, file_size, compress_level, compress_type) =
parse_meta_tlv(mmap, META_START, meta_length as usize)?;
validate_compress_type(compress_type)?;
let state_length_offset = state_length_offset(meta_length);
ensure_len(
mmap,
state_length_offset + STATE_LENGTH_SIZE,
"data state_length",
)?;
let state_length = read_u16_from_mmap(mmap, state_length_offset);
if state_length < DATA_STATE_LENGTH_V1 {
return Err(TmslError::InvalidData(format!(
"data state_length {} is smaller than v1 {}",
state_length, DATA_STATE_LENGTH_V1
)));
}
let state_start = state_start(meta_length);
let header_size = header_size(meta_length, state_length);
ensure_len(mmap, header_size as usize, "data header")?;
let min_timestamp = read_i64_from_mmap(mmap, state_start + DS_MIN_TIMESTAMP);
let max_timestamp = read_i64_from_mmap(mmap, state_start + DS_MAX_TIMESTAMP);
let wrote_position = read_u64_from_mmap(mmap, state_start + DS_WROTE_POSITION);
let record_count = read_u64_from_mmap(mmap, state_start + DS_RECORD_COUNT);
let total_uncompressed_size = read_u64_from_mmap(mmap, state_start + DS_TOTAL_UNCOMP_SIZE);
let pending_block_offset = read_u64_from_mmap(mmap, state_start + DS_PENDING_BLOCK_OFFSET);
let pending_wrote_position =
read_u64_from_mmap(mmap, state_start + DS_PENDING_WROTE_POSITION);
let pending_record_count = read_u64_from_mmap(mmap, state_start + DS_PENDING_RECORD_COUNT);
let invalid_record_count = read_u64_from_mmap(mmap, state_start + DS_INVALID_RECORD_COUNT);
Ok(Self {
magic,
version,
file_type,
created_at,
file_offset,
file_size,
compress_level,
compress_type,
meta_length,
header_size,
min_timestamp,
max_timestamp,
wrote_position,
record_count,
total_uncompressed_size,
pending_block_offset,
pending_wrote_position,
pending_record_count,
invalid_record_count,
state_length,
})
}
pub fn sync(&self, mmap: &mut MmapMut) -> Result<()> {
mmap.flush()?;
Ok(())
}
#[allow(clippy::too_many_arguments)]
pub fn write_state_full(
&mut self,
mmap: &mut MmapMut,
min_ts: i64,
max_ts: i64,
wrote_pos: u64,
rec_count: u64,
uncomp_size: u64,
) {
self.min_timestamp = min_ts;
self.max_timestamp = max_ts;
self.wrote_position = wrote_pos;
self.record_count = rec_count;
self.total_uncompressed_size = uncomp_size;
let state_start = self.state_start();
write_i64_to_mmap(mmap, state_start + DS_MIN_TIMESTAMP, self.min_timestamp);
write_i64_to_mmap(mmap, state_start + DS_MAX_TIMESTAMP, self.max_timestamp);
write_u64_to_mmap(mmap, state_start + DS_WROTE_POSITION, self.wrote_position);
write_u64_to_mmap(mmap, state_start + DS_RECORD_COUNT, self.record_count);
write_u64_to_mmap(
mmap,
state_start + DS_TOTAL_UNCOMP_SIZE,
self.total_uncompressed_size,
);
}
pub fn write_pending(&mut self, mmap: &mut MmapMut) {
let state_start = self.state_start();
write_u64_to_mmap(
mmap,
state_start + DS_PENDING_BLOCK_OFFSET,
self.pending_block_offset,
);
write_u64_to_mmap(
mmap,
state_start + DS_PENDING_WROTE_POSITION,
self.pending_wrote_position,
);
write_u64_to_mmap(
mmap,
state_start + DS_PENDING_RECORD_COUNT,
self.pending_record_count,
);
}
pub fn clear_pending(&mut self, mmap: &mut MmapMut) {
self.pending_block_offset = PENDING_NONE;
self.pending_wrote_position = 0;
self.pending_record_count = 0;
let state_start = self.state_start();
write_u64_to_mmap(mmap, state_start + DS_PENDING_BLOCK_OFFSET, PENDING_NONE);
write_u64_to_mmap(mmap, state_start + DS_PENDING_WROTE_POSITION, 0);
write_u64_to_mmap(mmap, state_start + DS_PENDING_RECORD_COUNT, 0);
}
}
#[derive(Debug, Clone)]
pub struct IndexFileMetadata {
pub magic: [u8; 4],
pub version: u16,
pub file_type: u8,
pub created_at: i64,
pub file_offset: i64,
pub file_size: u64,
pub compress_level: u8,
pub compress_type: u8,
pub base_block_offset: u64,
pub index_data_size: u32,
pub meta_length: u16,
pub header_size: u64,
pub wrote_position: u64,
pub state_length: u16,
}
impl IndexFileMetadata {
pub fn create_default(
file_offset: i64,
file_size: u64,
compress_level: u8,
compress_type: u8,
base_block_offset: u64,
index_data_size: u32,
) -> Self {
Self {
magic: MAGIC,
version: INDEX_VERSION,
file_type: FILE_TYPE_INDEX,
created_at: std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_millis() as i64)
.unwrap_or(0),
file_offset,
file_size,
compress_level,
compress_type,
base_block_offset,
index_data_size,
meta_length: META_LENGTH_INDEX,
header_size: INDEX_HEADER_SIZE,
wrote_position: INDEX_HEADER_SIZE,
state_length: INDEX_STATE_LENGTH_V1,
}
}
fn state_start(&self) -> usize {
state_start(self.meta_length)
}
pub fn write_to(&self, mmap: &mut MmapMut) {
write_shared_prefix(
mmap,
self.magic,
self.version,
self.file_type,
self.meta_length,
);
write_index_meta_tlv(
mmap,
self.created_at,
self.file_offset,
self.file_size,
self.compress_level,
self.compress_type,
self.base_block_offset,
self.index_data_size,
);
let state_length_offset = state_length_offset(self.meta_length);
let state_start = self.state_start();
write_u16_to_mmap(mmap, state_length_offset, self.state_length);
write_u64_to_mmap(mmap, state_start + IS_WROTE_POSITION, self.wrote_position);
}
pub fn read_from(mmap: &[u8]) -> Result<Self> {
let (magic, version, file_type, meta_length) = read_shared_prefix(mmap)?;
if version != INDEX_VERSION {
return Err(TmslError::InvalidData(format!(
"unsupported index file version {}, expected {}",
version, INDEX_VERSION
)));
}
if file_type != FILE_TYPE_INDEX {
return Err(TmslError::InvalidData(format!(
"unsupported index file_type {file_type}, expected {FILE_TYPE_INDEX}"
)));
}
let index_meta = parse_index_meta_tlv(mmap, META_START, meta_length as usize)?;
let created_at = index_meta.created_at;
let file_offset = index_meta.file_offset;
let file_size = index_meta.file_size;
let compress_level = index_meta.compress_level;
let compress_type = index_meta.compress_type;
let base_block_offset = index_meta.base_block_offset;
let index_data_size = index_meta.index_data_size;
validate_compress_type(compress_type)?;
let state_length_offset = state_length_offset(meta_length);
ensure_len(
mmap,
state_length_offset + STATE_LENGTH_SIZE,
"index state_length",
)?;
let state_length = read_u16_from_mmap(mmap, state_length_offset);
if state_length < INDEX_STATE_LENGTH_V1 {
return Err(TmslError::InvalidData(format!(
"index state_length {} is smaller than v1 {}",
state_length, INDEX_STATE_LENGTH_V1
)));
}
let state_start = state_start(meta_length);
let dynamic_header_size = header_size(meta_length, state_length);
if dynamic_header_size > INDEX_HEADER_SIZE {
return Err(TmslError::InvalidData(format!(
"index header content length {} exceeds reserved entry start {}",
dynamic_header_size, INDEX_HEADER_SIZE
)));
}
let header_size = INDEX_HEADER_SIZE;
ensure_len(mmap, header_size as usize, "index header reserved area")?;
let stride = crate::config::index_entry_stride(index_data_size)?;
let min_capacity = INDEX_HEADER_SIZE + stride as u64;
if file_size < min_capacity {
return Err(TmslError::InvalidData(format!(
"index file_size {file_size} cannot hold header {INDEX_HEADER_SIZE} plus one {stride}-byte entry (minimum {min_capacity})"
)));
}
let wrote_position = read_u64_from_mmap(mmap, state_start + IS_WROTE_POSITION);
Ok(Self {
magic,
version,
file_type,
created_at,
file_offset,
file_size,
compress_level,
compress_type,
base_block_offset,
index_data_size,
meta_length,
header_size,
wrote_position,
state_length,
})
}
pub fn sync(&self, mmap: &mut MmapMut) -> Result<()> {
mmap.flush()?;
Ok(())
}
pub fn write_wrote_position(&mut self, mmap: &mut MmapMut, wrote_pos: u64) {
self.wrote_position = wrote_pos;
let state_start = self.state_start();
write_u64_to_mmap(mmap, state_start + IS_WROTE_POSITION, self.wrote_position);
}
}
fn write_shared_prefix(
mmap: &mut MmapMut,
magic: [u8; 4],
version: u16,
file_type: u8,
meta_length: u16,
) {
mmap[OFF_MAGIC..OFF_MAGIC + 4].copy_from_slice(&magic);
write_u16_to_mmap(mmap, OFF_VERSION, version);
mmap[OFF_FILE_TYPE] = file_type;
write_u16_to_mmap(mmap, OFF_META_LENGTH, meta_length);
}
fn write_meta_tlv(
mmap: &mut MmapMut,
created_at: i64,
file_offset: i64,
file_size: u64,
compress_level: u8,
compress_type: u8,
) {
let mut off = META_START;
mmap[off] = META_TYPE_CREATED_AT;
off += 1;
write_u16_to_mmap(mmap, off, 8);
off += 2;
write_i64_to_mmap(mmap, off, created_at);
off += 8;
mmap[off] = META_TYPE_FILE_OFFSET;
off += 1;
write_u16_to_mmap(mmap, off, 8);
off += 2;
write_i64_to_mmap(mmap, off, file_offset);
off += 8;
mmap[off] = META_TYPE_FILE_SIZE;
off += 1;
write_u16_to_mmap(mmap, off, 8);
off += 2;
write_u64_to_mmap(mmap, off, file_size);
off += 8;
mmap[off] = META_TYPE_COMPRESS_LEVEL;
off += 1;
write_u16_to_mmap(mmap, off, 1);
off += 2;
mmap[off] = compress_level;
off += 1;
mmap[off] = META_TYPE_COMPRESS_TYPE;
off += 1;
write_u16_to_mmap(mmap, off, 1);
off += 2;
mmap[off] = compress_type;
}
#[allow(clippy::too_many_arguments)]
fn write_index_meta_tlv(
mmap: &mut MmapMut,
created_at: i64,
file_offset: i64,
file_size: u64,
compress_level: u8,
compress_type: u8,
base_block_offset: u64,
index_data_size: u32,
) {
write_meta_tlv(
mmap,
created_at,
file_offset,
file_size,
compress_level,
compress_type,
);
let mut off = META_START + META_LENGTH_V1 as usize;
mmap[off] = META_TYPE_BASE_BLOCK_OFFSET;
off += 1;
write_u16_to_mmap(mmap, off, 8);
off += 2;
write_u64_to_mmap(mmap, off, base_block_offset);
off += 8;
mmap[off] = META_TYPE_INDEX_DATA_SIZE;
off += 1;
write_u16_to_mmap(mmap, off, 4);
off += 2;
write_u32_to_mmap(mmap, off, index_data_size);
}
struct IndexMetaTlv {
created_at: i64,
file_offset: i64,
file_size: u64,
compress_level: u8,
compress_type: u8,
base_block_offset: u64,
index_data_size: u32,
}
fn parse_index_meta_tlv(mmap: &[u8], start: usize, total_len: usize) -> Result<IndexMetaTlv> {
let mut created_at = 0i64;
let mut file_offset = 0i64;
let mut file_size = 0u64;
let mut compress_level = 0u8;
let mut compress_type = COMPRESS_TYPE_ZSTD;
let mut base_block_offset: Option<u64> = None;
let mut index_data_size: Option<u32> = None;
let mut off = start;
let end = start
.checked_add(total_len)
.ok_or_else(|| TmslError::InvalidData("index meta length overflow".into()))?;
ensure_len(mmap, end, "index meta tlv")?;
while off + 3 <= end {
let t = mmap[off];
off += 1;
let len = read_u16_from_mmap(mmap, off) as usize;
off += 2;
if off + len > end {
return Err(TmslError::InvalidData(
"index meta TLV entry exceeds meta length".into(),
));
}
match t {
META_TYPE_CREATED_AT if len == 8 => {
created_at = read_i64_from_mmap(mmap, off);
}
META_TYPE_FILE_OFFSET if len == 8 => {
file_offset = read_i64_from_mmap(mmap, off);
}
META_TYPE_FILE_SIZE if len == 8 => {
file_size = read_u64_from_mmap(mmap, off);
}
META_TYPE_COMPRESS_LEVEL if len == 1 => {
compress_level = mmap[off];
}
META_TYPE_COMPRESS_TYPE if len == 1 => {
compress_type = mmap[off];
}
META_TYPE_BASE_BLOCK_OFFSET => {
if len != 8 {
return Err(TmslError::InvalidData(format!(
"index base_block_offset TLV must be 8 bytes, got {len}"
)));
}
if base_block_offset.is_some() {
return Err(TmslError::InvalidData(
"index has duplicate base_block_offset TLV".into(),
));
}
base_block_offset = Some(read_u64_from_mmap(mmap, off));
}
META_TYPE_INDEX_DATA_SIZE => {
if len != 4 {
return Err(TmslError::InvalidData(format!(
"index index_data_size TLV must be 4 bytes, got {len}"
)));
}
if index_data_size.is_some() {
return Err(TmslError::InvalidData(
"index has duplicate index_data_size TLV".into(),
));
}
index_data_size = Some(read_u32_from_mmap(mmap, off));
}
_ => {} }
off += len;
}
if off != end {
return Err(TmslError::InvalidData(
"index meta TLV data has trailing partial entry".into(),
));
}
let base_block_offset = base_block_offset
.ok_or_else(|| TmslError::InvalidData("index missing base_block_offset".into()))?;
let index_data_size = index_data_size
.ok_or_else(|| TmslError::InvalidData("index missing index_data_size".into()))?;
Ok(IndexMetaTlv {
created_at,
file_offset,
file_size,
compress_level,
compress_type,
base_block_offset,
index_data_size,
})
}
fn read_shared_prefix(mmap: &[u8]) -> Result<([u8; 4], u16, u8, u16)> {
ensure_len(mmap, FIXED_PREFIX_SIZE, "fixed header prefix")?;
let magic: [u8; 4] = mmap[OFF_MAGIC..OFF_MAGIC + 4]
.try_into()
.map_err(|_| TmslError::InvalidData("cannot read magic".into()))?;
if magic != MAGIC {
return Err(TmslError::InvalidMagic);
}
let version = read_u16_from_mmap(mmap, OFF_VERSION);
if version > INDEX_VERSION {
log::warn!(
"File has version {} but we know {}, parsing known fields only",
version,
INDEX_VERSION,
);
}
let file_type = mmap[OFF_FILE_TYPE];
let meta_length = read_u16_from_mmap(mmap, OFF_META_LENGTH);
Ok((magic, version, file_type, meta_length))
}
fn parse_meta_tlv(mmap: &[u8], start: usize, total_len: usize) -> Result<(i64, i64, u64, u8, u8)> {
let mut created_at = 0i64;
let mut file_offset = 0i64;
let mut file_size = 0u64;
let mut compress_level = 0u8;
let mut compress_type = COMPRESS_TYPE_ZSTD;
let mut off = start;
let end = start
.checked_add(total_len)
.ok_or_else(|| TmslError::InvalidData("meta length overflow".into()))?;
ensure_len(mmap, end, "meta tlv")?;
while off + 3 <= end {
let t = mmap[off];
off += 1;
let len = read_u16_from_mmap(mmap, off) as usize;
off += 2;
if off + len > end {
break;
}
match t {
META_TYPE_CREATED_AT if len == 8 => {
created_at = read_i64_from_mmap(mmap, off);
}
META_TYPE_FILE_OFFSET if len == 8 => {
file_offset = read_i64_from_mmap(mmap, off);
}
META_TYPE_FILE_SIZE if len == 8 => {
file_size = read_u64_from_mmap(mmap, off);
}
META_TYPE_COMPRESS_LEVEL if len == 1 => {
compress_level = mmap[off];
}
META_TYPE_COMPRESS_TYPE if len == 1 => {
compress_type = mmap[off];
}
_ => {} }
off += len;
}
Ok((
created_at,
file_offset,
file_size,
compress_level,
compress_type,
))
}
pub fn clear_pending_from_mmap(mmap: &mut [u8]) -> Result<()> {
write_data_pending_state_to_mmap(mmap, PENDING_NONE, 0, 0)
}
#[cfg(test)]
mod tests {
use super::*;
use memmap2::MmapMut;
use std::fs::OpenOptions;
use std::path::PathBuf;
fn create_test_mmap(size: u64) -> (MmapMut, PathBuf) {
let dir = std::env::temp_dir().join("timslite_test_header");
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join(format!(
"test_header_{:?}.dat",
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
let file = OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(true)
.open(&path)
.unwrap();
file.set_len(size).unwrap();
file.sync_all().unwrap();
let mmap = unsafe { MmapMut::map_mut(&file).unwrap() };
(mmap, path)
}
#[test]
fn test_data_write_read_roundtrip() {
let (mut mmap, _path) = create_test_mmap(DATA_HEADER_SIZE);
let meta = DataFileMetadata::create_default(
0,
64 * 1024 * 1024,
6,
crate::compress::COMPRESS_TYPE_ZSTD,
);
meta.write_to(&mut mmap);
let read = DataFileMetadata::read_from(&mmap).unwrap();
assert_eq!(read.magic, MAGIC);
assert_eq!(read.version, VERSION);
assert_eq!(read.file_type, FILE_TYPE_DATA);
assert_eq!(read.file_offset, 0);
assert_eq!(read.file_size, 64 * 1024 * 1024);
assert_eq!(read.compress_type, crate::compress::COMPRESS_TYPE_ZSTD);
assert_eq!(read.pending_block_offset, PENDING_NONE);
assert_eq!(read.min_timestamp, TIMESTAMP_MIN_SENTINEL);
assert_eq!(read.max_timestamp, TIMESTAMP_MAX_SENTINEL);
}
#[test]
fn test_header_file_size_roundtrips_u64() {
let (mut mmap, _path) = create_test_mmap(DATA_HEADER_SIZE);
let large_size = u32::MAX as u64 + 4096;
let meta =
DataFileMetadata::create_default(0, large_size, 6, crate::compress::COMPRESS_TYPE_ZSTD);
meta.write_to(&mut mmap);
let read = DataFileMetadata::read_from(&mmap).unwrap();
assert_eq!(read.file_size, large_size);
}
#[test]
fn test_header_rejects_invalid_compress_type() {
let (mut mmap, _path) = create_test_mmap(DATA_HEADER_SIZE);
let meta = DataFileMetadata::create_default(0, 4096, 6, 99);
meta.write_to(&mut mmap);
let result = DataFileMetadata::read_from(&mmap);
assert!(matches!(result, Err(TmslError::InvalidData(_))));
}
#[test]
fn test_index_segment_roundtrip() {
let (mut mmap, _path) = create_test_mmap(INDEX_HEADER_SIZE);
let meta = IndexFileMetadata::create_default(
1700000000,
4 * 1024 * 1024,
6,
crate::compress::COMPRESS_TYPE_ZSTD,
0x1234_5678_9ABC,
16,
);
meta.write_to(&mut mmap);
let read = IndexFileMetadata::read_from(&mmap).unwrap();
assert_eq!(read.file_type, FILE_TYPE_INDEX);
assert_eq!(read.file_offset, 1700000000);
assert_eq!(read.wrote_position, INDEX_HEADER_SIZE);
assert_eq!(read.base_block_offset, 0x1234_5678_9ABC);
assert_eq!(read.index_data_size, 16);
}
#[test]
fn test_index_entries_start_at_reserved_128_bytes() {
let meta = IndexFileMetadata::create_default(
1700000000,
4 * 1024 * 1024,
6,
crate::compress::COMPRESS_TYPE_ZSTD,
0,
0,
);
assert_eq!(meta.header_size, 128);
assert_eq!(meta.wrote_position, 128);
assert_eq!(INDEX_HEADER_SIZE, 128);
}
#[test]
fn test_data_header_reads_state_after_extended_meta() {
let extra_meta = [0xEE, 4, 0, 1, 2, 3, 4];
let meta_length = META_LENGTH_V1 + extra_meta.len() as u16;
let state_length_offset = META_START + meta_length as usize;
let state_start = state_length_offset + 2;
let header_size = state_start as u64 + DATA_STATE_LENGTH_V1 as u64;
let (mut mmap, _path) = create_test_mmap(header_size);
write_shared_prefix(&mut mmap, MAGIC, VERSION, FILE_TYPE_DATA, meta_length);
write_meta_tlv(
&mut mmap,
1000,
0,
1024,
6,
crate::compress::COMPRESS_TYPE_ZSTD,
);
let extra_start = META_START + META_LENGTH_V1 as usize;
mmap[extra_start..extra_start + extra_meta.len()].copy_from_slice(&extra_meta);
write_u16_to_mmap(&mut mmap, state_length_offset, DATA_STATE_LENGTH_V1);
write_i64_to_mmap(&mut mmap, state_start + DS_MIN_TIMESTAMP, 11);
write_i64_to_mmap(&mut mmap, state_start + DS_MAX_TIMESTAMP, 22);
write_u64_to_mmap(&mut mmap, state_start + DS_WROTE_POSITION, header_size);
write_u64_to_mmap(&mut mmap, state_start + DS_RECORD_COUNT, 2);
write_u64_to_mmap(&mut mmap, state_start + DS_TOTAL_UNCOMP_SIZE, 44);
write_u64_to_mmap(
&mut mmap,
state_start + DS_PENDING_BLOCK_OFFSET,
PENDING_NONE,
);
write_u64_to_mmap(&mut mmap, state_start + DS_PENDING_WROTE_POSITION, 0);
write_u64_to_mmap(&mut mmap, state_start + DS_PENDING_RECORD_COUNT, 0);
write_u64_to_mmap(&mut mmap, state_start + DS_INVALID_RECORD_COUNT, 3);
let read = DataFileMetadata::read_from(&mmap).unwrap();
assert_eq!(read.meta_length, meta_length);
assert_eq!(read.state_length, DATA_STATE_LENGTH_V1);
assert_eq!(read.min_timestamp, 11);
assert_eq!(read.max_timestamp, 22);
assert_eq!(read.wrote_position, header_size);
assert_eq!(read.record_count, 2);
assert_eq!(read.invalid_record_count, 3);
}
#[test]
fn test_index_header_reads_state_after_extended_meta() {
let extra_meta = [0xEF, 3, 0, 7, 8, 9];
let meta_length = META_LENGTH_INDEX + extra_meta.len() as u16;
let state_length_offset = META_START + meta_length as usize;
let state_start = state_length_offset + 2;
let dynamic_header_size = state_start as u64 + INDEX_STATE_LENGTH_V1 as u64;
let (mut mmap, _path) = create_test_mmap(INDEX_HEADER_SIZE);
write_shared_prefix(
&mut mmap,
MAGIC,
INDEX_VERSION,
FILE_TYPE_INDEX,
meta_length,
);
write_index_meta_tlv(
&mut mmap,
1000,
1700000000,
4096,
6,
crate::compress::COMPRESS_TYPE_ZSTD,
0,
0,
);
let extra_start = META_START + META_LENGTH_INDEX as usize;
mmap[extra_start..extra_start + extra_meta.len()].copy_from_slice(&extra_meta);
write_u16_to_mmap(&mut mmap, state_length_offset, INDEX_STATE_LENGTH_V1);
write_u64_to_mmap(
&mut mmap,
state_start + IS_WROTE_POSITION,
INDEX_HEADER_SIZE,
);
let read = IndexFileMetadata::read_from(&mmap).unwrap();
assert_eq!(read.meta_length, meta_length);
assert_eq!(read.state_length, INDEX_STATE_LENGTH_V1);
assert_eq!(read.file_offset, 1700000000);
assert!(dynamic_header_size < INDEX_HEADER_SIZE);
assert_eq!(read.header_size, INDEX_HEADER_SIZE);
assert_eq!(read.wrote_position, INDEX_HEADER_SIZE);
}
#[test]
fn test_index_header_uses_bumped_index_version() {
let (mut mmap, _path) = create_test_mmap(INDEX_HEADER_SIZE);
let meta = IndexFileMetadata::create_default(
1700000000,
4 * 1024 * 1024,
6,
crate::compress::COMPRESS_TYPE_ZSTD,
0,
0,
);
meta.write_to(&mut mmap);
assert_ne!(INDEX_VERSION, VERSION);
assert_eq!(read_u16_from_mmap(&mmap, OFF_VERSION), INDEX_VERSION);
let read = IndexFileMetadata::read_from(&mmap).unwrap();
assert_eq!(read.version, INDEX_VERSION);
}
#[test]
fn test_invalid_magic() {
let (mut mmap, _path) = create_test_mmap(DATA_HEADER_SIZE);
mmap[0..4].copy_from_slice(b"XXXX");
let result = DataFileMetadata::read_from(&mmap);
assert!(result.is_err());
match result.unwrap_err() {
TmslError::InvalidMagic => {}
other => panic!("expected InvalidMagic, got: {other:?}"),
}
}
#[test]
fn test_data_update_state() {
let (mut mmap, _path) = create_test_mmap(DATA_HEADER_SIZE);
let mut meta = DataFileMetadata::create_default(
0,
64 * 1024 * 1024,
6,
crate::compress::COMPRESS_TYPE_ZSTD,
);
meta.write_to(&mut mmap);
meta.write_state_full(&mut mmap, 100, 200, 256, 10, 2000);
assert_eq!(
read_i64_from_mmap(&mmap, STATE_START + DS_MIN_TIMESTAMP),
100
);
assert_eq!(
read_i64_from_mmap(&mmap, STATE_START + DS_MAX_TIMESTAMP),
200
);
assert_eq!(
read_u64_from_mmap(&mmap, STATE_START + DS_WROTE_POSITION),
256
);
assert_eq!(read_u64_from_mmap(&mmap, STATE_START + DS_RECORD_COUNT), 10);
}
#[test]
fn test_data_pending_state() {
let (mut mmap, _path) = create_test_mmap(DATA_HEADER_SIZE);
let mut meta = DataFileMetadata::create_default(
0,
64 * 1024 * 1024,
6,
crate::compress::COMPRESS_TYPE_ZSTD,
);
meta.write_to(&mut mmap);
meta.pending_block_offset = 128;
meta.pending_wrote_position = 1000;
meta.pending_record_count = 5;
meta.write_pending(&mut mmap);
assert_eq!(
read_u64_from_mmap(&mmap, STATE_START + DS_PENDING_BLOCK_OFFSET),
128
);
assert_eq!(
read_u64_from_mmap(&mmap, STATE_START + DS_PENDING_WROTE_POSITION),
1000
);
assert_eq!(
read_u64_from_mmap(&mmap, STATE_START + DS_PENDING_RECORD_COUNT),
5
);
meta.clear_pending(&mut mmap);
assert_eq!(
read_u64_from_mmap(&mmap, STATE_START + DS_PENDING_BLOCK_OFFSET),
PENDING_NONE
);
}
#[test]
fn test_index_update_wrote_position() {
let (mut mmap, _path) = create_test_mmap(INDEX_HEADER_SIZE);
let mut meta = IndexFileMetadata::create_default(
0,
4 * 1024 * 1024,
6,
crate::compress::COMPRESS_TYPE_ZSTD,
0,
0,
);
meta.write_to(&mut mmap);
meta.write_wrote_position(
&mut mmap,
INDEX_HEADER_SIZE + crate::index::INDEX_ENTRY_SIZE as u64,
);
assert_eq!(
read_u64_from_mmap(&mmap, state_start(META_LENGTH_INDEX) + IS_WROTE_POSITION),
INDEX_HEADER_SIZE + crate::index::INDEX_ENTRY_SIZE as u64
);
}
#[test]
fn test_future_version_parse() {
let (mut mmap, _path) = create_test_mmap(DATA_HEADER_SIZE);
let meta = DataFileMetadata::create_default(
0,
64 * 1024 * 1024,
6,
crate::compress::COMPRESS_TYPE_ZSTD,
);
meta.write_to(&mut mmap);
write_u16_to_mmap(&mut mmap, OFF_VERSION, 5);
let read = DataFileMetadata::read_from(&mmap).unwrap();
assert_eq!(read.version, 5);
assert_eq!(read.file_type, FILE_TYPE_DATA);
}
#[test]
fn test_header_sizes() {
assert_eq!(DATA_HEADER_SIZE, 124);
assert_eq!(INDEX_HEADER_SIZE, 128);
}
#[test]
fn test_data_read_from_invalid_magic_returns_invalid_magic() {
let (mut mmap, _path) = create_test_mmap(DATA_HEADER_SIZE);
let meta =
DataFileMetadata::create_default(0, 4096, 6, crate::compress::COMPRESS_TYPE_ZSTD);
meta.write_to(&mut mmap);
mmap[0..4].copy_from_slice(b"JUNK");
let result = DataFileMetadata::read_from(&mmap);
assert!(matches!(result, Err(TmslError::InvalidMagic)));
}
#[test]
fn test_data_read_from_truncated_buffer_returns_invalid_data() {
let buf = [0u8; 4];
let result = DataFileMetadata::read_from(&buf);
assert!(result.is_err());
}
#[test]
fn test_data_read_from_truncated_meta_tlv() {
let (mut mmap, _path) = create_test_mmap(DATA_HEADER_SIZE);
let meta =
DataFileMetadata::create_default(0, 4096, 6, crate::compress::COMPRESS_TYPE_ZSTD);
meta.write_to(&mut mmap);
let short = &mmap[..FIXED_PREFIX_SIZE + 5];
let result = DataFileMetadata::read_from(short);
assert!(result.is_err());
}
#[test]
fn test_data_read_from_state_length_too_small() {
let (mut mmap, _path) = create_test_mmap(DATA_HEADER_SIZE);
let meta =
DataFileMetadata::create_default(0, 4096, 6, crate::compress::COMPRESS_TYPE_ZSTD);
meta.write_to(&mut mmap);
let state_len_off = state_length_offset(META_LENGTH_V1);
write_u16_to_mmap(&mut mmap, state_len_off, 0);
let result = DataFileMetadata::read_from(&mmap);
assert!(result.is_err());
match result.unwrap_err() {
TmslError::InvalidData(msg) => {
assert!(msg.contains("state_length"), "msg: {}", msg);
}
other => panic!("expected InvalidData, got: {other:?}"),
}
}
#[test]
fn test_index_read_from_invalid_magic_returns_invalid_magic() {
let (mut mmap, _path) = create_test_mmap(INDEX_HEADER_SIZE);
let meta = IndexFileMetadata::create_default(
0,
4096,
6,
crate::compress::COMPRESS_TYPE_ZSTD,
0,
0,
);
meta.write_to(&mut mmap);
mmap[0..4].copy_from_slice(b"NOPE");
let result = IndexFileMetadata::read_from(&mmap);
assert!(matches!(result, Err(TmslError::InvalidMagic)));
}
#[test]
fn test_index_read_from_wrong_version_returns_error() {
let (mut mmap, _path) = create_test_mmap(INDEX_HEADER_SIZE);
let meta = IndexFileMetadata::create_default(
0,
4096,
6,
crate::compress::COMPRESS_TYPE_ZSTD,
0,
0,
);
meta.write_to(&mut mmap);
write_u16_to_mmap(&mut mmap, OFF_VERSION, 0);
let result = IndexFileMetadata::read_from(&mmap);
assert!(result.is_err());
match result.unwrap_err() {
TmslError::InvalidData(msg) => {
assert!(msg.contains("version"), "msg: {}", msg);
}
other => panic!("expected InvalidData, got: {other:?}"),
}
}
#[test]
fn test_index_read_from_truncated_buffer() {
let buf = [0u8; 5];
let result = IndexFileMetadata::read_from(&buf);
assert!(result.is_err());
}
#[test]
fn test_index_read_from_state_length_too_small() {
let (mut mmap, _path) = create_test_mmap(INDEX_HEADER_SIZE);
let meta = IndexFileMetadata::create_default(
0,
4096,
6,
crate::compress::COMPRESS_TYPE_ZSTD,
0,
0,
);
meta.write_to(&mut mmap);
let state_len_off = state_length_offset(META_LENGTH_INDEX);
write_u16_to_mmap(&mut mmap, state_len_off, 0);
let result = IndexFileMetadata::read_from(&mmap);
assert!(result.is_err());
}
#[test]
fn test_write_data_core_state_rejects_wrong_file_type() {
let (mut mmap, _path) = create_test_mmap(INDEX_HEADER_SIZE);
let meta = IndexFileMetadata::create_default(
0,
4096,
6,
crate::compress::COMPRESS_TYPE_ZSTD,
0,
0,
);
meta.write_to(&mut mmap);
let result = write_data_core_state_to_mmap(&mut mmap, 100, 200, 256, 10, 2000);
assert!(result.is_err());
match result.unwrap_err() {
TmslError::InvalidData(msg) => {
assert!(msg.contains("file_type"), "msg: {}", msg);
}
other => panic!("expected InvalidData, got: {other:?}"),
}
}
#[test]
fn test_write_index_wrote_position_rejects_wrong_file_type() {
let (mut mmap, _path) = create_test_mmap(DATA_HEADER_SIZE);
let meta =
DataFileMetadata::create_default(0, 4096, 6, crate::compress::COMPRESS_TYPE_ZSTD);
meta.write_to(&mut mmap);
let result = write_index_wrote_position_to_mmap(&mut mmap, 256);
assert!(result.is_err());
match result.unwrap_err() {
TmslError::InvalidData(msg) => {
assert!(msg.contains("file_type"), "msg: {}", msg);
}
other => panic!("expected InvalidData, got: {other:?}"),
}
}
#[test]
fn test_write_data_pending_state_rejects_wrong_file_type() {
let (mut mmap, _path) = create_test_mmap(INDEX_HEADER_SIZE);
let meta = IndexFileMetadata::create_default(
0,
4096,
6,
crate::compress::COMPRESS_TYPE_ZSTD,
0,
0,
);
meta.write_to(&mut mmap);
let result = write_data_pending_state_to_mmap(&mut mmap, 128, 1000, 5);
assert!(result.is_err());
}
#[test]
fn test_write_data_invalid_record_count_rejects_wrong_file_type() {
let (mut mmap, _path) = create_test_mmap(INDEX_HEADER_SIZE);
let meta = IndexFileMetadata::create_default(
0,
4096,
6,
crate::compress::COMPRESS_TYPE_ZSTD,
0,
0,
);
meta.write_to(&mut mmap);
let result = write_data_invalid_record_count_to_mmap(&mut mmap, 3);
assert!(result.is_err());
}
#[test]
fn test_write_data_core_state_roundtrip() {
let (mut mmap, _path) = create_test_mmap(DATA_HEADER_SIZE);
let meta =
DataFileMetadata::create_default(0, 4096, 6, crate::compress::COMPRESS_TYPE_ZSTD);
meta.write_to(&mut mmap);
write_data_core_state_to_mmap(&mut mmap, 100, 200, 256, 10, 2000).unwrap();
let read = DataFileMetadata::read_from(&mmap).unwrap();
assert_eq!(read.min_timestamp, 100);
assert_eq!(read.max_timestamp, 200);
assert_eq!(read.wrote_position, 256);
assert_eq!(read.record_count, 10);
assert_eq!(read.total_uncompressed_size, 2000);
}
#[test]
fn test_write_data_pending_state_roundtrip() {
let (mut mmap, _path) = create_test_mmap(DATA_HEADER_SIZE);
let meta =
DataFileMetadata::create_default(0, 4096, 6, crate::compress::COMPRESS_TYPE_ZSTD);
meta.write_to(&mut mmap);
write_data_pending_state_to_mmap(&mut mmap, 128, 1000, 5).unwrap();
let read = DataFileMetadata::read_from(&mmap).unwrap();
assert_eq!(read.pending_block_offset, 128);
assert_eq!(read.pending_wrote_position, 1000);
assert_eq!(read.pending_record_count, 5);
}
#[test]
fn test_write_data_invalid_record_count_roundtrip() {
let (mut mmap, _path) = create_test_mmap(DATA_HEADER_SIZE);
let meta =
DataFileMetadata::create_default(0, 4096, 6, crate::compress::COMPRESS_TYPE_ZSTD);
meta.write_to(&mut mmap);
write_data_invalid_record_count_to_mmap(&mut mmap, 42).unwrap();
let read = DataFileMetadata::read_from(&mmap).unwrap();
assert_eq!(read.invalid_record_count, 42);
}
#[test]
fn test_write_index_wrote_position_roundtrip() {
let (mut mmap, _path) = create_test_mmap(INDEX_HEADER_SIZE);
let meta = IndexFileMetadata::create_default(
0,
4096,
6,
crate::compress::COMPRESS_TYPE_ZSTD,
0,
0,
);
meta.write_to(&mut mmap);
let new_pos = INDEX_HEADER_SIZE + 128;
write_index_wrote_position_to_mmap(&mut mmap, new_pos).unwrap();
let read = IndexFileMetadata::read_from(&mmap).unwrap();
assert_eq!(read.wrote_position, new_pos);
}
#[test]
fn test_clear_pending_from_mmap_roundtrip() {
let (mut mmap, _path) = create_test_mmap(DATA_HEADER_SIZE);
let meta =
DataFileMetadata::create_default(0, 4096, 6, crate::compress::COMPRESS_TYPE_ZSTD);
meta.write_to(&mut mmap);
write_data_pending_state_to_mmap(&mut mmap, 128, 1000, 5).unwrap();
let read = DataFileMetadata::read_from(&mmap).unwrap();
assert_eq!(read.pending_block_offset, 128);
clear_pending_from_mmap(&mut mmap).unwrap();
let read = DataFileMetadata::read_from(&mmap).unwrap();
assert_eq!(read.pending_block_offset, PENDING_NONE);
assert_eq!(read.pending_wrote_position, 0);
assert_eq!(read.pending_record_count, 0);
}
#[test]
fn test_data_header_size_field_matches_constant() {
let meta =
DataFileMetadata::create_default(0, 4096, 6, crate::compress::COMPRESS_TYPE_ZSTD);
assert_eq!(meta.header_size, DATA_HEADER_SIZE);
assert_eq!(meta.header_size, 124);
}
#[test]
fn test_index_header_size_field_matches_constant() {
let meta = IndexFileMetadata::create_default(
0,
4096,
6,
crate::compress::COMPRESS_TYPE_ZSTD,
0,
0,
);
assert_eq!(meta.header_size, INDEX_HEADER_SIZE);
assert_eq!(meta.header_size, 128);
}
#[test]
fn test_data_header_size_preserved_after_read() {
let (mut mmap, _path) = create_test_mmap(DATA_HEADER_SIZE);
let meta =
DataFileMetadata::create_default(0, 4096, 6, crate::compress::COMPRESS_TYPE_ZSTD);
meta.write_to(&mut mmap);
let read = DataFileMetadata::read_from(&mmap).unwrap();
assert_eq!(read.header_size, DATA_HEADER_SIZE);
}
#[test]
fn test_index_header_size_preserved_after_read() {
let (mut mmap, _path) = create_test_mmap(INDEX_HEADER_SIZE);
let meta = IndexFileMetadata::create_default(
0,
4096,
6,
crate::compress::COMPRESS_TYPE_ZSTD,
0,
0,
);
meta.write_to(&mut mmap);
let read = IndexFileMetadata::read_from(&mmap).unwrap();
assert_eq!(read.header_size, INDEX_HEADER_SIZE);
}
#[test]
fn test_data_boundary_timestamps_min_max() {
let (mut mmap, _path) = create_test_mmap(DATA_HEADER_SIZE);
let meta =
DataFileMetadata::create_default(0, 4096, 6, crate::compress::COMPRESS_TYPE_ZSTD);
meta.write_to(&mut mmap);
write_data_core_state_to_mmap(&mut mmap, i64::MIN, i64::MAX, 0, 0, 0).unwrap();
let read = DataFileMetadata::read_from(&mmap).unwrap();
assert_eq!(read.min_timestamp, i64::MIN);
assert_eq!(read.max_timestamp, i64::MAX);
}
#[test]
fn test_data_boundary_timestamps_zero() {
let (mut mmap, _path) = create_test_mmap(DATA_HEADER_SIZE);
let meta =
DataFileMetadata::create_default(0, 4096, 6, crate::compress::COMPRESS_TYPE_ZSTD);
meta.write_to(&mut mmap);
write_data_core_state_to_mmap(&mut mmap, 0, 0, 0, 0, 0).unwrap();
let read = DataFileMetadata::read_from(&mmap).unwrap();
assert_eq!(read.min_timestamp, 0);
assert_eq!(read.max_timestamp, 0);
}
#[test]
fn test_data_boundary_wrote_position_max() {
let (mut mmap, _path) = create_test_mmap(DATA_HEADER_SIZE);
let meta =
DataFileMetadata::create_default(0, 4096, 6, crate::compress::COMPRESS_TYPE_ZSTD);
meta.write_to(&mut mmap);
write_data_core_state_to_mmap(&mut mmap, 0, 0, u64::MAX, u64::MAX, u64::MAX).unwrap();
let read = DataFileMetadata::read_from(&mmap).unwrap();
assert_eq!(read.wrote_position, u64::MAX);
assert_eq!(read.record_count, u64::MAX);
assert_eq!(read.total_uncompressed_size, u64::MAX);
}
#[test]
fn test_data_boundary_pending_fields_max() {
let (mut mmap, _path) = create_test_mmap(DATA_HEADER_SIZE);
let meta =
DataFileMetadata::create_default(0, 4096, 6, crate::compress::COMPRESS_TYPE_ZSTD);
meta.write_to(&mut mmap);
write_data_pending_state_to_mmap(&mut mmap, u64::MAX, u64::MAX, u64::MAX).unwrap();
let read = DataFileMetadata::read_from(&mmap).unwrap();
assert_eq!(read.pending_block_offset, u64::MAX);
assert_eq!(read.pending_wrote_position, u64::MAX);
assert_eq!(read.pending_record_count, u64::MAX);
}
#[test]
fn test_data_boundary_file_size_zero() {
let (mut mmap, _path) = create_test_mmap(DATA_HEADER_SIZE);
let meta = DataFileMetadata::create_default(0, 0, 6, crate::compress::COMPRESS_TYPE_ZSTD);
meta.write_to(&mut mmap);
let read = DataFileMetadata::read_from(&mmap).unwrap();
assert_eq!(read.file_size, 0);
}
#[test]
fn test_data_boundary_file_offset_negative() {
let (mut mmap, _path) = create_test_mmap(DATA_HEADER_SIZE);
let meta = DataFileMetadata::create_default(
i64::MIN,
4096,
6,
crate::compress::COMPRESS_TYPE_ZSTD,
);
meta.write_to(&mut mmap);
let read = DataFileMetadata::read_from(&mmap).unwrap();
assert_eq!(read.file_offset, i64::MIN);
}
#[test]
fn test_index_boundary_wrote_position_reset() {
let (mut mmap, _path) = create_test_mmap(INDEX_HEADER_SIZE);
let meta = IndexFileMetadata::create_default(
0,
4096,
6,
crate::compress::COMPRESS_TYPE_ZSTD,
0,
0,
);
meta.write_to(&mut mmap);
write_index_wrote_position_to_mmap(&mut mmap, INDEX_HEADER_SIZE).unwrap();
let read = IndexFileMetadata::read_from(&mmap).unwrap();
assert_eq!(read.wrote_position, INDEX_HEADER_SIZE);
}
#[test]
fn test_data_negative_timestamps() {
let (mut mmap, _path) = create_test_mmap(DATA_HEADER_SIZE);
let meta =
DataFileMetadata::create_default(0, 4096, 6, crate::compress::COMPRESS_TYPE_ZSTD);
meta.write_to(&mut mmap);
write_data_core_state_to_mmap(&mut mmap, -1000, -1, 0, 0, 0).unwrap();
let read = DataFileMetadata::read_from(&mmap).unwrap();
assert_eq!(read.min_timestamp, -1000);
assert_eq!(read.max_timestamp, -1);
}
#[test]
fn test_data_create_default_sentinel_values() {
let meta =
DataFileMetadata::create_default(0, 4096, 6, crate::compress::COMPRESS_TYPE_ZSTD);
assert_eq!(meta.min_timestamp, TIMESTAMP_MIN_SENTINEL);
assert_eq!(meta.max_timestamp, TIMESTAMP_MAX_SENTINEL);
assert_eq!(meta.pending_block_offset, PENDING_NONE);
assert_eq!(meta.wrote_position, DATA_HEADER_SIZE);
assert_eq!(meta.record_count, 0);
assert_eq!(meta.total_uncompressed_size, 0);
assert_eq!(meta.pending_wrote_position, 0);
assert_eq!(meta.pending_record_count, 0);
assert_eq!(meta.invalid_record_count, 0);
}
#[test]
fn test_index_create_default_values() {
let meta = IndexFileMetadata::create_default(
100,
4096,
6,
crate::compress::COMPRESS_TYPE_ZSTD,
0,
0,
);
assert_eq!(meta.file_offset, 100);
assert_eq!(meta.wrote_position, INDEX_HEADER_SIZE);
assert_eq!(meta.state_length, INDEX_STATE_LENGTH_V1);
assert_eq!(meta.meta_length, META_LENGTH_INDEX);
assert_eq!(meta.base_block_offset, 0);
assert_eq!(meta.index_data_size, 0);
}
#[test]
fn test_sentinel_values() {
assert_eq!(PENDING_NONE, u64::MAX);
assert_eq!(TIMESTAMP_MIN_SENTINEL, i64::MAX);
assert_eq!(TIMESTAMP_MAX_SENTINEL, i64::MIN);
}
#[test]
fn test_multiple_state_updates_last_wins() {
let (mut mmap, _path) = create_test_mmap(DATA_HEADER_SIZE);
let meta =
DataFileMetadata::create_default(0, 4096, 6, crate::compress::COMPRESS_TYPE_ZSTD);
meta.write_to(&mut mmap);
write_data_core_state_to_mmap(&mut mmap, 10, 20, 100, 5, 500).unwrap();
write_data_core_state_to_mmap(&mut mmap, 30, 40, 200, 10, 1000).unwrap();
let read = DataFileMetadata::read_from(&mmap).unwrap();
assert_eq!(read.min_timestamp, 30);
assert_eq!(read.max_timestamp, 40);
assert_eq!(read.wrote_position, 200);
assert_eq!(read.record_count, 10);
assert_eq!(read.total_uncompressed_size, 1000);
}
#[test]
fn test_pending_state_update_after_core_state() {
let (mut mmap, _path) = create_test_mmap(DATA_HEADER_SIZE);
let meta =
DataFileMetadata::create_default(0, 4096, 6, crate::compress::COMPRESS_TYPE_ZSTD);
meta.write_to(&mut mmap);
write_data_core_state_to_mmap(&mut mmap, 10, 20, 100, 5, 500).unwrap();
write_data_pending_state_to_mmap(&mut mmap, 64, 300, 2).unwrap();
let read = DataFileMetadata::read_from(&mmap).unwrap();
assert_eq!(read.min_timestamp, 10);
assert_eq!(read.max_timestamp, 20);
assert_eq!(read.wrote_position, 100);
assert_eq!(read.pending_block_offset, 64);
assert_eq!(read.pending_wrote_position, 300);
assert_eq!(read.pending_record_count, 2);
}
#[test]
fn test_data_compress_level_zero() {
let (mut mmap, _path) = create_test_mmap(DATA_HEADER_SIZE);
let meta =
DataFileMetadata::create_default(0, 4096, 0, crate::compress::COMPRESS_TYPE_ZSTD);
meta.write_to(&mut mmap);
let read = DataFileMetadata::read_from(&mmap).unwrap();
assert_eq!(read.compress_level, 0);
}
#[test]
fn test_data_compress_level_max() {
let (mut mmap, _path) = create_test_mmap(DATA_HEADER_SIZE);
let meta =
DataFileMetadata::create_default(0, 4096, 22, crate::compress::COMPRESS_TYPE_ZSTD);
meta.write_to(&mut mmap);
let read = DataFileMetadata::read_from(&mmap).unwrap();
assert_eq!(read.compress_level, 22);
}
#[test]
fn test_index_compress_type_roundtrip() {
let (mut mmap, _path) = create_test_mmap(INDEX_HEADER_SIZE);
let meta = IndexFileMetadata::create_default(
0,
4096,
6,
crate::compress::COMPRESS_TYPE_ZSTD,
0,
0,
);
meta.write_to(&mut mmap);
let read = IndexFileMetadata::read_from(&mmap).unwrap();
assert_eq!(read.compress_type, crate::compress::COMPRESS_TYPE_ZSTD);
assert_eq!(read.compress_level, 6);
}
#[test]
fn test_write_data_state_field_offset_exceeds_state_length() {
let (mut mmap, _path) = create_test_mmap(DATA_HEADER_SIZE);
let meta =
DataFileMetadata::create_default(0, 4096, 6, crate::compress::COMPRESS_TYPE_ZSTD);
meta.write_to(&mut mmap);
let state_len_off = state_length_offset(META_LENGTH_V1);
write_u16_to_mmap(&mut mmap, state_len_off, 16);
let result = write_data_core_state_to_mmap(&mut mmap, 0, 0, 0, 0, 0);
assert!(result.is_err());
match result.unwrap_err() {
TmslError::InvalidData(msg) => {
assert!(msg.contains("state field offset"), "msg: {}", msg);
}
other => panic!("expected InvalidData, got: {other:?}"),
}
}
const INDEX_BASE_TLV: usize = META_START + META_LENGTH_V1 as usize;
const INDEX_SIZE_TLV: usize = INDEX_BASE_TLV + 11;
fn write_valid_index_header(mmap: &mut MmapMut, index_data_size: u32) {
let meta = IndexFileMetadata::create_default(
1_000,
4096,
6,
crate::compress::COMPRESS_TYPE_ZSTD,
123,
index_data_size,
);
meta.write_to(mmap);
}
#[test]
fn test_index_meta_rejects_trailing_partial_tlv() {
for extra in [1u16, 2] {
let meta_length = META_LENGTH_INDEX + extra;
let state_length_offset = META_START + meta_length as usize;
let (mut mmap, _path) = create_test_mmap(INDEX_HEADER_SIZE);
write_shared_prefix(
&mut mmap,
MAGIC,
INDEX_VERSION,
FILE_TYPE_INDEX,
meta_length,
);
write_index_meta_tlv(
&mut mmap,
1_000,
1700000000,
4096,
6,
crate::compress::COMPRESS_TYPE_ZSTD,
0,
0,
);
write_u16_to_mmap(&mut mmap, state_length_offset, INDEX_STATE_LENGTH_V1);
let err = IndexFileMetadata::read_from(&mmap).unwrap_err();
assert!(format!("{err}").contains("trailing partial"), "{err}");
}
}
#[test]
fn test_index_read_from_rejects_wrong_file_type() {
let (mut mmap, _path) = create_test_mmap(INDEX_HEADER_SIZE);
write_valid_index_header(&mut mmap, 0);
mmap[OFF_FILE_TYPE] = FILE_TYPE_DATA;
let err = IndexFileMetadata::read_from(&mmap).unwrap_err();
assert!(format!("{err}").contains("file_type"), "{err}");
}
#[test]
fn test_index_meta_requires_base_block_offset_tlv() {
let (mut mmap, _path) = create_test_mmap(INDEX_HEADER_SIZE);
write_valid_index_header(&mut mmap, 0);
mmap[INDEX_BASE_TLV] = 0x7F;
let err = IndexFileMetadata::read_from(&mmap).unwrap_err();
assert!(
format!("{err}").contains("missing base_block_offset"),
"{err}"
);
}
#[test]
fn test_index_meta_requires_index_data_size_tlv() {
let (mut mmap, _path) = create_test_mmap(INDEX_HEADER_SIZE);
write_valid_index_header(&mut mmap, 0);
mmap[INDEX_SIZE_TLV] = 0x7F;
let err = IndexFileMetadata::read_from(&mmap).unwrap_err();
assert!(
format!("{err}").contains("missing index_data_size"),
"{err}"
);
}
#[test]
fn test_index_meta_rejects_duplicate_base_block_offset_tlv() {
let extra = [META_TYPE_BASE_BLOCK_OFFSET, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0];
let meta_length = META_LENGTH_INDEX + extra.len() as u16;
let state_length_offset = META_START + meta_length as usize;
let (mut mmap, _path) = create_test_mmap(INDEX_HEADER_SIZE);
write_shared_prefix(
&mut mmap,
MAGIC,
INDEX_VERSION,
FILE_TYPE_INDEX,
meta_length,
);
write_index_meta_tlv(
&mut mmap,
1_000,
1700000000,
4096,
6,
crate::compress::COMPRESS_TYPE_ZSTD,
0,
0,
);
let extra_start = META_START + META_LENGTH_INDEX as usize;
mmap[extra_start..extra_start + extra.len()].copy_from_slice(&extra);
write_u16_to_mmap(&mut mmap, state_length_offset, INDEX_STATE_LENGTH_V1);
let err = IndexFileMetadata::read_from(&mmap).unwrap_err();
assert!(
format!("{err}").contains("duplicate base_block_offset"),
"{err}"
);
}
#[test]
fn test_index_meta_rejects_duplicate_index_data_size_tlv() {
let extra = [META_TYPE_INDEX_DATA_SIZE, 4, 0, 0, 0, 0, 0];
let meta_length = META_LENGTH_INDEX + extra.len() as u16;
let state_length_offset = META_START + meta_length as usize;
let (mut mmap, _path) = create_test_mmap(INDEX_HEADER_SIZE);
write_shared_prefix(
&mut mmap,
MAGIC,
INDEX_VERSION,
FILE_TYPE_INDEX,
meta_length,
);
write_index_meta_tlv(
&mut mmap,
1_000,
1700000000,
4096,
6,
crate::compress::COMPRESS_TYPE_ZSTD,
0,
0,
);
let extra_start = META_START + META_LENGTH_INDEX as usize;
mmap[extra_start..extra_start + extra.len()].copy_from_slice(&extra);
write_u16_to_mmap(&mut mmap, state_length_offset, INDEX_STATE_LENGTH_V1);
let err = IndexFileMetadata::read_from(&mmap).unwrap_err();
assert!(
format!("{err}").contains("duplicate index_data_size"),
"{err}"
);
}
#[test]
fn test_index_meta_rejects_wrong_size_sizing_tlvs() {
let (mut mmap, _path) = create_test_mmap(INDEX_HEADER_SIZE);
write_valid_index_header(&mut mmap, 16);
write_u16_to_mmap(&mut mmap, INDEX_BASE_TLV + 1, 4);
let err = IndexFileMetadata::read_from(&mmap).unwrap_err();
assert!(format!("{err}").contains("base_block_offset TLV"), "{err}");
let (mut mmap, _path) = create_test_mmap(INDEX_HEADER_SIZE);
write_valid_index_header(&mut mmap, 16);
write_u16_to_mmap(&mut mmap, INDEX_SIZE_TLV + 1, 2);
let err = IndexFileMetadata::read_from(&mmap).unwrap_err();
assert!(format!("{err}").contains("index_data_size TLV"), "{err}");
}
#[test]
fn test_index_header_rejects_content_beyond_reserved_bytes() {
let meta_length = META_LENGTH_INDEX + 60;
let state_length_offset = META_START + meta_length as usize;
let (mut mmap, _path) = create_test_mmap(160);
write_shared_prefix(
&mut mmap,
MAGIC,
INDEX_VERSION,
FILE_TYPE_INDEX,
meta_length,
);
write_index_meta_tlv(
&mut mmap,
1_000,
1700000000,
4096,
6,
crate::compress::COMPRESS_TYPE_ZSTD,
0,
0,
);
write_u16_to_mmap(&mut mmap, state_length_offset, INDEX_STATE_LENGTH_V1);
let err = IndexFileMetadata::read_from(&mmap).unwrap_err();
assert!(format!("{err}").contains("exceeds reserved"), "{err}");
}
#[test]
fn test_index_capacity_must_fit_header_plus_one_entry() {
for file_size in [143u64, 144] {
let meta = IndexFileMetadata::create_default(
0,
file_size,
6,
crate::compress::COMPRESS_TYPE_ZSTD,
0,
0,
);
let (mut mmap, _path) = create_test_mmap(INDEX_HEADER_SIZE);
meta.write_to(&mut mmap);
let result = IndexFileMetadata::read_from(&mmap);
assert_eq!(result.is_ok(), file_size == 144, "N=0 size {file_size}");
}
for file_size in [151u64, 152] {
let meta = IndexFileMetadata::create_default(
0,
file_size,
6,
crate::compress::COMPRESS_TYPE_ZSTD,
0,
3,
);
let (mut mmap, _path) = create_test_mmap(INDEX_HEADER_SIZE);
meta.write_to(&mut mmap);
let result = IndexFileMetadata::read_from(&mmap);
assert_eq!(result.is_ok(), file_size == 152, "N=3 size {file_size}");
}
for file_size in [159u64, 160] {
let meta = IndexFileMetadata::create_default(
0,
file_size,
6,
crate::compress::COMPRESS_TYPE_ZSTD,
0,
16,
);
let (mut mmap, _path) = create_test_mmap(INDEX_HEADER_SIZE);
meta.write_to(&mut mmap);
let result = IndexFileMetadata::read_from(&mmap);
assert_eq!(result.is_ok(), file_size == 160, "N=16 size {file_size}");
}
}
}