zexc 1.2.0

zex native lossless compressor — CLI zexc, library zexc, files *.zex
Documentation
//! Seekable multi-frame ZEX files with a trailing index.
//!
//! ## Layout
//! ```text
//! [ ZEX1 frame for chunk 0 ]
//! [ ZEX1 frame for chunk 1 ]
//! …
//! [ Seek index trailer ]
//! ```
//!
//! ### Trailer (`ZXSI` = ZEX Seek Index)
//! ```text
//! magic:        "ZXSI" (4)
//! version:      u16 LE = 1
//! num_entries:  u32 LE
//! entries[n]:
//!   comp_offset: u64 LE   // offset of frame from file start
//!   comp_size:   u32 LE
//!   decomp_size: u32 LE
//! trailer_size: u32 LE    // entire trailer length (for locating from EOF)
//! ```
//!
//! Decompressed offset of entry `i` is the sum of `decomp_size` of entries `0..i`.

use crate::core::decoder::{DecompressOptions, decompress_with_options};
use crate::core::encoder::{CompressOptions, compress};
use crate::{Error, Result};
use std::io::{Read, Seek, SeekFrom, Write};

/// Seek index magic.
pub const SEEK_MAGIC: [u8; 4] = *b"ZXSI";
/// Seek index version.
pub const SEEK_VERSION: u16 = 1;

/// One seek table entry.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct SeekEntry {
    /// Byte offset of the compressed frame in the file.
    pub comp_offset: u64,
    /// Compressed frame size.
    pub comp_size: u32,
    /// Decompressed size of this frame.
    pub decomp_size: u32,
}

/// Full seek table.
#[derive(Debug, Clone)]
pub struct SeekTable {
    /// Per-frame entries in order.
    pub entries: Vec<SeekEntry>,
}

impl SeekTable {
    /// Total decompressed size.
    pub fn total_decomp(&self) -> u64 {
        self.entries.iter().map(|e| e.decomp_size as u64).sum()
    }

    /// Find frame index covering decompressed offset `pos`.
    pub fn frame_for_offset(&self, pos: u64) -> Option<(usize, u64)> {
        let mut acc = 0u64;
        for (i, e) in self.entries.iter().enumerate() {
            let next = acc + e.decomp_size as u64;
            if pos < next {
                return Some((i, acc));
            }
            acc = next;
        }
        None
    }

    /// Serialize trailer bytes.
    pub fn to_bytes(&self) -> Vec<u8> {
        let n = self.entries.len() as u32;
        let mut v = Vec::with_capacity(4 + 2 + 4 + self.entries.len() * 16 + 4);
        v.extend_from_slice(&SEEK_MAGIC);
        v.extend_from_slice(&SEEK_VERSION.to_le_bytes());
        v.extend_from_slice(&n.to_le_bytes());
        for e in &self.entries {
            v.extend_from_slice(&e.comp_offset.to_le_bytes());
            v.extend_from_slice(&e.comp_size.to_le_bytes());
            v.extend_from_slice(&e.decomp_size.to_le_bytes());
        }
        let trailer_size = (v.len() + 4) as u32;
        v.extend_from_slice(&trailer_size.to_le_bytes());
        v
    }

    /// Parse trailer from the last `trailer_size` bytes of a file (or full buffer).
    pub fn from_trailer(data: &[u8]) -> Result<Self> {
        if data.len() < 4 + 2 + 4 + 4 {
            return Err(Error::InvalidHeader("seek trailer too short".into()));
        }
        // trailer_size is last 4 bytes
        let ts = u32::from_le_bytes(data[data.len() - 4..].try_into().unwrap()) as usize;
        if ts > data.len() || ts < 14 {
            return Err(Error::InvalidHeader("bad seek trailer_size".into()));
        }
        let t = &data[data.len() - ts..];
        if t[..4] != SEEK_MAGIC {
            return Err(Error::InvalidHeader("not a ZXSI trailer".into()));
        }
        let ver = u16::from_le_bytes([t[4], t[5]]);
        if ver != SEEK_VERSION {
            return Err(Error::UnsupportedVersion {
                found: ver,
                supported: SEEK_VERSION,
            });
        }
        let n = u32::from_le_bytes(t[6..10].try_into().unwrap()) as usize;
        let need = 10 + n * 16 + 4;
        if t.len() < need {
            return Err(Error::InvalidHeader("seek entries truncated".into()));
        }
        let mut entries = Vec::with_capacity(n);
        let mut ip = 10usize;
        for _ in 0..n {
            let comp_offset = u64::from_le_bytes(t[ip..ip + 8].try_into().unwrap());
            ip += 8;
            let comp_size = u32::from_le_bytes(t[ip..ip + 4].try_into().unwrap());
            ip += 4;
            let decomp_size = u32::from_le_bytes(t[ip..ip + 4].try_into().unwrap());
            ip += 4;
            entries.push(SeekEntry {
                comp_offset,
                comp_size,
                decomp_size,
            });
        }
        Ok(Self { entries })
    }
}

/// Compress `src` into independent frames of `frame_size` + seek trailer.
pub fn compress_seekable(src: &[u8], frame_size: usize, opts: &CompressOptions) -> Result<Vec<u8>> {
    let frame_size = frame_size.clamp(4 * 1024, 16 * 1024 * 1024);
    let mut out = Vec::new();
    let mut entries = Vec::new();
    let mut off = 0usize;
    while off < src.len() || (src.is_empty() && entries.is_empty()) {
        let end = (off + frame_size).min(src.len());
        let chunk = &src[off..end];
        let comp_offset = out.len() as u64;
        let frame = compress(chunk, opts)?;
        let comp_size = frame.len() as u32;
        out.extend_from_slice(&frame);
        entries.push(SeekEntry {
            comp_offset,
            comp_size,
            decomp_size: chunk.len() as u32,
        });
        if src.is_empty() {
            break;
        }
        off = end;
    }
    let table = SeekTable { entries };
    out.extend_from_slice(&table.to_bytes());
    Ok(out)
}

/// Decompress entire seekable blob.
pub fn decompress_seekable(data: &[u8], dopts: &DecompressOptions) -> Result<Vec<u8>> {
    let table = SeekTable::from_trailer(data)?;
    let mut out = Vec::with_capacity(table.total_decomp() as usize);
    for e in &table.entries {
        let start = e.comp_offset as usize;
        let end = start + e.comp_size as usize;
        if end > data.len() {
            return Err(Error::CorruptBlock("seek frame OOB".into()));
        }
        let plain = decompress_with_options(&data[start..end], dopts)?;
        if plain.len() as u32 != e.decomp_size {
            return Err(Error::SizeMismatch {
                expected: e.decomp_size as u64,
                actual: plain.len() as u64,
            });
        }
        out.extend_from_slice(&plain);
    }
    Ok(out)
}

/// Read a window of decompressed data without decoding all frames.
pub fn decompress_range(
    data: &[u8],
    decomp_start: u64,
    decomp_len: usize,
    dopts: &DecompressOptions,
) -> Result<Vec<u8>> {
    let table = SeekTable::from_trailer(data)?;
    let total = table.total_decomp();
    if decomp_start >= total {
        return Ok(Vec::new());
    }
    let decomp_end = (decomp_start + decomp_len as u64).min(total);
    let (first_i, first_base) = table
        .frame_for_offset(decomp_start)
        .ok_or_else(|| Error::InvalidParameter("seek offset past end".into()))?;

    let mut out = Vec::new();
    let mut acc = first_base;
    for e in table.entries.iter().skip(first_i) {
        if acc >= decomp_end {
            break;
        }
        let start = e.comp_offset as usize;
        let end = start + e.comp_size as usize;
        let plain = decompress_with_options(&data[start..end], dopts)?;
        let frame_start = acc;
        let frame_end = acc + plain.len() as u64;
        let slice_lo = decomp_start.saturating_sub(frame_start) as usize;
        let slice_hi = (decomp_end.min(frame_end) - frame_start) as usize;
        if slice_lo < plain.len() && slice_hi > slice_lo {
            out.extend_from_slice(&plain[slice_lo.min(plain.len())..slice_hi.min(plain.len())]);
        }
        acc = frame_end;
    }
    Ok(out)
}

/// Read seek table from a seekable `Read + Seek` (reads last 4 bytes then trailer).
pub fn read_seek_table<R: Read + Seek>(r: &mut R) -> Result<SeekTable> {
    r.seek(SeekFrom::End(-4)).map_err(Error::Io)?;
    let mut ts_buf = [0u8; 4];
    r.read_exact(&mut ts_buf).map_err(Error::Io)?;
    let ts = u32::from_le_bytes(ts_buf) as i64;
    if ts < 14 {
        return Err(Error::InvalidHeader("seek trailer_size too small".into()));
    }
    r.seek(SeekFrom::End(-ts)).map_err(Error::Io)?;
    let mut trailer = vec![0u8; ts as usize];
    r.read_exact(&mut trailer).map_err(Error::Io)?;
    SeekTable::from_trailer(&trailer)
}

/// Write seekable file to a writer (streaming chunks from a reader).
pub fn compress_seekable_writer<R: Read, W: Write>(
    mut input: R,
    output: &mut W,
    frame_size: usize,
    opts: &CompressOptions,
) -> Result<SeekTable> {
    let frame_size = frame_size.clamp(4 * 1024, 16 * 1024 * 1024);
    let mut entries = Vec::new();
    let mut buf = vec![0u8; frame_size];
    let mut file_off = 0u64;
    loop {
        let mut filled = 0usize;
        while filled < frame_size {
            let n = input.read(&mut buf[filled..]).map_err(Error::Io)?;
            if n == 0 {
                break;
            }
            filled += n;
        }
        if filled == 0 && !entries.is_empty() {
            break;
        }
        let chunk = &buf[..filled];
        let frame = compress(chunk, opts)?;
        let comp_size = frame.len() as u32;
        output.write_all(&frame).map_err(Error::Io)?;
        entries.push(SeekEntry {
            comp_offset: file_off,
            comp_size,
            decomp_size: filled as u32,
        });
        file_off += comp_size as u64;
        if filled < frame_size {
            break;
        }
    }
    if entries.is_empty() {
        // empty input: one empty frame
        let frame = compress(&[], opts)?;
        let comp_size = frame.len() as u32;
        output.write_all(&frame).map_err(Error::Io)?;
        entries.push(SeekEntry {
            comp_offset: 0,
            comp_size,
            decomp_size: 0,
        });
    }
    let table = SeekTable { entries };
    output.write_all(&table.to_bytes()).map_err(Error::Io)?;
    Ok(table)
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::core::encoder::CompressOptions;

    #[test]
    fn seekable_full_and_range() {
        let mut src = Vec::new();
        for i in 0..50_000u32 {
            src.extend_from_slice(&i.to_le_bytes());
            src.extend_from_slice(b"SEEK");
        }
        let opts = CompressOptions::default().level(3).threads(false);
        let blob = compress_seekable(&src, 16 * 1024, &opts).unwrap();
        assert!(blob.len() > 32);
        let table = SeekTable::from_trailer(&blob).unwrap();
        assert!(table.entries.len() > 1);
        assert_eq!(table.total_decomp() as usize, src.len());

        let full = decompress_seekable(&blob, &DecompressOptions::default()).unwrap();
        assert_eq!(full, src);

        let mid = decompress_range(&blob, 10_000, 500, &DecompressOptions::default()).unwrap();
        assert_eq!(mid, src[10_000..10_500]);
    }
}