zexc 1.0.0

zex native lossless compressor — CLI zexc, library zexc, files *.zex
Documentation
//! ZEX encoder — native algorithm + optional AEAD encryption.
//!
//! Pipeline:
//! 1. Build [`ZexHeader`]
//! 2. Compress blocks with [`crate::algorithms::zex_native`]
//! 3. End + optional CRC over **plaintext**
//! 4. If encryption key set → ChaCha20-Poly1305 seal of body

use crate::algorithms::{turbo, zex_native};
use crate::core::dictionary::Dictionary;
use crate::format::block::{BlockHeader, BlockType};
use crate::format::header::{
    CompressionMode, HeaderFlags, ZexHeader, DEFAULT_BLOCK_SIZE, HEADER_SIZE,
};
use crate::utils::crc::crc32;
use crate::utils::security::SecurityPolicy;
use crate::Result;
use rayon::prelude::*;
use std::io::Write;

#[cfg(feature = "encryption")]
use crate::crypto::{
    derive_key_from_password, encrypt_body, fill_random, EncryptionKey, NONCE_LEN, SALT_LEN,
};

/// Compression options (library API).
#[derive(Debug, Clone)]
pub struct CompressOptions {
    /// Profile: turbo / balanced / max.
    pub mode: CompressionMode,
    /// Level 1–22.
    pub level: u8,
    /// Max uncompressed block size.
    pub block_size: u32,
    /// CRC32 over plaintext (default true).
    pub checksum: bool,
    /// Compress blocks in parallel.
    pub threads: bool,
    /// Independent blocks.
    pub independent: bool,
    /// Optional dictionary.
    pub dictionary: Option<Dictionary>,
    /// Security policy.
    pub security: SecurityPolicy,
    /// Optional AEAD key (feature `encryption`).
    #[cfg(feature = "encryption")]
    pub encryption: Option<EncryptionKey>,
}

impl Default for CompressOptions {
    fn default() -> Self {
        Self {
            mode: CompressionMode::Balanced,
            level: 3,
            block_size: DEFAULT_BLOCK_SIZE,
            checksum: true,
            threads: true,
            independent: true,
            dictionary: None,
            security: SecurityPolicy::default(),
            #[cfg(feature = "encryption")]
            encryption: None,
        }
    }
}

impl CompressOptions {
    /// Builder: profile.
    pub fn mode(mut self, mode: CompressionMode) -> Self {
        self.mode = mode;
        self
    }
    /// Builder: level 1–22.
    pub fn level(mut self, level: u8) -> Self {
        self.level = level.clamp(1, 22);
        self
    }
    /// Builder: block size.
    pub fn block_size(mut self, size: u32) -> Self {
        self.block_size = size.clamp(1024, 16 * 1024 * 1024);
        self
    }
    /// Builder: checksum.
    pub fn checksum(mut self, on: bool) -> Self {
        self.checksum = on;
        self
    }
    /// Builder: threads.
    pub fn threads(mut self, on: bool) -> Self {
        self.threads = on;
        self
    }
    /// Builder: dictionary.
    pub fn dictionary(mut self, dict: Dictionary) -> Self {
        self.dictionary = Some(dict);
        self
    }
    /// Builder: strict security.
    pub fn secure(mut self) -> Self {
        self.security = SecurityPolicy::strict();
        self.checksum = true;
        self.independent = true;
        self
    }

    /// Builder: encrypt with key/password (`encryption` feature).
    #[cfg(feature = "encryption")]
    pub fn encrypt(mut self, key: EncryptionKey) -> Self {
        self.encryption = Some(key);
        self
    }
}

/// Streaming / reusable encoder.
pub struct Encoder {
    opts: CompressOptions,
}

impl Encoder {
    /// Create with options.
    pub fn new(opts: CompressOptions) -> Self {
        Self { opts }
    }

    /// Compress full buffer → `.zex` bytes.
    pub fn compress(&self, src: &[u8]) -> Result<Vec<u8>> {
        let mut out = Vec::with_capacity(src.len() / 2 + HEADER_SIZE + 64);
        self.compress_to_writer(src, &mut out)?;
        Ok(out)
    }

    /// Compress and write.
    pub fn compress_to_writer<W: Write>(&self, src: &[u8], out: &mut W) -> Result<()> {
        let opts = &self.opts;
        let level = opts.mode.effective_level(opts.level);
        let block_size = (opts.block_size as usize).min(opts.security.max_block_uncompressed);

        let mut header = ZexHeader::new(src.len() as u64, opts.mode, level);
        header.block_size = block_size as u32;
        header.flags.set(HeaderFlags::CHECKSUM, opts.checksum);
        header.flags.set(HeaderFlags::INDEPENDENT, opts.independent);
        header
            .flags
            .set(HeaderFlags::MULTITHREAD, opts.threads && src.len() > block_size);
        header.flags.set(
            HeaderFlags::SECURE,
            opts.security.max_output <= 512 * 1024 * 1024,
        );
        if let Some(ref d) = opts.dictionary {
            header.dictionary_id = d.id;
            header.flags.insert(HeaderFlags::DICTIONARY);
        }

        #[cfg(feature = "encryption")]
        let want_enc = opts.encryption.is_some();
        #[cfg(not(feature = "encryption"))]
        let want_enc = false;

        if want_enc {
            header.flags.insert(HeaderFlags::ENCRYPTED);
        }

        // Build plaintext body (blocks + end + crc)
        let mut body = Vec::with_capacity(src.len() / 2 + 128);
        let chunks: Vec<&[u8]> = if src.is_empty() {
            vec![src]
        } else {
            src.chunks(block_size).collect()
        };

        let dict_bytes: Option<&[u8]> = opts.dictionary.as_ref().map(|d| d.data.as_slice());

        let compressed: Vec<(BlockType, Vec<u8>, u32)> = if opts.threads && chunks.len() > 1 {
            chunks
                .par_iter()
                .map(|chunk| compress_one_block(chunk, level, dict_bytes))
                .collect::<Result<Vec<_>>>()?
        } else {
            chunks
                .iter()
                .map(|chunk| compress_one_block(chunk, level, dict_bytes))
                .collect::<Result<Vec<_>>>()?
        };

        for (ty, payload, unc) in compressed {
            opts.security
                .check_block(payload.len(), unc as usize)?;
            let bh = BlockHeader {
                block_type: ty,
                uncompressed_size: unc,
                compressed_size: payload.len() as u32,
            };
            bh.write_to(&mut body)?;
            body.write_all(&payload)?;
        }
        BlockHeader::end().write_to(&mut body)?;
        if opts.checksum {
            body.write_all(&crc32(src).to_le_bytes())?;
        }

        header.write_to(out)?;

        #[cfg(feature = "encryption")]
        if let Some(ref ek) = opts.encryption {
            let mut salt = [0u8; SALT_LEN];
            let mut nonce = [0u8; NONCE_LEN];
            fill_random(&mut salt)?;
            fill_random(&mut nonce)?;

            let key = match ek {
                EncryptionKey::Raw(k) => {
                    salt = [0u8; SALT_LEN]; // mark raw-key mode
                    *k
                }
                EncryptionKey::Password(pw) => derive_key_from_password(pw, &salt)?,
            };

            let ct = encrypt_body(&key, &nonce, &body)?;
            out.write_all(&salt)?;
            out.write_all(&nonce)?;
            out.write_all(&ct)?;
            return Ok(());
        }

        let _ = want_enc;
        out.write_all(&body)?;
        Ok(())
    }
}

fn compress_one_block(
    chunk: &[u8],
    level: u8,
    dict: Option<&[u8]>,
) -> Result<(BlockType, Vec<u8>, u32)> {
    let unc = chunk.len() as u32;
    if chunk.is_empty() {
        return Ok((BlockType::Raw, Vec::new(), 0));
    }
    // With a dictionary even tiny blocks can shrink via history matches.
    if chunk.len() < 16 && dict.is_none() {
        return Ok((BlockType::Raw, chunk.to_vec(), unc));
    }

    // Turbo path: skip analysis on high-entropy data; use specialized encoder
    if turbo::should_use_turbo(level) && dict.is_none() && turbo::looks_incompressible(chunk) {
        return Ok((BlockType::Raw, chunk.to_vec(), unc));
    }

    let payload = if turbo::should_use_turbo(level) {
        let mut buf = Vec::with_capacity(chunk.len() / 2 + 32);
        turbo::compress_into(chunk, &mut buf, dict)?;
        buf
    } else {
        zex_native::compress_with_history(chunk, level, dict)?
    };

    if payload.len() >= chunk.len() {
        Ok((BlockType::Raw, chunk.to_vec(), unc))
    } else {
        Ok((BlockType::Zex, payload, unc))
    }
}

/// One-shot compress.
pub fn compress(src: &[u8], opts: &CompressOptions) -> Result<Vec<u8>> {
    Encoder::new(opts.clone()).compress(src)
}

/// Compress with dictionary.
pub fn compress_with_dict(src: &[u8], opts: &CompressOptions, dict: Dictionary) -> Result<Vec<u8>> {
    let mut o = opts.clone();
    o.dictionary = Some(dict);
    Encoder::new(o).compress(src)
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::core::decoder::{decompress, decompress_with_options, DecompressOptions};
    use crate::core::dictionary::Dictionary;

    #[test]
    fn encode_decode_native() {
        let src = b"ZEX native encoder roundtrip. ".repeat(500);
        let opts = CompressOptions::default()
            .mode(CompressionMode::Balanced)
            .level(3);
        let c = compress(&src, &opts).unwrap();
        assert_eq!(&c[..4], b"ZEX1");
        let d = decompress(&c).unwrap();
        assert_eq!(d, src);
    }

    #[test]
    fn profiles_roundtrip() {
        let src = b"ABCDEFGHIJ".repeat(1000);
        for mode in [
            CompressionMode::Turbo,
            CompressionMode::Balanced,
            CompressionMode::Max,
        ] {
            let opts = CompressOptions::default().mode(mode).level(5).threads(false);
            let c = compress(&src, &opts).unwrap();
            let d = decompress(&c).unwrap();
            assert_eq!(d, src, "mode {mode}");
        }
    }

    #[test]
    fn empty_input() {
        let c = compress(b"", &CompressOptions::default()).unwrap();
        let d = decompress(&c).unwrap();
        assert!(d.is_empty());
    }

    #[test]
    fn secure_preset() {
        let src = b"secure mode data ".repeat(100);
        let c = compress(&src, &CompressOptions::default().secure()).unwrap();
        assert_eq!(decompress(&c).unwrap(), src);
    }

    #[test]
    fn dictionary_history_frame_roundtrip() {
        let dict = Dictionary::from_bytes(
            b"SHARED-PREFIX-HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n\r\n".to_vec(),
        )
        .unwrap();
        let msg =
            b"SHARED-PREFIX-HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n\r\nhello dict history";

        let without = compress(msg, &CompressOptions::default().level(6).threads(false)).unwrap();
        let with = compress(
            msg,
            &CompressOptions::default()
                .level(6)
                .threads(false)
                .dictionary(dict.clone()),
        )
        .unwrap();
        assert!(
            with.len() <= without.len(),
            "dict frame {} vs plain {}",
            with.len(),
            without.len()
        );

        let h = ZexHeader::from_bytes(&with).unwrap();
        assert_eq!(h.dictionary_id, dict.id);
        assert!(h.flags.contains(HeaderFlags::DICTIONARY));

        let d = decompress_with_options(
            &with,
            &DecompressOptions {
                dictionary: Some(dict),
                ..Default::default()
            },
        )
        .unwrap();
        assert_eq!(d, msg);
    }

    #[cfg(feature = "encryption")]
    #[test]
    fn encrypt_password_roundtrip() {
        let src = b"top secret payload for ZEX AEAD test!!!!".repeat(20);
        let opts = CompressOptions::default()
            .level(4)
            .encrypt(EncryptionKey::Password("hunter2".into()));
        let c = compress(&src, &opts).unwrap();
        let h = ZexHeader::from_bytes(&c).unwrap();
        assert!(h.is_encrypted());

        let dopts = DecompressOptions {
            encryption: Some(EncryptionKey::Password("hunter2".into())),
            ..Default::default()
        };
        let d = decompress_with_options(&c, &dopts).unwrap();
        assert_eq!(d, src);

        let bad = DecompressOptions {
            encryption: Some(EncryptionKey::Password("wrong".into())),
            ..Default::default()
        };
        assert!(decompress_with_options(&c, &bad).is_err());
    }
}