zexc 1.2.1

zex native lossless compressor — CLI zexc, library zexc, files *.zex
Documentation
//! Dictionary training & attach (Zstd-inspired).
//!
//! Phase-1: raw dictionary blob stored/referenced by `dictionary_id`.
//! Full COVER/fastcover training can be layered on top of sample harvesting
//! (see zstd `lib/dictBuilder/cover.c`). For now we support:
//! - attaching a raw dictionary buffer to the encoder/decoder
//! - simple sampling: take the first N bytes of samples as a seed dict

use alloc::vec::Vec;

use crate::utils::crc::crc32;
use crate::{Error, Result};

/// In-memory dictionary.
#[derive(Debug, Clone)]
pub struct Dictionary {
    /// Stable id (CRC of content by default).
    pub id: u32,
    /// Dictionary bytes (prepended as virtual history for match finding).
    pub data: Vec<u8>,
}

impl Dictionary {
    /// Create from raw bytes; id = CRC32 of content (non-zero).
    pub fn from_bytes(data: Vec<u8>) -> Result<Self> {
        if data.is_empty() {
            return Err(Error::Dictionary("empty dictionary".into()));
        }
        if data.len() > 32 * 1024 * 1024 {
            return Err(Error::Dictionary("dictionary too large (>32MiB)".into()));
        }
        let mut id = crc32(&data);
        if id == 0 {
            id = 1;
        }
        Ok(Self { id, data })
    }

    /// Create with explicit id.
    pub fn with_id(id: u32, data: Vec<u8>) -> Result<Self> {
        if data.is_empty() {
            return Err(Error::Dictionary("empty dictionary".into()));
        }
        Ok(Self { id, data })
    }

    /// COVER-style training (recommended). See [`crate::core::cover::train_cover`].
    #[cfg(feature = "std")]
    pub fn train_cover(samples: &[&[u8]], max_size: usize) -> Result<Self> {
        crate::core::cover::train_cover(
            samples,
            crate::core::cover::CoverParams {
                dict_size: max_size,
                ..Default::default()
            },
        )
    }

    /// Naive train: concatenate sample tails into a dict of `max_size`.
    ///
    /// Prefer [`train_cover`] for better dictionaries on structured protocols.
    pub fn train_simple(samples: &[&[u8]], max_size: usize) -> Result<Self> {
        if samples.is_empty() {
            return Err(Error::Dictionary("no samples".into()));
        }
        let max_size = max_size.clamp(256, 8 * 1024 * 1024);
        let mut data = Vec::with_capacity(max_size);
        // Prefer tails of samples (Zstd dict convention: recent content at end)
        for s in samples.iter().rev() {
            if data.len() >= max_size {
                break;
            }
            let take = (max_size - data.len()).min(s.len());
            let start = s.len() - take;
            data.extend_from_slice(&s[start..]);
        }
        if data.is_empty() {
            return Err(Error::Dictionary("samples empty".into()));
        }
        Self::from_bytes(data)
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn simple_train() {
        let samples: Vec<&[u8]> = vec![
            b"HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n",
            b"HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n",
            b"HTTP/1.1 500 Error\r\nContent-Type: text/plain\r\n",
        ];
        let d = Dictionary::train_simple(&samples, 1024).unwrap();
        assert!(!d.data.is_empty());
        assert_ne!(d.id, 0);
    }
}