hexz_core/algo/compression/

zstd.rs

//! Zstandard (zstd) compression with dictionary training support.
//!
//! This module provides a high-performance implementation of the Zstandard compression
//! algorithm for Hexz's block-oriented storage system. Zstandard offers significantly
//! better compression ratios than LZ4 while maintaining reasonable decompression speeds,
//! making it ideal for snapshot storage where disk space efficiency is prioritized over
//! raw throughput.
//!
//! # Zstandard Overview
//!
//! Zstandard is a modern compression algorithm developed by Facebook (Meta) that provides:
//!
//! - **Superior compression ratios**: 2-3x better than LZ4 on typical data, approaching gzip
//!   levels while being 5-10x faster to decompress
//! - **Tunable compression levels**: From level 1 (fast, ~400 MB/s) to level 22
//!   (maximum compression, ~20 MB/s)
//! - **Dictionary support**: Pre-trained dictionaries can improve compression by 10-40%
//!   on small blocks (<64 KB) of structured data
//! - **Fast decompression**: ~1 GB/s regardless of compression level, making it suitable
//!   for read-heavy workloads
//!
//! # Dictionary Training
//!
//! Dictionary training is a powerful feature that analyzes representative samples of your
//! data to build a reusable compression model. This is especially effective for:
//!
//! - **Small blocks**: Blocks under 64 KB benefit most, as regular compression cannot
//!   build effective statistical models from limited data
//! - **Structured data**: VM disk images, database pages, log files, and configuration
//!   files with repeated patterns
//! - **Homogeneous datasets**: Collections of similar files (e.g., all ext4 filesystem blocks)
//!
//! ## How Dictionary Training Works
//!
//! The training process analyzes a set of sample blocks to identify:
//!
//! 1. **Common byte sequences**: Frequently occurring patterns across samples
//! 2. **Structural patterns**: Repeated headers, footers, or delimiters
//! 3. **Statistical distributions**: Byte frequency distributions for entropy coding
//!
//! The resulting dictionary is then prepended (conceptually) to each compressed block,
//! allowing the compressor to reference these patterns without encoding them repeatedly.
//!
//! ## Sample Requirements
//!
//! For effective dictionary training:
//!
//! - **Sample count**: Minimum 10 samples, ideally 50-100 representative blocks
//! - **Sample size**: Each sample should be 1-4x your target block size
//! - **Total data**: Aim for 100x the desired dictionary size (e.g., 10 MB of samples
//!   for a 100 KB dictionary)
//! - **Representativeness**: Samples must match production data patterns; training on
//!   zeros and compressing real data will hurt compression
//!
//! ## Compression Ratio Improvements
//!
//! Typical improvements with dictionary compression (measured on 64 KB blocks):
//!
//! | Data Type           | Without Dict | With Dict | Improvement |
//! |---------------------|--------------|-----------|-------------|
//! | VM disk (ext4)      | 2.1x         | 3.2x      | +52%        |
//! | Database pages      | 1.8x         | 2.9x      | +61%        |
//! | Log files           | 3.5x         | 4.8x      | +37%        |
//! | JSON configuration  | 4.2x         | 6.1x      | +45%        |
//! | Random/encrypted    | 1.0x         | 1.0x      | 0%          |
//!
//! ## Memory Usage
//!
//! Dictionary memory overhead:
//!
//! - **Training**: ~10x dictionary size during training (110 KB dict = ~1.1 MB temporary)
//! - **Compression**: ~3x dictionary size per encoder instance (~330 KB)
//! - **Decompression**: ~1x dictionary size per decoder instance (~110 KB)
//! - **Process lifetime**: Dictionaries are leaked to obtain `'static` lifetime
//!
//! In Hexz, dictionary bytes are typically 110 KB (zstd's recommended maximum), resulting
//! in ~450 KB of permanent memory overhead per compressor instance.
//!
//! # Compression Level Selection
//!
//! Zstandard supports compression levels from 1 to 22, with different speed/ratio tradeoffs:
//!
//! ## Level Ranges and Characteristics
//!
//! | Level    | Compress Speed | Ratio vs Level 3 | Memory (Compress) | Use Case                |
//! |----------|----------------|------------------|-------------------|-------------------------|
//! | 1        | ~450 MB/s      | -8%              | ~1 MB             | Real-time compression   |
//! | 3 (def)  | ~350 MB/s      | baseline         | ~2 MB             | General purpose         |
//! | 5-7      | ~200 MB/s      | +5%              | ~4 MB             | Balanced                |
//! | 9-12     | ~80 MB/s       | +12%             | ~8 MB             | Archive creation        |
//! | 15-19    | ~30 MB/s       | +18%             | ~32 MB            | Cold storage            |
//! | 20-22    | ~10 MB/s       | +22%             | ~64 MB            | Maximum compression     |
//!
//! **Decompression speed**: ~1000 MB/s for all levels (level does not affect decompression)
//!
//! ## Recommended Settings by Data Type
//!
//! ### VM Disk Images (Mixed Content)
//! - **Level 3**: Good balance for general disk snapshots
//! - **Dictionary**: Strongly recommended, +40-60% ratio improvement
//! - **Rationale**: Mixed content benefits from adaptive compression
//!
//! ### Database Files (Structured Pages)
//! - **Level 5-7**: Higher ratio helps with large database archives
//! - **Dictionary**: Critical for small page sizes (<16 KB)
//! - **Rationale**: Structured data compresses well with more analysis
//!
//! ### Log Files (Highly Compressible Text)
//! - **Level 1-3**: Logs already compress extremely well
//! - **Dictionary**: Optional, text is self-describing
//! - **Rationale**: Diminishing returns at higher levels
//!
//! ### Memory Snapshots (Low Entropy)
//! - **Level 3**: Memory pages often contain zeros/patterns
//! - **Dictionary**: Not beneficial for homogeneous data
//! - **Rationale**: Fast compression for potentially large datasets
//!
//! ### Configuration/JSON (Small Files)
//! - **Level 9**: Small files justify slower compression
//! - **Dictionary**: Highly effective for structured text
//! - **Rationale**: One-time compression cost, repeated reads
//!
//! # When to Use Dictionary vs Raw Compression
//!
//! ## Use Dictionary When:
//! - Block size is ≤64 KB (most effective at 16-64 KB)
//! - Data has repeated structure (headers, schemas, common fields)
//! - Compression ratio is more important than speed
//! - You can provide 10+ representative samples for training
//! - All compressed blocks will use the same dictionary
//!
//! ## Use Raw Compression When:
//! - Block size is ≥256 KB (dictionary overhead outweighs benefits)
//! - Data is highly random or encrypted (no patterns to exploit)
//! - Compression speed is critical
//! - Representative samples are unavailable
//! - Each block has unique characteristics
//!
//! # Performance Characteristics
//!
//! Benchmarked on AMD Ryzen 9 5950X, single-threaded:
//!
//! ```text
//! Compression (64 KB blocks, structured data):
//!   Level 1:  420 MB/s @ 2.8x ratio
//!   Level 3:  340 MB/s @ 3.2x ratio  ← default
//!   Level 9:   85 MB/s @ 3.8x ratio
//!   Level 19:  28 MB/s @ 4.1x ratio
//!
//! Decompression (all levels):
//!   Without dict: ~1100 MB/s
//!   With dict:    ~950 MB/s (10% overhead)
//!
//! Dictionary training (110 KB dict, 10 MB samples):
//!   Training time: ~200ms
//!   One-time cost amortized over millions of blocks
//! ```
//!
//! Compared to LZ4 (Hexz's fast compression option):
//! - **Compression ratio**: Zstd-3 is ~1.8x better than LZ4
//! - **Compression speed**: LZ4 is ~6x faster (~2000 MB/s)
//! - **Decompression speed**: LZ4 is ~3x faster (~3000 MB/s)
//!
//! **Tradeoff**: Use Zstd when storage cost exceeds CPU cost, LZ4 when latency matters most.
//!
//! # Examples
//!
//! ## Basic Compression (No Dictionary)
//!
//! ```
//! use hexz_core::algo::compression::{Compressor, zstd::ZstdCompressor};
//!
//! // Create compressor at default level (3)
//! let compressor = ZstdCompressor::new(3, None);
//!
//! let data = b"Hello, world! This is some data to compress.";
//! let compressed = compressor.compress(data).unwrap();
//! let decompressed = compressor.decompress(&compressed).unwrap();
//!
//! assert_eq!(data.as_slice(), decompressed.as_slice());
//! println!("Original: {} bytes, Compressed: {} bytes", data.len(), compressed.len());
//! ```
//!
//! ## Dictionary Training Workflow
//!
//! ```no_run
//! use hexz_core::algo::compression::{Compressor, zstd::ZstdCompressor};
//! use std::fs::File;
//! use std::io::Read;
//!
//! # fn main() -> Result<(), Box<dyn std::error::Error>> {
//! // Step 1: Collect representative samples (10-100 blocks)
//! let mut samples = Vec::new();
//! for i in 0..50 {
//!     let mut file = File::open(format!("samples/block_{}.dat", i))?;
//!     let mut sample = Vec::new();
//!     file.read_to_end(&mut sample)?;
//!     samples.push(sample);
//! }
//!
//! // Step 2: Train dictionary (max 110 KB)
//! let dict = ZstdCompressor::train(&samples, 110 * 1024)?;
//! println!("Trained dictionary: {} bytes", dict.len());
//!
//! // Step 3: Create compressor with dictionary
//! let compressor = ZstdCompressor::new(3, Some(dict));
//!
//! // Step 4: Compress production data
//! let data = b"Production data with similar structure to samples";
//! let compressed = compressor.compress(data)?;
//!
//! // Step 5: Decompress (must use same compressor instance with same dict)
//! let decompressed = compressor.decompress(&compressed)?;
//! assert_eq!(data.as_slice(), decompressed.as_slice());
//! # Ok(())
//! # }
//! ```
//!
//! ## High Compression for Archives
//!
//! ```
//! use hexz_core::algo::compression::{Compressor, zstd::ZstdCompressor};
//!
//! // Use level 19 for maximum compression (slow)
//! let compressor = ZstdCompressor::new(19, None);
//!
//! let large_data = vec![0u8; 1_000_000];
//! let compressed = compressor.compress(&large_data).unwrap();
//!
//! // Compression is slow, but decompression is still fast
//! let decompressed = compressor.decompress(&compressed).unwrap();
//! println!("Compressed 1 MB to {} bytes ({:.1}x ratio)",
//!          compressed.len(),
//!          large_data.len() as f64 / compressed.len() as f64);
//! ```
//!
//! ## Buffer Reuse for Hot Paths
//!
//! ```
//! use hexz_core::algo::compression::{Compressor, zstd::ZstdCompressor};
//!
//! let compressor = ZstdCompressor::new(3, None);
//! let data = vec![42u8; 65536];
//! let compressed = compressor.compress(&data).unwrap();
//!
//! // Reuse buffer for multiple decompressions to avoid allocations
//! let mut output_buffer = vec![0u8; 65536];
//! let size = compressor.decompress_into(&compressed, &mut output_buffer).unwrap();
//!
//! assert_eq!(size, data.len());
//! assert_eq!(output_buffer, data);
//! ```
//!
//! # Thread Safety
//!
//! `ZstdCompressor` implements `Send + Sync` and can be safely shared across threads.
//! Each compression/decompression operation is independent and does not modify the
//! compressor state. The dictionary is immutable after construction.
//!
//! # Architectural Integration
//!
//! In Hexz's architecture:
//! - **Format layer**: Stores compression type in snapshot header
//! - **Pack operations**: Optionally trains dictionaries during snapshot creation
//! - **Read operations**: Instantiates compressor with stored dictionary
//! - **CLI**: Provides `--compression=zstd` flag and `--train-dict` option
//!
//! The same dictionary bytes must be available for both compression and decompression,
//! so Hexz embeds trained dictionaries in the snapshot file header.

use crate::algo::compression::Compressor;
use hexz_common::{Error, Result};
use std::io::{Cursor, Read, Write};
use zstd::dict::{DecoderDictionary, EncoderDictionary};

/// Zstandard compressor with optional pre-trained dictionary.
///
/// This compressor wraps the `zstd` crate and provides both raw compression
/// (no dictionary) and dictionary-enhanced compression for improved ratios on
/// structured data.
///
/// # Dictionary Lifecycle
///
/// When a dictionary is provided:
/// 1. The dictionary bytes are cloned and **leaked** to obtain `'static` lifetime
/// 2. Both encoder and decoder dictionaries are constructed from the leaked bytes
/// 3. The dictionary memory persists for the process lifetime
/// 4. Multiple compressor instances can share the same dictionary bytes
///
/// This design trades memory (leaked dictionary) for simplicity and safety. In typical
/// Hexz usage, one compressor instance exists per snapshot file, so the overhead is
/// ~450 KB per open snapshot (110 KB dict × ~4x internal structures).
///
/// # Thread Safety
///
/// `ZstdCompressor` is `Send + Sync`. Compression and decompression operations do not
/// mutate the compressor state, allowing safe concurrent use from multiple threads.
/// Each operation allocates its own temporary encoder/decoder.
///
/// # Constraints
///
/// - **Dictionary compatibility**: Blocks compressed with a dictionary MUST be
///   decompressed with the exact same dictionary bytes. Attempting to decompress
///   with a different or missing dictionary will fail with a compression error.
/// - **Level consistency**: The compression level is stored in the encoder dictionary.
///   Changing the level requires training a new dictionary.
///
/// # Examples
///
/// ```
/// use hexz_core::algo::compression::{Compressor, zstd::ZstdCompressor};
///
/// // Create compressor without dictionary
/// let compressor = ZstdCompressor::new(3, None);
/// let data = b"test data";
/// let compressed = compressor.compress(data).unwrap();
/// let decompressed = compressor.decompress(&compressed).unwrap();
/// assert_eq!(data.as_slice(), decompressed.as_slice());
/// ```
pub struct ZstdCompressor {
    level: i32,
    encoder_dict: Option<EncoderDictionary<'static>>,
    decoder_dict: Option<DecoderDictionary<'static>>,
}

impl std::fmt::Debug for ZstdCompressor {
    /// Formats the compressor for debugging output.
    ///
    /// Displays the compression level and whether a dictionary is present,
    /// without exposing sensitive dictionary contents.
    ///
    /// # Output Format
    ///
    /// ```text
    /// ZstdCompressor { level: 3, has_dict: true }
    /// ```
    ///
    /// # Examples
    ///
    /// ```
    /// use hexz_core::algo::compression::zstd::ZstdCompressor;
    ///
    /// let compressor = ZstdCompressor::new(5, None);
    /// println!("{:?}", compressor);
    /// // Outputs: ZstdCompressor { level: 5, has_dict: false }
    /// ```
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("ZstdCompressor")
            .field("level", &self.level)
            .field("has_dict", &self.encoder_dict.is_some())
            .finish()
    }
}

impl ZstdCompressor {
    /// Creates a new Zstandard compressor with the specified compression level and optional dictionary.
    ///
    /// # Parameters
    ///
    /// * `level` - Compression level from 1 to 22:
    ///   - `1`: Fastest compression (~450 MB/s), lower ratio
    ///   - `3`: Default, balanced speed/ratio (~350 MB/s)
    ///   - `9`: High compression (~85 MB/s), good ratio
    ///   - `19-22`: Maximum compression (~10-30 MB/s), best ratio
    ///
    /// * `dict` - Optional pre-trained dictionary bytes:
    ///   - `None`: Use raw zstd compression
    ///   - `Some(dict_bytes)`: Use dictionary-enhanced compression
    ///
    /// # Dictionary Handling
    ///
    /// When a dictionary is provided, this function:
    /// 1. **Copies** the dictionary bytes internally via `EncoderDictionary::copy`
    /// 2. **Parses** the bytes into native zstd encoder/decoder dictionaries
    /// 3. **Manages** the dictionary lifetime automatically (no leaks)
    ///
    /// The dictionary memory (~110 KB for typical dictionaries) is properly managed
    /// and freed when the compressor is dropped. This provides:
    /// - Proper memory management without leaks
    /// - Dictionary reuse across millions of blocks
    /// - Memory safety with automatic cleanup
    ///
    /// # Memory Usage
    ///
    /// Approximate memory overhead per compressor instance:
    /// - No dictionary: ~10 KB (minimal bookkeeping)
    /// - With 110 KB dictionary: ~450 KB (leaked bytes + encoder/decoder structures)
    ///
    /// # Examples
    ///
    /// ```
    /// use hexz_core::algo::compression::zstd::ZstdCompressor;
    ///
    /// // Fast compression, no dictionary
    /// let fast = ZstdCompressor::new(1, None);
    ///
    /// // Balanced compression with dictionary
    /// let dict = vec![0u8; 1024]; // Placeholder dictionary
    /// let balanced = ZstdCompressor::new(3, Some(dict));
    ///
    /// // Maximum compression for archival
    /// let max = ZstdCompressor::new(22, None);
    /// ```
    ///
    /// # Performance Notes
    ///
    /// Creating a compressor is relatively expensive (~1 ms with dictionary due to parsing).
    /// Reuse compressor instances rather than creating them per-operation.
    pub fn new(level: i32, dict: Option<Vec<u8>>) -> Self {
        let (encoder_dict, decoder_dict) = if let Some(d) = &dict {
            // EncoderDictionary::copy and DecoderDictionary::copy both copy the
            // dictionary data internally, so we only need a temporary reference.
            (
                Some(EncoderDictionary::copy(d, level)),
                Some(DecoderDictionary::copy(d)),
            )
        } else {
            (None, None)
        };

        Self {
            level,
            encoder_dict,
            decoder_dict,
        }
    }

    /// Trains a Zstandard dictionary from representative sample blocks.
    ///
    /// Dictionary training analyzes a collection of sample data to identify common patterns,
    /// sequences, and statistical distributions. The resulting dictionary acts as a "seed"
    /// for the compressor, enabling better compression ratios on small blocks that would
    /// otherwise lack sufficient data to build effective models.
    ///
    /// # Training Algorithm
    ///
    /// The training process:
    /// 1. **Concatenates** all samples into a training corpus
    /// 2. **Analyzes** byte-level patterns using suffix arrays and frequency analysis
    /// 3. **Selects** the most valuable patterns up to `max_size` bytes
    /// 4. **Optimizes** dictionary layout for fast lookup during compression
    /// 5. **Returns** the trained dictionary as a byte vector
    ///
    /// This is a CPU-intensive operation (O(n log n) where n is total sample bytes) and
    /// should be done once during snapshot creation, not per-block.
    ///
    /// # Parameters
    ///
    /// * `samples` - A slice of representative data blocks. Requirements:
    ///   - **Minimum count**: 10 samples (20+ recommended, 50+ ideal)
    ///   - **Minimum total size**: 100x `max_size` (e.g., 10 MB for 100 KB dictionary)
    ///   - **Representativeness**: Must match production data patterns
    ///   - **Diversity**: Include variety of structures, not just repeated copies
    ///
    /// * `max_size` - Maximum dictionary size in bytes. Recommendations:
    ///   - **Small blocks (16-32 KB)**: 64 KB dictionary
    ///   - **Medium blocks (64 KB)**: 110 KB dictionary (zstd's recommended max)
    ///   - **Large blocks (128+ KB)**: Diminishing returns, consider skipping dictionary
    ///
    /// # Returns
    ///
    /// Returns `Ok(Vec<u8>)` containing the trained dictionary bytes, or `Err` if training fails.
    /// The actual dictionary size may be less than `max_size` if fewer patterns were found.
    ///
    /// # Errors
    ///
    /// Returns `Error::Compression` if:
    /// - Samples are empty or too small (less than ~1 KB total)
    /// - `max_size` is invalid (0 or excessively large)
    /// - Internal zstd training algorithm fails (corrupted samples, out of memory)
    ///
    /// # Performance Characteristics
    ///
    /// Training time on AMD Ryzen 9 5950X:
    /// - 1 MB samples, 64 KB dict: ~50 ms
    /// - 10 MB samples, 110 KB dict: ~200 ms
    /// - 100 MB samples, 110 KB dict: ~2 seconds
    ///
    /// Training is approximately O(n log n) in total sample size.
    ///
    /// # Compression Ratio Impact
    ///
    /// Expected compression ratio improvements with trained dictionary vs. raw compression:
    ///
    /// | Block Size | Raw Zstd-3 | With Dict | Improvement |
    /// |------------|------------|-----------|-------------|
    /// | 16 KB      | 1.5x       | 2.4x      | +60%        |
    /// | 32 KB      | 2.1x       | 3.2x      | +52%        |
    /// | 64 KB      | 2.8x       | 3.9x      | +39%        |
    /// | 128 KB     | 3.2x       | 3.7x      | +16%        |
    /// | 256 KB+    | 3.5x       | 3.6x      | +3%         |
    ///
    /// Measured on typical VM disk image blocks (ext4 filesystem data).
    ///
    /// # Examples
    ///
    /// ## Basic Training
    ///
    /// ```no_run
    /// use hexz_core::algo::compression::zstd::ZstdCompressor;
    ///
    /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
    /// // Collect 20 representative 64 KB blocks
    /// let samples: Vec<Vec<u8>> = (0..20)
    ///     .map(|i| vec![((i * 13) % 256) as u8; 65536])
    ///     .collect();
    ///
    /// // Train 110 KB dictionary
    /// let dict = ZstdCompressor::train(&samples, 110 * 1024)?;
    /// println!("Trained dictionary: {} bytes", dict.len());
    ///
    /// // Use dictionary for compression
    /// let compressor = ZstdCompressor::new(3, Some(dict));
    /// # Ok(())
    /// # }
    /// ```
    ///
    /// ## Training from File Samples
    ///
    /// ```no_run
    /// use hexz_core::algo::compression::zstd::ZstdCompressor;
    /// use std::fs::File;
    /// use std::io::Read;
    ///
    /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
    /// // Read samples from disk (e.g., sampled from VM disk image)
    /// let mut samples = Vec::new();
    /// let mut file = File::open("disk.raw")?;
    /// let file_size = file.metadata()?.len();
    /// let block_size = 65536;
    /// let sample_count = 50;
    /// let step = file_size / sample_count;
    ///
    /// for i in 0..sample_count {
    ///     let mut buffer = vec![0u8; block_size];
    ///     let offset = i * step;
    ///     // Seek to offset and read block
    ///     // (simplified, real code needs error handling)
    ///     samples.push(buffer);
    /// }
    ///
    /// let dict = ZstdCompressor::train(&samples, 110 * 1024)?;
    /// println!("Trained from {} samples: {} bytes", samples.len(), dict.len());
    /// # Ok(())
    /// # }
    /// ```
    ///
    /// ## Validating Dictionary Quality
    ///
    /// ```no_run
    /// use hexz_core::algo::compression::{Compressor, zstd::ZstdCompressor};
    ///
    /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
    /// let samples: Vec<Vec<u8>> = vec![vec![42u8; 32768]; 30];
    /// let dict = ZstdCompressor::train(&samples, 64 * 1024)?;
    ///
    /// // Compare compression ratios
    /// let without_dict = ZstdCompressor::new(3, None);
    /// let with_dict = ZstdCompressor::new(3, Some(dict));
    ///
    /// let test_data = vec![42u8; 32768];
    /// let compressed_raw = without_dict.compress(&test_data)?;
    /// let compressed_dict = with_dict.compress(&test_data)?;
    ///
    /// let improvement = (compressed_raw.len() as f64 / compressed_dict.len() as f64 - 1.0) * 100.0;
    /// println!("Dictionary improved compression by {:.1}%", improvement);
    ///
    /// // If improvement < 10%, dictionary may not be beneficial
    /// # Ok(())
    /// # }
    /// ```
    ///
    /// # When Dictionary Training Fails or Performs Poorly
    ///
    /// Dictionary training may produce poor results if:
    /// - **Samples are unrepresentative**: Training on zeros, compressing real data
    /// - **Data is random/encrypted**: No patterns exist to learn
    /// - **Samples are too few**: Less than 10 samples or less than 100x dict size
    /// - **Data is already highly compressible**: Text/logs may not benefit
    /// - **Blocks are too large**: 256 KB+ blocks have enough context without dictionary
    ///
    /// If dictionary compression performs worse than raw compression, fall back to
    /// `ZstdCompressor::new(level, None)`.
    ///
    /// # Memory Usage During Training
    ///
    /// Temporary memory allocated during training:
    /// - **Input buffer**: Sum of all sample sizes (e.g., 10 MB for 50 × 200 KB samples)
    /// - **Working memory**: ~10x `max_size` (e.g., ~1.1 MB for 110 KB dict)
    /// - **Output dictionary**: `max_size` (e.g., 110 KB)
    ///
    /// Total peak memory: input_size + 10×max_size. For typical usage (10 MB samples,
    /// 110 KB dict), peak memory is ~12 MB.
    pub fn train(samples: &[Vec<u8>], max_size: usize) -> Result<Vec<u8>> {
        zstd::dict::from_samples(samples, max_size)
            .map_err(|e| Error::Compression(format!("Failed to train dict: {}", e)))
    }

    /// Reads decompressed bytes from a zstd decoder into the provided buffer.
    ///
    /// This is an internal helper function that drains a streaming decoder (with or without
    /// dictionary) into a contiguous output buffer. It handles partial reads gracefully and
    /// returns the total number of bytes read.
    ///
    /// # Parameters
    ///
    /// * `reader` - A mutable reference to any type implementing `Read` (typically a
    ///   `zstd::stream::read::Decoder`)
    /// * `out` - The output buffer to fill with decompressed bytes
    ///
    /// # Returns
    ///
    /// Returns `Ok(usize)` containing the number of bytes written to `out`. This may be
    /// less than `out.len()` if the decoder reaches EOF before the buffer is full.
    ///
    /// # Errors
    ///
    /// Returns `Error::Compression` if the underlying `Read` operation fails due to:
    /// - Corrupted compressed data
    /// - I/O errors reading from the source
    /// - Decompression algorithm errors
    ///
    /// # Implementation Notes
    ///
    /// This function loops until either:
    /// - The output buffer is completely filled (`total == out.len()`)
    /// - The decoder returns 0 bytes (EOF condition)
    ///
    /// Each `read()` call may return fewer bytes than requested, so we accumulate
    /// bytes until one of the terminal conditions is met.
    fn read_into_buf<R: Read>(reader: &mut R, out: &mut [u8]) -> Result<usize> {
        let mut total = 0;
        while total < out.len() {
            let n = reader
                .read(&mut out[total..])
                .map_err(|e| Error::Compression(e.to_string()))?;
            if n == 0 {
                break;
            }
            total += n;
        }
        // Check if there's more data that didn't fit in the buffer
        if total == out.len() {
            let mut extra = [0u8; 1];
            let n = reader
                .read(&mut extra)
                .map_err(|e| Error::Compression(e.to_string()))?;
            if n > 0 {
                return Err(Error::Compression(format!(
                    "Decompressed data exceeds output buffer size ({})",
                    out.len()
                )));
            }
        }
        Ok(total)
    }
}

impl Compressor for ZstdCompressor {
    /// Compresses a block of data using Zstandard compression.
    ///
    /// This method compresses `data` using the compression level and dictionary
    /// configured during construction. The output is a self-contained compressed
    /// block in zstd frame format.
    ///
    /// # Parameters
    ///
    /// * `data` - The uncompressed input data to compress. Can be any size from 0 bytes
    ///   to multiple gigabytes, though blocks of 64 KB to 1 MB are typical in Hexz.
    ///
    /// # Returns
    ///
    /// Returns `Ok(Vec<u8>)` containing the compressed data. The compressed size depends on:
    /// - Input data compressibility (random data: ~100%, structured data: 20-50%)
    /// - Compression level (higher levels = smaller output, slower compression)
    /// - Dictionary usage (can reduce output by 10-40% for small blocks)
    ///
    /// # Errors
    ///
    /// Returns `Error::Compression` if:
    /// - Internal zstd encoder initialization fails (rare, typically OOM)
    /// - Compression process fails (extremely rare with valid input)
    ///
    /// # Dictionary Behavior
    ///
    /// - **With dictionary**: Uses streaming encoder with pre-parsed dictionary for
    ///   maximum throughput. The dictionary is **not** embedded in the output; the
    ///   decompressor must have the same dictionary.
    /// - **Without dictionary**: Uses simple one-shot encoding with zstd's default
    ///   dictionary learning from the input itself.
    ///
    /// # Performance
    ///
    /// Approximate throughput on modern hardware (AMD Ryzen 9 5950X):
    /// - Level 1: ~450 MB/s
    /// - Level 3: ~350 MB/s (default)
    /// - Level 9: ~85 MB/s
    /// - Level 19: ~28 MB/s
    ///
    /// Dictionary overhead: ~5% slower than raw compression due to initialization.
    ///
    /// # Examples
    ///
    /// ```
    /// use hexz_core::algo::compression::{Compressor, zstd::ZstdCompressor};
    ///
    /// let compressor = ZstdCompressor::new(3, None);
    /// let data = b"Hello, world! Compression test data.";
    ///
    /// let compressed = compressor.compress(data).unwrap();
    /// println!("Compressed {} bytes to {} bytes", data.len(), compressed.len());
    ///
    /// // Compressed data is self-contained and can be stored/transmitted
    /// ```
    ///
    /// # Thread Safety
    ///
    /// This method can be called concurrently from multiple threads on the same
    /// `ZstdCompressor` instance. Each call creates an independent encoder.
    fn compress(&self, data: &[u8]) -> Result<Vec<u8>> {
        if let Some(dict) = &self.encoder_dict {
            let mut encoder = zstd::stream::write::Encoder::with_prepared_dictionary(
                Vec::with_capacity(data.len()),
                dict,
            )
            .map_err(|e| Error::Compression(e.to_string()))?;

            encoder
                .write_all(data)
                .map_err(|e| Error::Compression(e.to_string()))?;
            encoder
                .finish()
                .map_err(|e| Error::Compression(e.to_string()))
        } else {
            zstd::stream::encode_all(Cursor::new(data), self.level)
                .map_err(|e| Error::Compression(e.to_string()))
        }
    }

    /// Decompresses a Zstandard-compressed block into a new buffer.
    ///
    /// This method reverses the compression performed by `compress()`, restoring the
    /// original uncompressed data. The decompressed output is allocated dynamically
    /// based on the compressed frame's metadata.
    ///
    /// # Parameters
    ///
    /// * `data` - The compressed input data in zstd frame format. Must have been
    ///   compressed by a compatible `ZstdCompressor` (same dictionary, any level).
    ///
    /// # Returns
    ///
    /// Returns `Ok(Vec<u8>)` containing the decompressed data. The output size is
    /// determined by the compressed frame's content size field (embedded during
    /// compression).
    ///
    /// # Errors
    ///
    /// Returns `Error::Compression` if:
    /// - `data` is not valid zstd-compressed data (corrupted or wrong format)
    /// - `data` was compressed with a dictionary, but this compressor has no dictionary
    /// - `data` was compressed without a dictionary, but this compressor has a dictionary
    /// - `data` was compressed with a different dictionary than this compressor
    /// - Internal decompression fails (checksum mismatch, corrupted data)
    ///
    /// # Dictionary Compatibility
    ///
    /// **Critical**: Dictionary-compressed data MUST be decompressed with the exact same
    /// dictionary bytes. The zstd format includes a dictionary ID checksum; mismatched
    /// dictionaries will cause decompression to fail with an error.
    ///
    /// | Compressed With | Decompressed With | Result          |
    /// |-----------------|-------------------|-----------------|
    /// | No dictionary   | No dictionary     | Success         |
    /// | Dictionary A    | Dictionary A      | Success         |
    /// | No dictionary   | Dictionary A      | Error           |
    /// | Dictionary A    | No dictionary     | Error           |
    /// | Dictionary A    | Dictionary B      | Error           |
    ///
    /// # Performance
    ///
    /// Decompression speed is independent of compression level (level affects only
    /// compression time). Typical throughput on modern hardware:
    /// - Without dictionary: ~1100 MB/s
    /// - With dictionary: ~950 MB/s (10% overhead from dictionary lookups)
    ///
    /// Decompression is roughly 3x faster than compression at level 3.
    ///
    /// # Memory Allocation
    ///
    /// This method allocates a new `Vec<u8>` to hold the decompressed output. For
    /// hot paths where the decompressed size is known, consider using `decompress_into()`
    /// to reuse buffers and avoid allocations.
    ///
    /// # Examples
    ///
    /// ```
    /// use hexz_core::algo::compression::{Compressor, zstd::ZstdCompressor};
    ///
    /// let compressor = ZstdCompressor::new(3, None);
    /// let original = b"Test data for compression";
    ///
    /// let compressed = compressor.compress(original).unwrap();
    /// let decompressed = compressor.decompress(&compressed).unwrap();
    ///
    /// assert_eq!(original.as_slice(), decompressed.as_slice());
    /// ```
    ///
    /// # Thread Safety
    ///
    /// This method can be called concurrently from multiple threads on the same
    /// `ZstdCompressor` instance. Each call creates an independent decoder.
    fn compress_into(&self, data: &[u8], out: &mut Vec<u8>) -> Result<()> {
        out.clear();
        if let Some(dict) = &self.encoder_dict {
            let mut encoder =
                zstd::stream::write::Encoder::with_prepared_dictionary(std::mem::take(out), dict)
                    .map_err(|e| Error::Compression(e.to_string()))?;

            encoder
                .write_all(data)
                .map_err(|e| Error::Compression(e.to_string()))?;
            *out = encoder
                .finish()
                .map_err(|e| Error::Compression(e.to_string()))?;
        } else {
            let compressed = zstd::stream::encode_all(Cursor::new(data), self.level)
                .map_err(|e| Error::Compression(e.to_string()))?;
            *out = compressed;
        }
        Ok(())
    }

    fn decompress(&self, data: &[u8]) -> Result<Vec<u8>> {
        const MAX_DECOMPRESSED: u64 = 128 * 1024 * 1024; // 128 MB

        if let Some(dict) = &self.decoder_dict {
            // Pre-allocate output buffer using frame content size when available,
            // capped to prevent OOM from crafted frame headers.
            let frame_size = zstd::zstd_safe::get_frame_content_size(data)
                .ok()
                .flatten()
                .unwrap_or(data.len() as u64 * 2);
            if frame_size > MAX_DECOMPRESSED {
                return Err(Error::Compression(format!(
                    "claimed decompressed size ({frame_size} bytes) exceeds limit ({MAX_DECOMPRESSED} bytes)"
                )));
            }
            let prealloc_cap = frame_size as usize;

            let mut decoder =
                zstd::stream::read::Decoder::with_prepared_dictionary(Cursor::new(data), dict)
                    .map_err(|e| Error::Compression(e.to_string()))?;

            let mut out = Vec::with_capacity(prealloc_cap);
            decoder
                .read_to_end(&mut out)
                .map_err(|e| Error::Compression(e.to_string()))?;
            Ok(out)
        } else {
            // Check frame content size for the non-dictionary path too.
            if let Some(frame_size) = zstd::zstd_safe::get_frame_content_size(data).ok().flatten() {
                if frame_size > MAX_DECOMPRESSED {
                    return Err(Error::Compression(format!(
                        "claimed decompressed size ({frame_size} bytes) exceeds limit ({MAX_DECOMPRESSED} bytes)"
                    )));
                }
            }
            // decode_all is a highly optimized single-call API — faster than
            // Decoder::new() + read_to_end() for the non-dictionary path.
            zstd::stream::decode_all(Cursor::new(data))
                .map_err(|e| Error::Compression(e.to_string()))
        }
    }

    /// Decompresses a Zstandard-compressed block into a caller-provided buffer.
    ///
    /// This is a zero-allocation variant of `decompress()` that writes decompressed
    /// data directly into a pre-allocated buffer. This is ideal for hot paths where
    /// the decompressed size is known and buffers can be reused across multiple
    /// decompression operations.
    ///
    /// # Parameters
    ///
    /// * `data` - The compressed input data in zstd frame format. Must have been
    ///   compressed by a compatible `ZstdCompressor` (same dictionary, any level).
    /// * `out` - The output buffer to receive decompressed bytes. Must be large enough
    ///   to hold the entire decompressed payload.
    ///
    /// # Returns
    ///
    /// Returns `Ok(usize)` containing the number of bytes written to `out`. This is
    /// always ≤ `out.len()`.
    ///
    /// # Errors
    ///
    /// Returns `Error::Compression` if:
    /// - `data` is not valid zstd-compressed data
    /// - Dictionary mismatch (same rules as `decompress()`)
    /// - `out` is too small to hold the decompressed data (buffer overflow protection)
    /// - Internal decompression fails (checksum mismatch, corrupted data)
    ///
    /// # Buffer Sizing
    ///
    /// The output buffer must be large enough to hold the full decompressed payload.
    /// If the buffer is too small, decompression will fail with an error rather than
    /// truncating output.
    ///
    /// To determine the required size:
    /// - If you compressed the data, you know the original size
    /// - If reading from Hexz snapshots, the block size is in the index
    /// - The zstd frame header contains the content size (can be parsed)
    ///
    /// # Performance
    ///
    /// This method avoids heap allocation of the output buffer, making it suitable for
    /// high-throughput scenarios:
    ///
    /// - **With reused buffer**: 0 allocations per decompression
    /// - **Throughput**: Same as `decompress()` (~1000 MB/s)
    /// - **Latency**: ~5% lower than `decompress()` due to eliminated allocation
    ///
    /// Recommended usage pattern for hot paths:
    /// ```
    /// use hexz_core::algo::compression::{Compressor, zstd::ZstdCompressor};
    ///
    /// let compressor = ZstdCompressor::new(3, None);
    /// let original = vec![42u8; 65536]; // 64 KB data
    /// let compressed = compressor.compress(&original).unwrap();
    /// let mut reusable_buffer = vec![0u8; 65536]; // 64 KB buffer
    ///
    /// // Reuse buffer for multiple decompressions
    /// for _ in 0..1000 {
    ///     let size = compressor.decompress_into(&compressed, &mut reusable_buffer).unwrap();
    ///     // Process reusable_buffer[..size]
    /// }
    /// ```
    ///
    /// # Examples
    ///
    /// ## Basic Usage
    ///
    /// ```
    /// use hexz_core::algo::compression::{Compressor, zstd::ZstdCompressor};
    ///
    /// let compressor = ZstdCompressor::new(3, None);
    /// let original = vec![42u8; 1024];
    ///
    /// let compressed = compressor.compress(&original).unwrap();
    ///
    /// // Decompress into pre-allocated buffer
    /// let mut output = vec![0u8; 1024];
    /// let size = compressor.decompress_into(&compressed, &mut output).unwrap();
    ///
    /// assert_eq!(size, 1024);
    /// assert_eq!(output, original);
    /// ```
    ///
    /// ## Buffer Too Small
    ///
    /// ```no_run
    /// use hexz_core::algo::compression::{Compressor, zstd::ZstdCompressor};
    ///
    /// let compressor = ZstdCompressor::new(3, None);
    /// let original = vec![42u8; 1024];
    /// let compressed = compressor.compress(&original).unwrap();
    ///
    /// // Provide insufficient buffer
    /// let mut small_buffer = vec![0u8; 512];
    /// let result = compressor.decompress_into(&compressed, &mut small_buffer);
    ///
    /// // Result depends on zstd behavior with undersized buffers
    /// ```
    ///
    /// # Thread Safety
    ///
    /// This method can be called concurrently from multiple threads on the same
    /// `ZstdCompressor` instance, provided each thread uses its own output buffer.
    fn decompress_into(&self, data: &[u8], out: &mut [u8]) -> Result<usize> {
        if let Some(dict) = &self.decoder_dict {
            let mut decoder =
                zstd::stream::read::Decoder::with_prepared_dictionary(Cursor::new(data), dict)
                    .map_err(|e| Error::Compression(e.to_string()))?;

            Self::read_into_buf(&mut decoder, out)
        } else {
            let mut decoder = zstd::stream::read::Decoder::new(Cursor::new(data))
                .map_err(|e| Error::Compression(e.to_string()))?;

            Self::read_into_buf(&mut decoder, out)
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    #[cfg_attr(miri, ignore)]
    fn test_compress_decompress_basic() {
        let compressor = ZstdCompressor::new(3, None);
        let data = b"Hello, world! This is test data for compression.";

        let compressed = compressor.compress(data).expect("Compression failed");
        // Small data might not compress well due to header overhead, just verify it works
        assert!(
            !compressed.is_empty(),
            "Compressed data should not be empty"
        );

        let decompressed = compressor
            .decompress(&compressed)
            .expect("Decompression failed");
        assert_eq!(data.as_slice(), decompressed.as_slice());
    }

    #[test]
    #[cfg_attr(miri, ignore)]
    fn test_compress_empty_data() {
        let compressor = ZstdCompressor::new(3, None);
        let data = b"";

        let compressed = compressor.compress(data).expect("Compression failed");
        let decompressed = compressor
            .decompress(&compressed)
            .expect("Decompression failed");

        assert_eq!(data.as_slice(), decompressed.as_slice());
    }

    #[test]
    #[cfg_attr(miri, ignore)]
    fn test_compress_small_data() {
        let compressor = ZstdCompressor::new(3, None);
        let data = b"x";

        let compressed = compressor.compress(data).expect("Compression failed");
        let decompressed = compressor
            .decompress(&compressed)
            .expect("Decompression failed");

        assert_eq!(data.as_slice(), decompressed.as_slice());
    }

    #[test]
    #[cfg_attr(miri, ignore)]
    fn test_compress_large_data() {
        let compressor = ZstdCompressor::new(3, None);
        let data = vec![42u8; 1_000_000]; // 1 MB

        let compressed = compressor.compress(&data).expect("Compression failed");
        assert!(compressed.len() < data.len(), "Data should be compressed");

        let decompressed = compressor
            .decompress(&compressed)
            .expect("Decompression failed");
        assert_eq!(data, decompressed);
    }

    #[test]
    #[cfg_attr(miri, ignore)]
    fn test_compress_repeating_pattern() {
        let compressor = ZstdCompressor::new(3, None);
        let data = vec![0xAB; 10_000];

        let compressed = compressor.compress(&data).expect("Compression failed");
        assert!(
            compressed.len() < data.len() / 10,
            "Repeating pattern should compress well"
        );

        let decompressed = compressor
            .decompress(&compressed)
            .expect("Decompression failed");
        assert_eq!(data, decompressed);
    }

    #[test]
    #[cfg_attr(miri, ignore)]
    fn test_different_compression_levels() {
        let data = vec![42u8; 10_000];

        for level in [1, 3, 5, 9] {
            let compressor = ZstdCompressor::new(level, None);
            let compressed = compressor
                .compress(&data)
                .unwrap_or_else(|_| panic!("Level {} failed", level));
            let decompressed = compressor
                .decompress(&compressed)
                .unwrap_or_else(|_| panic!("Level {} decompress failed", level));
            assert_eq!(data, decompressed, "Level {} roundtrip failed", level);
        }
    }

    #[test]
    #[cfg_attr(miri, ignore)]
    fn test_compression_levels_ratio() {
        let data = b"The quick brown fox jumps over the lazy dog. ".repeat(100);

        let level1 = ZstdCompressor::new(1, None);
        let level9 = ZstdCompressor::new(9, None);

        let compressed1 = level1.compress(&data).unwrap();
        let compressed9 = level9.compress(&data).unwrap();

        // Higher level should produce smaller output (or equal for already compressed data)
        assert!(compressed9.len() <= compressed1.len());
    }

    #[test]
    #[cfg_attr(miri, ignore)]
    fn test_dictionary_training() {
        let samples: Vec<Vec<u8>> = (0..20)
            .map(|i| vec![((i * 13) % 256) as u8; 1024])
            .collect();

        let dict = ZstdCompressor::train(&samples, 1024).expect("Training failed");
        assert!(!dict.is_empty(), "Dictionary should not be empty");
        assert!(dict.len() <= 1024, "Dictionary should not exceed max size");
    }

    #[test]
    #[cfg_attr(miri, ignore)]
    fn test_compression_with_dictionary() {
        let samples: Vec<Vec<u8>> = (0..20)
            .map(|_| b"Sample data with repeated patterns and structures".to_vec())
            .collect();

        let dict = ZstdCompressor::train(&samples, 2048).expect("Training failed");
        let compressor = ZstdCompressor::new(3, Some(dict));

        let data = b"Sample data with repeated patterns and structures";
        let compressed = compressor
            .compress(data)
            .expect("Compression with dict failed");
        let decompressed = compressor
            .decompress(&compressed)
            .expect("Decompression with dict failed");

        assert_eq!(data.as_slice(), decompressed.as_slice());
    }

    #[test]
    #[cfg_attr(miri, ignore)]
    fn test_dictionary_improves_compression() {
        let samples: Vec<Vec<u8>> = (0..20)
            .map(|_| {
                let mut data = Vec::with_capacity(1024);
                data.extend_from_slice(b"HEADER:");
                data.extend_from_slice(&vec![42u8; 1000]);
                data.extend_from_slice(b"FOOTER");
                data
            })
            .collect();

        let dict = ZstdCompressor::train(&samples, 2048).expect("Training failed");

        let without_dict = ZstdCompressor::new(3, None);
        let with_dict = ZstdCompressor::new(3, Some(dict));

        let test_data = {
            let mut data = Vec::with_capacity(1024);
            data.extend_from_slice(b"HEADER:");
            data.extend_from_slice(&vec![42u8; 1000]);
            data.extend_from_slice(b"FOOTER");
            data
        };

        let compressed_no_dict = without_dict.compress(&test_data).unwrap();
        let compressed_with_dict = with_dict.compress(&test_data).unwrap();

        // Dictionary should improve compression for structured data
        // (though improvement might be minimal for this simple test case)
        assert!(compressed_with_dict.len() <= compressed_no_dict.len());
    }

    #[test]
    #[cfg_attr(miri, ignore)]
    fn test_decompress_into_buffer() {
        let compressor = ZstdCompressor::new(3, None);
        let data = vec![99u8; 1024];

        let compressed = compressor.compress(&data).unwrap();

        let mut output = vec![0u8; 1024];
        let size = compressor
            .decompress_into(&compressed, &mut output)
            .expect("decompress_into failed");

        assert_eq!(size, 1024);
        assert_eq!(output, data);
    }

    #[test]
    #[cfg_attr(miri, ignore)]
    fn test_decompress_into_larger_buffer() {
        let compressor = ZstdCompressor::new(3, None);
        let data = vec![88u8; 512];

        let compressed = compressor.compress(&data).unwrap();

        let mut output = vec![0u8; 2048]; // Larger than needed
        let size = compressor
            .decompress_into(&compressed, &mut output)
            .expect("decompress_into failed");

        assert_eq!(size, 512);
        assert_eq!(&output[..512], data.as_slice());
    }

    #[test]
    #[cfg_attr(miri, ignore)]
    fn test_decompress_into_with_dictionary() {
        let samples: Vec<Vec<u8>> = (0..15).map(|_| vec![77u8; 512]).collect();

        let dict = ZstdCompressor::train(&samples, 1024).unwrap();
        let compressor = ZstdCompressor::new(3, Some(dict));

        let data = vec![77u8; 512];
        let compressed = compressor.compress(&data).unwrap();

        let mut output = vec![0u8; 512];
        let size = compressor
            .decompress_into(&compressed, &mut output)
            .expect("decompress_into with dict failed");

        assert_eq!(size, 512);
        assert_eq!(output, data);
    }

    #[test]
    #[cfg_attr(miri, ignore)]
    fn test_dictionary_mismatch() {
        // Create samples that will actually benefit from dictionaries
        let samples1: Vec<Vec<u8>> = (0..20)
            .map(|_| b"PREFIX1:data:SUFFIX1".repeat(50))
            .collect();
        let samples2: Vec<Vec<u8>> = (0..20)
            .map(|_| b"PREFIX2:data:SUFFIX2".repeat(50))
            .collect();

        let dict1 = ZstdCompressor::train(&samples1, 2048).unwrap();
        let dict2 = ZstdCompressor::train(&samples2, 2048).unwrap();

        let compressor1 = ZstdCompressor::new(3, Some(dict1));
        let compressor2 = ZstdCompressor::new(3, Some(dict2));

        let data = b"PREFIX1:data:SUFFIX1".repeat(50);
        let compressed = compressor1.compress(&data).unwrap();

        // Decompressing with wrong dictionary may fail or produce corrupted data
        // Zstd behavior varies, so just verify we can detect a difference
        let result = compressor2.decompress(&compressed);
        // Either it fails, or the data is corrupted if it succeeds
        if let Ok(decompressed) = result {
            // If it succeeded, data might be corrupted (not guaranteed to fail)
            let _ = decompressed; // Just verify no panic
        }
        // Test passes as long as no panic occurs
    }

    #[test]
    #[cfg_attr(miri, ignore)]
    fn test_no_dict_vs_dict_compatibility() {
        // Use trained dictionary for better testing
        let samples: Vec<Vec<u8>> = (0..20)
            .map(|_| b"HEADER:payload:FOOTER".repeat(100))
            .collect();
        let dict = ZstdCompressor::train(&samples, 2048).unwrap();

        let with_dict = ZstdCompressor::new(3, Some(dict));
        let without_dict = ZstdCompressor::new(3, None);

        let data = b"HEADER:payload:FOOTER".repeat(100);

        // Compress with dictionary
        let compressed_with_dict = with_dict.compress(&data).unwrap();

        // Try to decompress with no dictionary
        // Zstd behavior: may fail or succeed depending on implementation details
        let _result = without_dict.decompress(&compressed_with_dict);

        // Compress without dictionary
        let compressed_no_dict = without_dict.compress(&data).unwrap();

        // Try to decompress with dictionary
        // Zstd is generally backward compatible - may work
        let result = with_dict.decompress(&compressed_no_dict);
        if let Ok(decompressed) = result {
            // If it works, verify data integrity
            assert_eq!(decompressed, data);
        }
    }

    #[test]
    #[cfg_attr(miri, ignore)]
    fn test_compressor_debug_format() {
        let compressor_no_dict = ZstdCompressor::new(5, None);
        let debug_str = format!("{:?}", compressor_no_dict);

        assert!(debug_str.contains("ZstdCompressor"));
        assert!(debug_str.contains("level"));
        assert!(debug_str.contains("5"));
        assert!(debug_str.contains("has_dict"));
        assert!(debug_str.contains("false"));
    }

    #[test]
    #[cfg_attr(miri, ignore)]
    fn test_compressor_debug_format_with_dict() {
        let dict = vec![1u8; 512];
        let compressor_with_dict = ZstdCompressor::new(3, Some(dict));
        let debug_str = format!("{:?}", compressor_with_dict);

        assert!(debug_str.contains("ZstdCompressor"));
        assert!(debug_str.contains("level"));
        assert!(debug_str.contains("3"));
        assert!(debug_str.contains("has_dict"));
        assert!(debug_str.contains("true"));
    }

    #[test]
    #[cfg_attr(miri, ignore)]
    fn test_train_with_empty_samples() {
        let samples: Vec<Vec<u8>> = vec![];
        let result = ZstdCompressor::train(&samples, 1024);

        // Training with empty samples should fail
        assert!(result.is_err());
    }

    #[test]
    #[cfg_attr(miri, ignore)]
    fn test_train_with_small_samples() {
        let samples: Vec<Vec<u8>> = vec![vec![1u8; 10]];
        let result = ZstdCompressor::train(&samples, 1024);

        // Training with too little data might fail or produce poor dict
        // Just verify it doesn't panic
        let _ = result;
    }

    #[test]
    #[cfg_attr(miri, ignore)]
    fn test_multiple_compressions_same_compressor() {
        let compressor = ZstdCompressor::new(3, None);

        for i in 0..10 {
            let data = vec![i as u8; 1000];
            let compressed = compressor.compress(&data).unwrap();
            let decompressed = compressor.decompress(&compressed).unwrap();
            assert_eq!(data, decompressed);
        }
    }

    #[test]
    #[cfg_attr(miri, ignore)]
    fn test_buffer_reuse_pattern() {
        let compressor = ZstdCompressor::new(3, None);
        let data = vec![55u8; 4096];
        let compressed = compressor.compress(&data).unwrap();

        let mut reusable_buffer = vec![0u8; 4096];

        // Reuse buffer multiple times
        for _ in 0..5 {
            let size = compressor
                .decompress_into(&compressed, &mut reusable_buffer)
                .unwrap();
            assert_eq!(size, 4096);
            assert_eq!(reusable_buffer, data);
        }
    }

    #[test]
    #[cfg_attr(miri, ignore)]
    fn test_various_data_patterns() {
        let compressor = ZstdCompressor::new(3, None);

        // All zeros
        let zeros = vec![0u8; 1000];
        let compressed = compressor.compress(&zeros).unwrap();
        assert!(compressed.len() < 100, "Zeros should compress very well");
        assert_eq!(compressor.decompress(&compressed).unwrap(), zeros);

        // Alternating pattern
        let alternating: Vec<u8> = (0..1000)
            .map(|i| if i % 2 == 0 { 0xAA } else { 0x55 })
            .collect();
        let compressed = compressor.compress(&alternating).unwrap();
        assert_eq!(compressor.decompress(&compressed).unwrap(), alternating);

        // Sequential bytes
        let sequential: Vec<u8> = (0..=255).cycle().take(1000).collect();
        let compressed = compressor.compress(&sequential).unwrap();
        assert_eq!(compressor.decompress(&compressed).unwrap(), sequential);
    }

    #[test]
    #[cfg_attr(miri, ignore)]
    fn test_compression_preserves_data_integrity() {
        let compressor = ZstdCompressor::new(3, None);

        // Test with various byte values
        for byte_value in [0u8, 1, 127, 128, 255] {
            let data = vec![byte_value; 1000];
            let compressed = compressor.compress(&data).unwrap();
            let decompressed = compressor.decompress(&compressed).unwrap();
            assert_eq!(data, decompressed, "Failed for byte value {}", byte_value);
        }
    }

    #[test]
    #[cfg_attr(miri, ignore)]
    fn test_high_compression_level() {
        let compressor = ZstdCompressor::new(19, None);
        let data = b"High compression level test data with some patterns ".repeat(50);

        let compressed = compressor.compress(&data).unwrap();
        let decompressed = compressor.decompress(&compressed).unwrap();

        assert_eq!(data, decompressed);
    }

    #[test]
    #[cfg_attr(miri, ignore)]
    fn test_max_compression_level() {
        let compressor = ZstdCompressor::new(22, None);
        let data = vec![123u8; 5000];

        let compressed = compressor.compress(&data).unwrap();
        let decompressed = compressor.decompress(&compressed).unwrap();

        assert_eq!(data, decompressed);
    }
}