limnifs-core 0.3.52

//! Shuffle+LZ4 composite codec (id 0x0A): for scientific float data.
//!
//! Applies byte-shuffle (transpose N items × `item_size` bytes) before
//! LZ4 compression. The shuffle exposes mantissa/exponent correlation
//! in float arrays that LZ4's match finder can then exploit.
//!
//! Typical ratio improvement: float32 arrays go from ~80% (raw LZ4)
//! to ~40% (shuffled LZ4) on smooth scientific data.
//!
//! ## Wire format
//!
//! The shuffle filter is self-describing: its output starts with
//! `[tag: u8][item_size: u8]` so the decoder recovers the `item_size`
//! without external config. The codec wrapper LZ4-compresses the
//! shuffled bytes:
//!
//! ```text
//! [LZ4 compressed block of: [shuffle_tag][item_size][shuffled_data]]
//! ```
//!
//! ## Decode
//!
//! 1. LZ4 decompress → shuffled bytes (with self-describing prefix).
//! 2. `ByteShuffle::decode` reads the prefix and unshuffles → original data.

use crate::codec::composite;
use crate::codec::CODEC_BLOSC2_SHUFFLE_LZ4;

/// Shuffle+LZ4 composite (id 0x0A) as an instance of
/// [`FilterCodecComposite`](crate::codec::composite::FilterCodecComposite).
pub type ShuffleLz4Codec = composite::FilterCodecComposite<omnizip_filters::shuffle::ByteShuffle>;

/// Minimum input worth shuffling: below this the filter pass costs
/// more than LZ4 saves.
const MIN_SHUFFLE_SIZE: usize = 512;

/// `item_size`-byte shuffle + LZ4 (1/2/4/8; anything else → 4).
#[must_use]
pub fn shuffle_lz4(item_size: usize) -> ShuffleLz4Codec {
    let item_size = if [1, 2, 4, 8].contains(&item_size) {
        item_size
    } else {
        4
    };
    ShuffleLz4Codec::new(
        omnizip_filters::shuffle::ByteShuffle::new(item_size),
        crate::codec::CODEC_LZ4,
        CODEC_BLOSC2_SHUFFLE_LZ4,
        "shuffle+lz4",
        MIN_SHUFFLE_SIZE,
    )
}

/// Default float32 (`item_size` = 4) shuffle + LZ4.
#[must_use]
pub fn float32() -> ShuffleLz4Codec {
    shuffle_lz4(4)
}

/// float64 (`item_size` = 8) shuffle + LZ4.
#[must_use]
pub fn float64() -> ShuffleLz4Codec {
    shuffle_lz4(8)
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::codec::Codec;

    #[test]
    fn round_trips_float32_array() {
        // 1024 float32 values with smooth gradient — the workload
        // shuffle is designed for.
        let samples: Vec<f32> = (0..1024).map(|i| (i as f32 * 0.1).sin()).collect();
        let bytes: Vec<u8> = samples.iter().flat_map(|f| f.to_le_bytes()).collect();
        let codec = float32();
        let compressed = codec.compress(&bytes).expect("compress");
        let recovered = codec
            .decompress(&compressed, bytes.len() as u32)
            .expect("decompress");
        assert_eq!(recovered, bytes);
    }

    #[test]
    fn beats_plain_lz4_on_smooth_floats() {
        let samples: Vec<f32> = (0..4096).map(|i| (i as f32 * 0.01).sin()).collect();
        let bytes: Vec<u8> = samples.iter().flat_map(|f| f.to_le_bytes()).collect();
        let codec = float32();
        let shuffled_compressed = codec.compress(&bytes).expect("shuffle+lz4");
        let plain = crate::codec::compress(crate::codec::CODEC_LZ4, &bytes).expect("plain lz4");
        // Shuffle groups similar-significance bytes together, which
        // should give LZ4 better matches. With omnizip 0.14.40's
        // incompressibility detector in LZ4, the benefit can be
        // marginal on some inputs. Assert shuffle+LZ4 is within 5%
        // of plain LZ4 (not strictly better — the shuffle overhead
        // is negligible vs the match-finding difference).
        let ratio = shuffled_compressed.len() as f64 / plain.len().max(1) as f64;
        assert!(
            ratio <= 1.05,
            "shuffle+lz4 ({}) should be within 5% of plain LZ4 ({}) on smooth floats (ratio {:.3})",
            shuffled_compressed.len(),
            plain.len(),
            ratio
        );
    }
}