Skip to main content

openreadout_codecs/
blosc.rs

1//! Blosc 1 chunk decoding and LZ4 block decoding (pure Rust).
2//!
3//! Blosc chunk layout from the public c-blosc documentation (`README_CHUNK_FORMAT.rst`) and the
4//! BSD-3-licensed c-blosc sources (`blosc.c` block splitting rules, `blosclz.c` decoder,
5//! `shuffle.c` bit-unshuffle rules); the bit transpose from the MIT-licensed bitshuffle
6//! library's description; Snappy from its public format description; LZ4 blocks through the
7//! MIT-licensed `lz4_flex` crate. Provenance: `docs/provenance/ome-zarr.md`.
8//!
9//! Every length in the input is checked before use; malformed input is an error, never a panic.
10
11use crate::{CodecError, Result};
12
13/// Byte-shuffle filter applied (flags bit 0).
14const FLAG_SHUFFLE: u8 = 0x01;
15/// The payload is stored uncompressed after the header (flags bit 1).
16const FLAG_MEMCPYED: u8 = 0x02;
17/// Bit-shuffle filter applied (flags bit 2).
18const FLAG_BITSHUFFLE: u8 = 0x04;
19/// Blocks are not split into one stream per byte of the type (flags bit 4).
20const FLAG_DONT_SPLIT: u8 = 0x10;
21const HEADER: usize = 16;
22const MAX_SPLITS: usize = 16;
23const MIN_BUFFERSIZE: usize = 128;
24/// Largest decoded chunk accepted (a guard against absurd headers).
25const MAX_DECODED: usize = 1 << 31;
26
27fn err(detail: impl Into<String>) -> CodecError {
28    CodecError::Decode {
29        codec: "blosc",
30        detail: detail.into(),
31    }
32}
33
34fn le32(b: &[u8], at: usize) -> Option<usize> {
35    let s = b.get(at..at.checked_add(4)?)?;
36    Some(u32::from_le_bytes([s[0], s[1], s[2], s[3]]) as usize)
37}
38
39/// The compressor recorded in a Blosc header (flags bits 5–7).
40#[derive(Debug, Clone, Copy, PartialEq, Eq)]
41#[non_exhaustive]
42pub enum BloscCompressor {
43    /// Blosc's own LZ77 variant.
44    BloscLz,
45    /// LZ4 block format.
46    Lz4,
47    /// Snappy.
48    Snappy,
49    /// zlib (deflate with a zlib header).
50    Zlib,
51    /// Zstandard.
52    Zstd,
53    /// A compressor code this decoder does not know.
54    Unknown(u8),
55}
56
57impl BloscCompressor {
58    fn from_flags(flags: u8) -> Self {
59        match flags >> 5 {
60            0 => Self::BloscLz,
61            1 => Self::Lz4,
62            2 => Self::Snappy,
63            3 => Self::Zlib,
64            4 => Self::Zstd,
65            n => Self::Unknown(n),
66        }
67    }
68    /// Name as numcodecs spells it (`cname`).
69    pub fn name(self) -> &'static str {
70        match self {
71            Self::BloscLz => "blosclz",
72            Self::Lz4 => "lz4",
73            Self::Snappy => "snappy",
74            Self::Zlib => "zlib",
75            Self::Zstd => "zstd",
76            Self::Unknown(_) => "unknown",
77        }
78    }
79}
80
81/// Header fields of a Blosc 1 chunk.
82#[derive(Debug, Clone, Copy, PartialEq, Eq)]
83pub struct BloscHeader {
84    /// Format version of the Blosc header.
85    pub version: u8,
86    /// Flag bits: shuffle mode (bits 0 and 2), memcpy'd (bit 1), compressor (bits 5 to 7).
87    pub flags: u8,
88    /// Size of one element in bytes, used by the shuffle filters.
89    pub typesize: usize,
90    /// Decoded size in bytes.
91    pub nbytes: usize,
92    /// Uncompressed size of one block in bytes.
93    pub blocksize: usize,
94    /// Compressed size including the header.
95    pub cbytes: usize,
96    /// The compressor that produced the blocks.
97    pub compressor: BloscCompressor,
98}
99
100/// Parse the 16-byte Blosc header.
101pub fn blosc_header(data: &[u8]) -> Result<BloscHeader> {
102    if data.len() < HEADER {
103        return Err(err(format!(
104            "{} bytes is shorter than the 16-byte header",
105            data.len()
106        )));
107    }
108    let flags = data[2];
109    Ok(BloscHeader {
110        version: data[0],
111        flags,
112        typesize: data[3] as usize,
113        nbytes: le32(data, 4).unwrap_or(0),
114        blocksize: le32(data, 8).unwrap_or(0),
115        cbytes: le32(data, 12).unwrap_or(0),
116        compressor: BloscCompressor::from_flags(flags),
117    })
118}
119
120/// Decode one Blosc 1 chunk (as written by c-blosc 1.x / numcodecs `Blosc`).
121pub fn blosc_decode(data: &[u8]) -> Result<Vec<u8>> {
122    let h = blosc_header(data)?;
123    if h.nbytes > MAX_DECODED {
124        return Err(err(format!("header declares {} decoded bytes", h.nbytes)));
125    }
126    if h.cbytes > data.len() {
127        return Err(err(format!(
128            "header declares {} compressed bytes, {} present (truncated)",
129            h.cbytes,
130            data.len()
131        )));
132    }
133    let data = &data[..h.cbytes];
134    if h.nbytes == 0 {
135        return Ok(Vec::new());
136    }
137    if h.flags & FLAG_MEMCPYED != 0 {
138        let body = data
139            .get(HEADER..HEADER + h.nbytes)
140            .ok_or_else(|| err("stored (memcpyed) payload is truncated"))?;
141        return Ok(body.to_vec());
142    }
143    if h.blocksize == 0 || h.typesize == 0 {
144        return Err(err("block size or type size is 0"));
145    }
146    if h.blocksize > MAX_DECODED {
147        return Err(err(format!("header declares a {}-byte block", h.blocksize)));
148    }
149    let nblocks = h.nbytes.div_ceil(h.blocksize);
150    let leftover = h.nbytes % h.blocksize;
151    let mut out = vec![0u8; h.nbytes];
152    let mut tmp = vec![0u8; h.blocksize.min(h.nbytes)];
153    let shuffle = h.flags & FLAG_SHUFFLE != 0 && h.typesize > 1;
154    let bitshuffle = h.flags & FLAG_BITSHUFFLE != 0 && h.blocksize >= h.typesize;
155    for b in 0..nblocks {
156        let start = le32(data, HEADER + 4 * b).ok_or_else(|| err("block offsets truncated"))?;
157        let is_leftover = leftover != 0 && b == nblocks - 1;
158        let bsize = if is_leftover { leftover } else { h.blocksize };
159        let nsplits = if h.flags & FLAG_DONT_SPLIT == 0
160            && h.typesize <= MAX_SPLITS
161            && h.blocksize / h.typesize >= MIN_BUFFERSIZE
162            && !is_leftover
163        {
164            h.typesize
165        } else {
166            1
167        };
168        let neblock = bsize / nsplits;
169        let mut src = start;
170        let mut filled = 0usize;
171        for _ in 0..nsplits {
172            let cbytes = le32(data, src).ok_or_else(|| err("split size truncated"))?;
173            src += 4;
174            let end = src
175                .checked_add(cbytes)
176                .ok_or_else(|| err("split size overflows"))?;
177            let stream = data
178                .get(src..end)
179                .ok_or_else(|| err(format!("block {b}: split runs past the chunk")))?;
180            let dst = tmp
181                .get_mut(filled..filled + neblock)
182                .ok_or_else(|| err("block larger than declared"))?;
183            if cbytes == neblock {
184                dst.copy_from_slice(stream);
185            } else {
186                let dec = decompress_stream(h.compressor, stream, neblock)?;
187                if dec.len() != neblock {
188                    return Err(err(format!(
189                        "block {b}: split decoded to {} bytes, expected {neblock}",
190                        dec.len()
191                    )));
192                }
193                dst.copy_from_slice(&dec);
194            }
195            src = end;
196            filled += neblock;
197        }
198        let dest = out
199            .get_mut(b * h.blocksize..b * h.blocksize + bsize)
200            .ok_or_else(|| err("block outside the decoded chunk"))?;
201        if shuffle {
202            unshuffle(h.typesize, &tmp[..bsize], dest);
203        } else if bitshuffle {
204            bitunshuffle(h.typesize, &tmp[..bsize], dest);
205        } else {
206            dest.copy_from_slice(&tmp[..bsize]);
207        }
208    }
209    Ok(out)
210}
211
212/// Undo the Blosc byte shuffle: byte `j` of element `i` was stored at `j * n + i`; trailing
213/// bytes that do not fill a whole element are stored as they are.
214fn unshuffle(typesize: usize, src: &[u8], dest: &mut [u8]) {
215    let n = src.len() / typesize;
216    for i in 0..n {
217        for j in 0..typesize {
218            dest[i * typesize + j] = src[j * n + i];
219        }
220    }
221    let done = n * typesize;
222    dest[done..].copy_from_slice(&src[done..]);
223}
224
225/// Undo the bit shuffle of one block (c-blosc `bitunshuffle`): when the block holds a multiple
226/// of 8 whole elements, bit `k` of byte `j` of element `i` was stored at bit `i % 8` of byte
227/// `(j * 8 + k) * n / 8 + i / 8` (`n` elements; one row of `n` bits per bit of each byte);
228/// bytes after the whole elements are stored as they are. Other blocks are stored unshuffled.
229fn bitunshuffle(typesize: usize, src: &[u8], dest: &mut [u8]) {
230    let n = src.len() / typesize;
231    if !n.is_multiple_of(8) {
232        dest.copy_from_slice(src);
233        return;
234    }
235    let row = n / 8;
236    let done = n * typesize;
237    // Eight bit rows of byte `j` give, at byte `g` of each row, an 8 × 8 bit matrix whose
238    // transpose is byte `j` of elements 8g..8g+8.
239    for j in 0..typesize {
240        let rows = &src[j * 8 * row..(j + 1) * 8 * row];
241        for g in 0..row {
242            let mut x = 0u64;
243            for k in 0..8 {
244                x |= u64::from(rows[k * row + g]) << (8 * k);
245            }
246            let y = transpose8(x).to_le_bytes();
247            for (i, b) in y.iter().enumerate() {
248                dest[(g * 8 + i) * typesize + j] = *b;
249            }
250        }
251    }
252    dest[done..].copy_from_slice(&src[done..]);
253}
254
255/// Transpose an 8 × 8 bit matrix stored as eight bytes (row `r` in byte `r`, column `c` in
256/// bit `c`): bit `8r + c` moves to bit `8c + r`.
257fn transpose8(mut x: u64) -> u64 {
258    let t = (x ^ (x >> 7)) & 0x00AA_00AA_00AA_00AA;
259    x ^= t ^ (t << 7);
260    let t = (x ^ (x >> 14)) & 0x0000_CCCC_0000_CCCC;
261    x ^= t ^ (t << 14);
262    let t = (x ^ (x >> 28)) & 0x0000_0000_F0F0_F0F0;
263    x ^ t ^ (t << 28)
264}
265
266/// The HDF5 bitshuffle filter (registered filter id 32008, bitshuffle library framing): with
267/// LZ4 (`compression` 2), a big-endian u64 decoded size and u32 block size in bytes, then per
268/// block of whole elements a big-endian u32 compressed size and an LZ4 block holding the
269/// bit-shuffled block; the elements left over after the last whole block of 8 are stored as
270/// they are at the end. Without compression (`compression` 0) the chunk is the bit-shuffled
271/// data itself, cut into blocks of `block_elems` elements (0: the library's default).
272/// `expected` is the decoded size when known (0: take it from the chunk).
273pub fn bitshuffle_decode(
274    src: &[u8],
275    elem_size: usize,
276    compression: u32,
277    block_elems: usize,
278    expected: usize,
279) -> Result<Vec<u8>> {
280    let e = |d: &str| CodecError::Decode {
281        codec: "bitshuffle",
282        detail: d.into(),
283    };
284    if elem_size == 0 {
285        return Err(e("element size 0"));
286    }
287    let be = |at: usize, n: usize| -> Option<u64> {
288        let s = src.get(at..at.checked_add(n)?)?;
289        Some(s.iter().fold(0u64, |a, &b| a << 8 | u64::from(b)))
290    };
291    let (total, block_bytes, mut at) = match compression {
292        0 => (
293            if expected == 0 { src.len() } else { expected },
294            block_elems.saturating_mul(elem_size),
295            0usize,
296        ),
297        2 => {
298            let total = usize::try_from(be(0, 8).ok_or_else(|| e("header truncated"))?)
299                .map_err(|_| e("declared size too large"))?;
300            let block = be(8, 4).ok_or_else(|| e("header truncated"))? as usize;
301            (total, block, 12)
302        }
303        other => {
304            return Err(CodecError::Decode {
305                codec: "bitshuffle",
306                detail: format!("compression {other} (only none and LZ4 are decoded)"),
307            });
308        }
309    };
310    if total > MAX_DECODED || (expected != 0 && total != expected) {
311        return Err(e("declared size does not match the chunk"));
312    }
313    // LZ4 expands at most about 255 times: a larger declared size is not this chunk's.
314    if compression == 2 && total > src.len().saturating_mul(256).saturating_add(64) {
315        return Err(e("declared size is larger than the chunk can hold"));
316    }
317    if !total.is_multiple_of(elem_size) {
318        return Err(e("decoded size is not a whole number of elements"));
319    }
320    let n = total / elem_size;
321    // The library's default block: 8 KiB of elements, a multiple of 8, at least 128.
322    let block_n = if block_bytes == 0 {
323        ((8192 / elem_size) / 8 * 8).max(128)
324    } else {
325        block_bytes / elem_size
326    };
327    if block_n == 0 || !block_n.is_multiple_of(8) {
328        return Err(e("block size is not a multiple of 8 elements"));
329    }
330    let mut out = vec![0u8; total];
331    let mut done = 0usize; // elements decoded
332    let tail = n % 8;
333    while done + 8 <= n {
334        let this = block_n.min((n - tail) - done);
335        let bytes = this * elem_size;
336        let shuffled: std::borrow::Cow<'_, [u8]> = if compression == 2 {
337            let c = be(at, 4).ok_or_else(|| e("block header truncated"))? as usize;
338            at += 4;
339            let end = at.checked_add(c).ok_or_else(|| e("block size overflows"))?;
340            let s = src
341                .get(at..end)
342                .ok_or_else(|| e("block runs past the chunk"))?;
343            at = end;
344            std::borrow::Cow::Owned(lz4_block_decode(s, bytes)?)
345        } else {
346            let s = src
347                .get(at..at + bytes)
348                .ok_or_else(|| e("chunk shorter than its data"))?;
349            at += bytes;
350            std::borrow::Cow::Borrowed(s)
351        };
352        if shuffled.len() != bytes {
353            return Err(e("block decoded to the wrong size"));
354        }
355        let dest = &mut out[done * elem_size..done * elem_size + bytes];
356        bitunshuffle(elem_size, &shuffled, dest);
357        done += this;
358    }
359    let rest = tail * elem_size;
360    let s = src
361        .get(at..at + rest)
362        .ok_or_else(|| e("chunk ends before its last elements"))?;
363    out[done * elem_size..].copy_from_slice(s);
364    Ok(out)
365}
366
367/// A Snappy raw block (public format description, `format_description.txt`): a varint decoded
368/// length, then literals and back references (1, 2 or 4-byte offsets). Must decode to exactly
369/// `expected` bytes.
370pub fn snappy_decode(src: &[u8], expected: usize) -> Result<Vec<u8>> {
371    let bad = |d: &str| CodecError::Decode {
372        codec: "snappy",
373        detail: d.into(),
374    };
375    let mut ip = 0usize;
376    let mut len = 0usize;
377    let mut shift = 0u32;
378    loop {
379        let b = *src.get(ip).ok_or_else(|| bad("length truncated"))?;
380        ip += 1;
381        len |= usize::from(b & 0x7f)
382            .checked_shl(shift)
383            .ok_or_else(|| bad("length overflows"))?;
384        if b & 0x80 == 0 {
385            break;
386        }
387        shift += 7;
388        if shift > 28 {
389            return Err(bad("length overflows"));
390        }
391    }
392    if len != expected || len > MAX_DECODED {
393        return Err(bad(&format!("declares {len} bytes, expected {expected}")));
394    }
395    let mut out: Vec<u8> = Vec::with_capacity(len);
396    let le = |at: usize, n: usize| -> Option<usize> {
397        let s = src.get(at..at.checked_add(n)?)?;
398        Some(s.iter().rev().fold(0usize, |a, &b| a << 8 | usize::from(b)))
399    };
400    while ip < src.len() {
401        let tag = src[ip];
402        ip += 1;
403        let (n, offset) = match tag & 3 {
404            0 => {
405                let mut n = usize::from(tag >> 2);
406                if n >= 60 {
407                    let extra = n - 59;
408                    n = le(ip, extra).ok_or_else(|| bad("literal length truncated"))?;
409                    ip += extra;
410                }
411                n += 1;
412                let lit = src
413                    .get(ip..ip.checked_add(n).ok_or_else(|| bad("literal overflows"))?)
414                    .ok_or_else(|| bad("literal runs past the input"))?;
415                if out.len() + n > len {
416                    return Err(bad("literal runs past the declared length"));
417                }
418                out.extend_from_slice(lit);
419                ip += n;
420                continue;
421            }
422            1 => {
423                let o = *src.get(ip).ok_or_else(|| bad("copy truncated"))?;
424                ip += 1;
425                (
426                    4 + usize::from((tag >> 2) & 7),
427                    usize::from(tag >> 5) << 8 | usize::from(o),
428                )
429            }
430            2 => {
431                let o = le(ip, 2).ok_or_else(|| bad("copy truncated"))?;
432                ip += 2;
433                (1 + usize::from(tag >> 2), o)
434            }
435            _ => {
436                let o = le(ip, 4).ok_or_else(|| bad("copy truncated"))?;
437                ip += 4;
438                (1 + usize::from(tag >> 2), o)
439            }
440        };
441        if offset == 0 || offset > out.len() || out.len() + n > len {
442            return Err(bad("back reference outside the output"));
443        }
444        let from = out.len() - offset;
445        for k in 0..n {
446            let v = out[from + k];
447            out.push(v);
448        }
449    }
450    if out.len() != len {
451        return Err(bad(&format!("decoded {} of {len} bytes", out.len())));
452    }
453    Ok(out)
454}
455
456fn decompress_stream(c: BloscCompressor, src: &[u8], expected: usize) -> Result<Vec<u8>> {
457    match c {
458        BloscCompressor::BloscLz => blosclz_decode(src, expected),
459        BloscCompressor::Lz4 => lz4_block_decode(src, expected),
460        BloscCompressor::Snappy => snappy_decode(src, expected),
461        BloscCompressor::Zlib => crate::zlib_decode(src, expected),
462        BloscCompressor::Zstd => crate::zstd_decode(src, expected),
463        BloscCompressor::Unknown(_) => Err(err(format!(
464            "the {} compressor inside Blosc is not decoded",
465            c.name()
466        ))),
467    }
468}
469
470/// BloscLZ (the default Blosc compressor), following `blosclz_decompress` in c-blosc (BSD-3).
471pub fn blosclz_decode(src: &[u8], maxout: usize) -> Result<Vec<u8>> {
472    const MAX_DISTANCE: usize = 8191;
473    let bad = || err("malformed BloscLZ stream");
474    let mut out: Vec<u8> = Vec::with_capacity(maxout);
475    if src.is_empty() {
476        return Ok(out);
477    }
478    let mut ip = 0usize;
479    let mut ctrl = usize::from(src[ip] & 31);
480    ip += 1;
481    loop {
482        if ctrl >= 32 {
483            let mut len = (ctrl >> 5) - 1;
484            let ofs = (ctrl & 31) << 8;
485            if len == 6 {
486                loop {
487                    if ip + 1 >= src.len() {
488                        return Err(bad());
489                    }
490                    let code = src[ip];
491                    ip += 1;
492                    len += usize::from(code);
493                    if code != 255 {
494                        break;
495                    }
496                }
497            } else if ip + 1 >= src.len() {
498                return Err(bad());
499            }
500            let code = usize::from(src[ip]);
501            ip += 1;
502            len += 3;
503            // distance back from the output position, minus one (see `ref--` in C)
504            let mut back = ofs + code;
505            if code == 255 && ofs == (31 << 8) {
506                if ip + 1 >= src.len() {
507                    return Err(bad());
508                }
509                let far = (usize::from(src[ip]) << 8) + usize::from(src[ip + 1]);
510                ip += 2;
511                back = far + MAX_DISTANCE;
512            }
513            back += 1;
514            if out.len() + len > maxout || back > out.len() {
515                return Err(bad());
516            }
517            let from = out.len() - back;
518            for k in 0..len {
519                let v = out[from + k];
520                out.push(v);
521            }
522            if ip >= src.len() {
523                break;
524            }
525            ctrl = usize::from(src[ip]);
526            ip += 1;
527        } else {
528            let n = ctrl + 1;
529            if out.len() + n > maxout || ip + n > src.len() {
530                return Err(bad());
531            }
532            out.extend_from_slice(&src[ip..ip + n]);
533            ip += n;
534            if ip >= src.len() {
535                break;
536            }
537            ctrl = usize::from(src[ip]);
538            ip += 1;
539        }
540    }
541    Ok(out)
542}
543
544/// One raw LZ4 block (no frame, no size prefix) that decodes to exactly `expected` bytes.
545pub fn lz4_block_decode(src: &[u8], expected: usize) -> Result<Vec<u8>> {
546    #[cfg(feature = "lz4")]
547    {
548        lz4_flex::block::decompress(src, expected).map_err(|e| CodecError::Decode {
549            codec: "lz4",
550            detail: e.to_string(),
551        })
552    }
553    #[cfg(not(feature = "lz4"))]
554    {
555        let _ = (src, expected);
556        Err(CodecError::NotCompiled {
557            codec: "lz4",
558            feature: "lz4",
559        })
560    }
561}
562
563/// numcodecs `LZ4`: a little-endian u32 decoded size, then one LZ4 block.
564pub fn lz4_sized_decode(src: &[u8]) -> Result<Vec<u8>> {
565    let n = le32(src, 0).ok_or_else(|| CodecError::Decode {
566        codec: "lz4",
567        detail: "missing 4-byte size prefix".into(),
568    })?;
569    if n > MAX_DECODED {
570        return Err(CodecError::Decode {
571            codec: "lz4",
572            detail: format!("size prefix declares {n} bytes"),
573        });
574    }
575    lz4_block_decode(&src[4..], n)
576}
577
578/// The HDF5 LZ4 filter (registered filter id 32004): a big-endian u64 decoded size and a
579/// big-endian u32 block size, then per block a big-endian u32 compressed size and the LZ4
580/// block (stored as is when the compressed size equals the block's decoded size).
581pub fn hdf5_lz4_decode(src: &[u8]) -> Result<Vec<u8>> {
582    let e = |d: &str| CodecError::Decode {
583        codec: "hdf5-lz4",
584        detail: d.into(),
585    };
586    let be = |at: usize, n: usize| -> Option<u64> {
587        let s = src.get(at..at.checked_add(n)?)?;
588        Some(s.iter().fold(0u64, |a, &b| a << 8 | u64::from(b)))
589    };
590    let total = usize::try_from(be(0, 8).ok_or_else(|| e("header truncated"))?)
591        .map_err(|_| e("declared size too large"))?;
592    let block = be(8, 4).ok_or_else(|| e("header truncated"))? as usize;
593    if total > MAX_DECODED {
594        return Err(e("declared size too large"));
595    }
596    if block == 0 && total > 0 {
597        return Err(e("block size 0"));
598    }
599    let mut out = Vec::with_capacity(total);
600    let mut at = 12usize;
601    while out.len() < total {
602        let want = block.min(total - out.len());
603        let c = be(at, 4).ok_or_else(|| e("block header truncated"))? as usize;
604        at += 4;
605        let end = at.checked_add(c).ok_or_else(|| e("block size overflows"))?;
606        let s = src
607            .get(at..end)
608            .ok_or_else(|| e("block runs past the chunk"))?;
609        if c == want {
610            out.extend_from_slice(s);
611        } else {
612            out.extend_from_slice(&lz4_block_decode(s, want)?);
613        }
614        at = end;
615    }
616    Ok(out)
617}
618
619#[cfg(test)]
620mod tests {
621    use super::*;
622
623    #[test]
624    fn memcpyed_and_shuffled_stored_blocks() {
625        // stored payload
626        let mut c = vec![
627            2u8,
628            1,
629            FLAG_MEMCPYED,
630            2,
631            4,
632            0,
633            0,
634            0,
635            4,
636            0,
637            0,
638            0,
639            20,
640            0,
641            0,
642            0,
643        ];
644        c.extend_from_slice(&[1, 2, 3, 4]);
645        assert_eq!(blosc_decode(&c).unwrap(), vec![1, 2, 3, 4]);
646        // one block of 4 u16 values, shuffled, split size == neblock (stored raw)
647        let mut c = vec![
648            2u8,
649            1,
650            FLAG_SHUFFLE | FLAG_DONT_SPLIT,
651            2,
652            8,
653            0,
654            0,
655            0,
656            8,
657            0,
658            0,
659            0,
660        ];
661        let body_start = 16 + 4;
662        c.extend_from_slice(&((body_start + 4 + 8) as u32).to_le_bytes());
663        c.extend_from_slice(&(body_start as u32).to_le_bytes());
664        c.extend_from_slice(&8u32.to_le_bytes());
665        // values 0x0201, 0x0403, 0x0605, 0x0807 shuffled: low bytes then high bytes
666        c.extend_from_slice(&[1, 3, 5, 7, 2, 4, 6, 8]);
667        assert_eq!(blosc_decode(&c).unwrap(), vec![1, 2, 3, 4, 5, 6, 7, 8]);
668        // truncated
669        assert!(blosc_decode(&c[..c.len() - 3]).is_err());
670        assert!(blosc_decode(&c[..10]).is_err());
671    }
672
673    #[test]
674    fn bitunshuffle_transposes_bits() {
675        // 8 u16 elements 0x0001, 0x0002, ..., 0x0008: bit rows of 8 bits (1 byte) each
676        let vals: Vec<u16> = (1..=8).collect();
677        let mut shuffled = vec![0u8; 16];
678        for (i, v) in vals.iter().enumerate() {
679            for j in 0..2 {
680                let byte = v.to_le_bytes()[j];
681                for k in 0..8 {
682                    if byte >> k & 1 == 1 {
683                        shuffled[j * 8 + k] |= 1 << i;
684                    }
685                }
686            }
687        }
688        let mut out = vec![0u8; 16];
689        bitunshuffle(2, &shuffled, &mut out);
690        let back: Vec<u16> = out
691            .chunks(2)
692            .map(|c| u16::from_le_bytes([c[0], c[1]]))
693            .collect();
694        assert_eq!(back, vals);
695        // not a multiple of 8 elements: stored as it is
696        let mut out = vec![0u8; 6];
697        bitunshuffle(2, &[1, 2, 3, 4, 5, 6], &mut out);
698        assert_eq!(out, [1, 2, 3, 4, 5, 6]);
699    }
700
701    #[test]
702    fn transpose8_matches_bit_by_bit() {
703        for seed in [1u64, 0x0123_4567_89ab_cdef, u64::MAX, 0x8000_0000_0000_0001] {
704            let t = transpose8(seed);
705            for r in 0..8 {
706                for c in 0..8 {
707                    assert_eq!(seed >> (8 * r + c) & 1, t >> (8 * c + r) & 1);
708                }
709            }
710        }
711    }
712
713    /// Bit-shuffle `vals` (u16) the slow way.
714    fn shuffle_u16(vals: &[u16]) -> Vec<u8> {
715        let n = vals.len();
716        let mut out = vec![0u8; n * 2];
717        for (i, v) in vals.iter().enumerate() {
718            for j in 0..2 {
719                for k in 0..8 {
720                    if v.to_le_bytes()[j] >> k & 1 == 1 {
721                        out[(j * 8 + k) * n / 8 + i / 8] |= 1 << (i % 8);
722                    }
723                }
724            }
725        }
726        out
727    }
728
729    #[cfg(feature = "lz4")]
730    #[test]
731    fn bitshuffle_lz4_chunks_with_a_tail() {
732        // 21 u16 values: two blocks of 8 (block size 16 bytes), then 5 left over stored raw.
733        let vals: Vec<u16> = (0..21).map(|i| (i * 977 + 3) as u16).collect();
734        let mut c = Vec::new();
735        c.extend_from_slice(&(42u64).to_be_bytes());
736        c.extend_from_slice(&(16u32).to_be_bytes());
737        for b in 0..2 {
738            let block = shuffle_u16(&vals[b * 8..b * 8 + 8]);
739            let z = lz4_flex::block::compress(&block);
740            c.extend_from_slice(&(z.len() as u32).to_be_bytes());
741            c.extend_from_slice(&z);
742        }
743        for v in &vals[16..] {
744            c.extend_from_slice(&v.to_le_bytes());
745        }
746        let out = bitshuffle_decode(&c, 2, 2, 0, 42).unwrap();
747        let back: Vec<u16> = out
748            .chunks(2)
749            .map(|c| u16::from_le_bytes([c[0], c[1]]))
750            .collect();
751        assert_eq!(back, vals);
752        assert!(bitshuffle_decode(&c[..c.len() - 1], 2, 2, 0, 42).is_err());
753        assert!(bitshuffle_decode(&c, 2, 2, 0, 40).is_err());
754        assert!(bitshuffle_decode(&c, 2, 3, 0, 42).is_err());
755        // fuzz finding: a declared size that is not a whole number of elements
756        assert!(bitshuffle_decode(&c, 4, 2, 0, 42).is_err());
757        // uncompressed: the shuffled data itself
758        let raw = shuffle_u16(&vals[..16]);
759        let out = bitshuffle_decode(&raw, 2, 0, 16, 32).unwrap();
760        assert_eq!(
761            out.chunks(2)
762                .map(|c| u16::from_le_bytes([c[0], c[1]]))
763                .collect::<Vec<_>>(),
764            vals[..16]
765        );
766    }
767
768    #[test]
769    fn snappy_literals_and_copies() {
770        // "abcabcabcd": literal "abc", copy len 6 offset 3 (1-byte offset form), literal "d"
771        let src = [10u8, 0x08, b'a', b'b', b'c', 0x09, 3, 0x00, b'd'];
772        assert_eq!(snappy_decode(&src, 10).unwrap(), b"abcabcabcd");
773        assert!(snappy_decode(&src, 9).is_err());
774        assert!(snappy_decode(&src[..6], 10).is_err());
775        assert!(snappy_decode(&[4, 0x05, 9, 0], 4).is_err());
776    }
777
778    #[test]
779    fn blosclz_literal_and_match() {
780        // literal run of 3 ("abc"), a match of length 3 at distance 3, a literal "d"
781        let src = [2u8, b'a', b'b', b'c', 0x20, 2, 0, b'd'];
782        assert_eq!(blosclz_decode(&src, 7).unwrap(), b"abcabcd");
783        assert!(blosclz_decode(&[0, b'x', 0x20, 5, 0, b'y'], 8).is_err());
784    }
785
786    #[test]
787    fn hdf5_lz4_stored_block() {
788        let mut s = Vec::new();
789        s.extend_from_slice(&4u64.to_be_bytes());
790        s.extend_from_slice(&4u32.to_be_bytes());
791        s.extend_from_slice(&4u32.to_be_bytes());
792        s.extend_from_slice(&[9, 8, 7, 6]);
793        assert_eq!(hdf5_lz4_decode(&s).unwrap(), vec![9, 8, 7, 6]);
794        assert!(hdf5_lz4_decode(&s[..14]).is_err());
795    }
796}