Skip to main content

cachekit_core/
byte_storage.rs

1//! LZ4 compression and xxHash3 checksums for raw byte storage.
2//!
3//! Provides integrity-protected byte storage with security validation:
4//! - 512MB size limits for decompression bomb protection
5//! - 1000x max compression ratio enforcement
6//! - xxHash3-64 checksums for corruption detection
7
8#[cfg(feature = "compression")]
9use lz4_flex;
10use serde::{Deserialize, Serialize};
11use thiserror::Error;
12
13/// Error types for ByteStorage operations
14#[derive(Debug, Error, Clone, PartialEq)]
15pub enum ByteStorageError {
16    #[error("input exceeds maximum size")]
17    InputTooLarge,
18
19    #[error("decompression ratio exceeds safety limit")]
20    DecompressionBomb,
21
22    #[error("integrity check failed")]
23    ChecksumMismatch,
24
25    #[error("compression failed")]
26    CompressionFailed,
27
28    #[error("decompression failed")]
29    DecompressionFailed,
30
31    #[error("size validation failed")]
32    SizeValidationFailed,
33
34    #[error("serialization failed: {0}")]
35    SerializationFailed(String),
36
37    #[error("deserialization failed: {0}")]
38    DeserializationFailed(String),
39}
40
41// Security constants - Production-safe limits
42const MAX_UNCOMPRESSED_SIZE: usize = 512 * 1024 * 1024; // 512MB limit
43const MAX_COMPRESSED_SIZE: usize = 512 * 1024 * 1024; // 512MB limit
44/// Maximum allowed compression ratio (1000:1)
45/// Uses u64 for integer-only arithmetic to prevent floating-point precision bypass attacks
46const MAX_COMPRESSION_RATIO: u64 = 1000;
47
48/// Buffer length from which `store` and `retrieve` keep their copying
49/// allocation pattern. `store` compares the encoded envelope length: from
50/// here on it encodes through `rmp_serde::to_vec`'s growing buffer instead of
51/// a buffer allocated once at the exact length. `retrieve` and `validate`
52/// compare the compressed payload length: from here on they copy the payload
53/// out of the input instead of borrowing it.
54///
55/// glibc's malloc raises its mmap threshold to the largest mmap-ed chunk
56/// freed so far, and its trim threshold to twice that, and trims the top of the heap once it holds more free
57/// memory than the trim threshold, so dropping one payload-sized buffer can
58/// send a large operation's other payload-sized buffers to fresh pages on
59/// every call. Measured on 1 MB incompressible values under glibc's defaults:
60/// an exact-size store buffer took about a third more page faults and 30%
61/// more CPU per store (`to_vec`'s doubled reallocation had been the chunk that
62/// raised the thresholds), and a borrowed payload took each read from no page
63/// faults to about a thousand, several times the CPU (its remaining buffers
64/// grew the heap past the trim threshold and were trimmed again every call).
65/// Below 64 KiB neither buffer reaches glibc's minimum mmap threshold
66/// (128 KiB), even doubled, so it is never mmap-ed, and removing it cannot
67/// move either threshold.
68#[cfg(all(feature = "compression", feature = "checksum", feature = "messagepack"))]
69const LARGE_PAYLOAD: usize = 64 * 1024;
70
71/// Inputs shorter than this are written as one literals-only LZ4 block instead
72/// of going through `lz4_flex::compress`.
73///
74/// `lz4_flex::compress` zeroes an 8 KiB hash table on every call, which costs
75/// more than the whole store for a tiny value, and below LZ4's 13-byte minimum
76/// its output is literals-only anyway. The price is stored size: a compressible
77/// input under the threshold is no longer shrunk; its block is the input plus
78/// 1 byte (plus 2 from 15 bytes up). Compressed bytes are not canonical
79/// (protocol `spec/wire-format.md`, "Compressed-byte reproducibility"), and
80/// every reader decodes a literals-only block.
81#[cfg(feature = "compression")]
82const LITERAL_BLOCK_THRESHOLD: usize = 64;
83
84// The literal block writes the length in at most one extra byte, which holds
85// for literal lengths up to 15 + 254.
86#[cfg(feature = "compression")]
87const _: () = assert!(LITERAL_BLOCK_THRESHOLD - 1 <= 15 + 254);
88
89/// The LZ4 block `StorageEnvelope::new` writes for `data`.
90///
91/// Below the threshold it is one LZ4 sequence with no match: a token whose
92/// high nibble is the literal length (15 = one more length byte follows), then
93/// the literals. This is the LZ4 block format's last-sequence form, so it is a
94/// complete block on its own.
95#[cfg(feature = "compression")]
96fn compress_block(data: &[u8]) -> Vec<u8> {
97    if data.len() >= LITERAL_BLOCK_THRESHOLD {
98        return lz4_flex::compress(data);
99    }
100    let len = data.len();
101    let mut block = Vec::with_capacity(len + 2);
102    if len < 15 {
103        block.push((len as u8) << 4);
104    } else {
105        block.push(0xF0);
106        block.push((len - 15) as u8);
107    }
108    block.extend_from_slice(data);
109    block
110}
111
112/// Storage envelope for raw byte storage
113/// Contains compressed data with integrity checking
114// `retrieve` and `validate` decode into its private borrowing twin,
115// `borrowed::StorageEnvelope`: change the two together.
116#[derive(Serialize, Deserialize)]
117pub struct StorageEnvelope {
118    /// Compressed payload data
119    ///
120    /// Serialized as msgpack `bin` (protocol 1.1, `envelope-bin-encoding.md`).
121    /// Scope is strictly this field: `checksum` stays array-of-ints by
122    /// normative exclusion (crypto-adjacent surface — MUST NOT flip).
123    #[serde(with = "serde_bytes")]
124    pub compressed_data: Vec<u8>,
125    /// xxHash3-64 checksum for integrity (8 bytes)
126    pub checksum: [u8; 8],
127    /// Original size for validation
128    pub original_size: u32,
129    /// Format identifier (e.g., "msgpack")
130    pub format: String,
131}
132
133impl StorageEnvelope {
134    /// Create new envelope with data compression and checksum.
135    ///
136    /// Takes the input by shared slice: it is only compressed and hashed (both
137    /// borrow), never retained, so there is no reason to own it. Avoids a
138    /// full-payload copy (up to `MAX_UNCOMPRESSED_SIZE`) on the write path.
139    #[cfg(all(feature = "compression", feature = "checksum"))]
140    pub fn new(data: &[u8], format: String) -> Result<Self, ByteStorageError> {
141        // Security: Check input size before compression
142        if data.len() > MAX_UNCOMPRESSED_SIZE {
143            return Err(ByteStorageError::InputTooLarge);
144        }
145
146        let original_size = data.len() as u32;
147
148        // Compress with LZ4 (a literals-only block below LITERAL_BLOCK_THRESHOLD)
149        let compressed_data = compress_block(data);
150
151        // Security: Check compressed size
152        if compressed_data.len() > MAX_COMPRESSED_SIZE {
153            return Err(ByteStorageError::InputTooLarge);
154        }
155
156        // Single canonical xxHash3-64 definition (see crate::checksum)
157        let checksum = crate::checksum::checksum(data);
158
159        Ok(StorageEnvelope {
160            compressed_data,
161            checksum,
162            original_size,
163            format,
164        })
165    }
166
167    /// Extract and validate data from envelope
168    #[cfg(all(feature = "compression", feature = "checksum"))]
169    pub fn extract(&self) -> Result<Vec<u8>, ByteStorageError> {
170        extract_payload(&self.compressed_data, &self.checksum, self.original_size)
171    }
172
173    /// Exact length of this envelope's MessagePack encoding: the bytes
174    /// [`encode_into`](Self::encode_into) writes and [`ByteStorage::store`]
175    /// returns.
176    ///
177    /// Computed from the field lengths and values without encoding anything,
178    /// so a caller can allocate the output once, at its final size.
179    #[cfg(feature = "messagepack")]
180    pub fn encoded_len(&self) -> usize {
181        // rmp_serde writes the struct as a 4-element array, `compressed_data`
182        // as bin, `checksum` as an array of 8 uints and every uint in its
183        // shortest form; `encoded_len_matches_rmp_serde` pins each width.
184        fn uint_len(v: u32) -> usize {
185            match v {
186                0..=0x7F => 1,
187                0x80..=0xFF => 2,
188                0x100..=0xFFFF => 3,
189                _ => 5,
190            }
191        }
192        fn bin_header_len(len: usize) -> usize {
193            match len {
194                0..=0xFF => 2,
195                0x100..=0xFFFF => 3,
196                _ => 5,
197            }
198        }
199        fn str_header_len(len: usize) -> usize {
200            match len {
201                0..=31 => 1,
202                32..=0xFF => 2,
203                0x100..=0xFFFF => 3,
204                _ => 5,
205            }
206        }
207        let checksum_len = 1 + self
208            .checksum
209            .iter()
210            .map(|&b| uint_len(b.into()))
211            .sum::<usize>();
212        let fixed = 1 // fixarray(4)
213            + bin_header_len(self.compressed_data.len())
214            + checksum_len
215            + uint_len(self.original_size)
216            + str_header_len(self.format.len());
217        fixed
218            .saturating_add(self.compressed_data.len())
219            .saturating_add(self.format.len())
220    }
221
222    /// Write this envelope's MessagePack encoding, the bytes
223    /// [`ByteStorage::store`] returns, into `out`, which must be exactly
224    /// [`encoded_len`](Self::encoded_len) bytes long.
225    ///
226    /// For a binding that owns the output memory (a Python `bytes`, say): it
227    /// sizes the buffer with `encoded_len` and writes once, with no
228    /// intermediate `Vec` and no MessagePack dependency of its own. Measure
229    /// before using it for envelopes of 64 KiB and up: removing a
230    /// payload-sized buffer there can make an operation slower under glibc's
231    /// malloc (see `ByteStorage::store`, which therefore does not).
232    ///
233    /// # Errors
234    ///
235    /// `InputTooLarge` when the encoding exceeds the envelope size cap, the
236    /// same final check `store` makes. `SerializationFailed` when `out` is not
237    /// `encoded_len` bytes; nothing written into it is then meaningful.
238    #[cfg(feature = "messagepack")]
239    pub fn encode_into(&self, out: &mut [u8]) -> Result<(), ByteStorageError> {
240        let len = self.capped_encoded_len()?;
241        if out.len() != len {
242            return Err(ByteStorageError::SerializationFailed(format!(
243                "output buffer is {} bytes, the envelope encodes to {len}",
244                out.len()
245            )));
246        }
247        let mut rest = &mut out[..];
248        rmp_serde::encode::write(&mut rest, self)
249            .map_err(|e| ByteStorageError::SerializationFailed(e.to_string()))?;
250        // encoded_len was exact if the write filled the buffer
251        if !rest.is_empty() {
252            return Err(ByteStorageError::SerializationFailed(format!(
253                "envelope encoded {} bytes short of its computed length",
254                rest.len()
255            )));
256        }
257        Ok(())
258    }
259
260    /// `encoded_len`, or `InputTooLarge` past the envelope size cap: the final
261    /// check `store` makes, before anything is encoded.
262    #[cfg(feature = "messagepack")]
263    fn capped_encoded_len(&self) -> Result<usize, ByteStorageError> {
264        let len = self.encoded_len();
265        if len > MAX_COMPRESSED_SIZE {
266            return Err(ByteStorageError::InputTooLarge);
267        }
268        Ok(len)
269    }
270
271    /// The encoding `store` returns. Below `LARGE_PAYLOAD` it goes into a
272    /// `Vec` allocated once at its final length; from there on it keeps
273    /// `rmp_serde::to_vec`'s growing buffer (see `LARGE_PAYLOAD` for why).
274    ///
275    /// `rmp_serde::to_vec` grows its `Vec` from empty, reallocating several
276    /// times for a small envelope and to twice the payload for a large one.
277    #[cfg(all(feature = "compression", feature = "checksum", feature = "messagepack"))]
278    fn encode(&self) -> Result<Vec<u8>, ByteStorageError> {
279        let len = self.capped_encoded_len()?;
280        if len >= LARGE_PAYLOAD {
281            return rmp_serde::to_vec(self)
282                .map_err(|e| ByteStorageError::SerializationFailed(e.to_string()));
283        }
284        let mut out = Vec::with_capacity(len);
285        rmp_serde::encode::write(&mut out, self)
286            .map_err(|e| ByteStorageError::SerializationFailed(e.to_string()))?;
287        debug_assert_eq!(out.len(), len, "encoded_len must be exact");
288        Ok(out)
289    }
290}
291
292/// Decompress and verify one envelope's payload. `StorageEnvelope::extract`
293/// forwards to it, and `retrieve` and `validate` call it on the borrowed
294/// decode, so every read path runs the same checks in the same order.
295#[cfg(all(feature = "compression", feature = "checksum"))]
296fn extract_payload(
297    compressed_data: &[u8],
298    checksum: &[u8; 8],
299    original_size: u32,
300) -> Result<Vec<u8>, ByteStorageError> {
301    // Unit tests only: the allocation probe's positive control.
302    #[cfg(test)]
303    crate::read_allocation_probe::reserve_first_control(compressed_data, original_size);
304
305    check_decompression_bound(compressed_data.len(), original_size)?;
306
307    // Decompress (with validated sizes)
308    let decompressed = lz4_flex::decompress(compressed_data, original_size as usize)
309        .map_err(|_| ByteStorageError::DecompressionFailed)?;
310
311    // Verify checksum (checksum validation happens AFTER decompression to prevent processing corrupted data)
312    // false -> ChecksumMismatch (preserve the error variant; verify_checksum returns bool)
313    if !crate::checksum::verify_checksum(&decompressed, checksum) {
314        return Err(ByteStorageError::ChecksumMismatch);
315    }
316
317    // Verify size (final safety check)
318    if decompressed.len() != original_size as usize {
319        return Err(ByteStorageError::SizeValidationFailed);
320    }
321
322    Ok(decompressed)
323}
324
325/// Decompression bound, checked by `extract_payload` before it allocates the
326/// output.
327///
328/// A separate function so the Kani proofs verify the predicate
329/// `extract_payload` actually runs, not a restatement of it.
330#[cfg(all(feature = "compression", feature = "checksum"))]
331fn check_decompression_bound(
332    compressed_len: usize,
333    original_size: u32,
334) -> Result<(), ByteStorageError> {
335    // Size caps first
336    if compressed_len > MAX_COMPRESSED_SIZE {
337        return Err(ByteStorageError::InputTooLarge);
338    }
339
340    if original_size as usize > MAX_UNCOMPRESSED_SIZE {
341        return Err(ByteStorageError::InputTooLarge);
342    }
343
344    // Security: Check compression ratio for decompression bomb protection
345    // Uses integer arithmetic to prevent floating-point precision bypass attacks
346    let compressed_size = compressed_len as u64;
347
348    // Step 1: Zero-length compressed data is always a bomb
349    if compressed_size == 0 {
350        return Err(ByteStorageError::DecompressionBomb);
351    }
352
353    // Step 2: Checked multiplication - overflow = bomb (fail-safe)
354    let max_allowed_original = MAX_COMPRESSION_RATIO
355        .checked_mul(compressed_size)
356        .ok_or(ByteStorageError::DecompressionBomb)?;
357
358    // Step 3: Compare original_size against computed maximum
359    if (original_size as u64) > max_allowed_original {
360        return Err(ByteStorageError::DecompressionBomb);
361    }
362
363    Ok(())
364}
365
366/// Raw byte storage engine (pure Rust core)
367/// Simple store/retrieve interface with no type awareness
368pub struct ByteStorage {
369    default_format: String,
370}
371
372impl ByteStorage {
373    /// Create new ByteStorage instance
374    pub fn new(default_format: Option<String>) -> Self {
375        ByteStorage {
376            default_format: default_format.unwrap_or_else(|| "msgpack".to_string()),
377        }
378    }
379
380    /// Store arbitrary bytes with compression and checksums
381    ///
382    /// Returns serialized StorageEnvelope bytes
383    #[cfg(all(feature = "compression", feature = "checksum", feature = "messagepack"))]
384    pub fn store(&self, data: &[u8], format: Option<String>) -> Result<Vec<u8>, ByteStorageError> {
385        // Security: Check input size before processing
386        if data.len() > MAX_UNCOMPRESSED_SIZE {
387            return Err(ByteStorageError::InputTooLarge);
388        }
389
390        let format = format.unwrap_or_else(|| self.default_format.clone());
391
392        let envelope = StorageEnvelope::new(data, format)?;
393
394        // Serialize envelope with MessagePack. Security: the final envelope
395        // size check runs inside, before the output is allocated.
396        envelope.encode()
397    }
398
399    /// Retrieve and validate stored bytes
400    ///
401    /// Returns (original_data, format_identifier)
402    ///
403    /// # Errors
404    ///
405    /// `envelope_bytes` is untrusted. Before it is decoded, a structural
406    /// pre-scan bounds its nesting depth and rejects any header that declares
407    /// more than the input can back (protocol Retrieve Flow, step 2). A
408    /// pre-scan rejection is `DeserializationFailed` whose message starts with
409    /// `decode pre-scan: `, the same variant as any other bytes that do not
410    /// decode as a `StorageEnvelope`.
411    #[cfg(all(feature = "compression", feature = "checksum", feature = "messagepack"))]
412    pub fn retrieve(&self, envelope_bytes: &[u8]) -> Result<(Vec<u8>, String), ByteStorageError> {
413        // Security: Check envelope size before deserializing
414        if envelope_bytes.len() > MAX_COMPRESSED_SIZE {
415            return Err(ByteStorageError::InputTooLarge);
416        }
417
418        let envelope = decode_envelope(envelope_bytes)?;
419
420        // Extract and validate data (all security checks happen inside extract_payload())
421        let data = envelope.extract()?;
422
423        Ok((data, envelope.format))
424    }
425
426    /// Get compression ratio for given data: input length over the length of
427    /// the LZ4 block `store()` writes for it
428    #[cfg(feature = "compression")]
429    pub fn estimate_compression(&self, data: &[u8]) -> Result<f64, ByteStorageError> {
430        // Security: Check size before compression
431        if data.len() > MAX_UNCOMPRESSED_SIZE {
432            return Err(ByteStorageError::InputTooLarge);
433        }
434
435        let compressed = compress_block(data);
436
437        Ok(data.len() as f64 / compressed.len() as f64)
438    }
439
440    /// Validate an envelope by fully extracting it and discarding the result
441    ///
442    /// This runs `extract_payload`, so it decompresses the payload and
443    /// allocates up to the decompression bound. It is not a cheap structural
444    /// pre-screen for untrusted envelopes.
445    #[cfg(all(feature = "compression", feature = "checksum", feature = "messagepack"))]
446    pub fn validate(&self, envelope_bytes: &[u8]) -> bool {
447        // Security: Check size before validating
448        if envelope_bytes.len() > MAX_COMPRESSED_SIZE {
449            return false; // Invalid due to size limit
450        }
451
452        match decode_envelope(envelope_bytes) {
453            Ok(envelope) => envelope.extract().is_ok(),
454            Err(_) => false,
455        }
456    }
457
458    /// Get security limits
459    pub fn max_uncompressed_size(&self) -> usize {
460        MAX_UNCOMPRESSED_SIZE
461    }
462
463    pub fn max_compressed_size(&self) -> usize {
464        MAX_COMPRESSED_SIZE
465    }
466
467    pub fn max_compression_ratio(&self) -> u64 {
468        MAX_COMPRESSION_RATIO
469    }
470}
471
472impl Default for ByteStorage {
473    fn default() -> Self {
474        Self::new(None)
475    }
476}
477
478/// Decode untrusted envelope bytes: structural pre-scan first, then the typed
479/// decode. Serde's derive skips an unknown map key with `IgnoredAny`, which
480/// recurses, so the depth bound has to hold before `rmp_serde` sees the bytes.
481#[cfg(all(feature = "compression", feature = "checksum", feature = "messagepack"))]
482fn decode_envelope(
483    envelope_bytes: &[u8],
484) -> Result<borrowed::StorageEnvelope<'_>, ByteStorageError> {
485    // Nesting bound for the envelope decode. The protocol requires 32..=1024;
486    // 100 matches cachekit-rs and cachekit-ts. A legitimate envelope nests 2 deep.
487    const MAX_DEPTH: usize = 100;
488    crate::check_msgpack_structure(envelope_bytes, MAX_DEPTH).map_err(|what| {
489        ByteStorageError::DeserializationFailed(format!("decode pre-scan: {what}"))
490    })?;
491    rmp_serde::from_slice(envelope_bytes)
492        .map_err(|e| ByteStorageError::DeserializationFailed(e.to_string()))
493}
494
495/// What `retrieve` and `validate` decode: `StorageEnvelope` with
496/// `compressed_data` borrowed from the input instead of copied out of it,
497/// below `LARGE_PAYLOAD`.
498///
499/// A module of its own so the struct can share `StorageEnvelope`'s Rust name:
500/// serde's derive puts that name in its "expected struct ..." errors, which
501/// bindings surface (cachekit-py pins one). Same fields, order and field
502/// decoders, so it accepts exactly the inputs `StorageEnvelope` does, with
503/// the same error messages (`borrowed_decode_matches_owned_decode`). A legacy
504/// int-array payload still decodes, into an owned buffer.
505#[cfg(all(feature = "compression", feature = "checksum", feature = "messagepack"))]
506mod borrowed {
507    use std::borrow::Cow;
508
509    use serde::de::{Deserializer, Error, SeqAccess, Visitor};
510    use serde::Deserialize;
511
512    use super::{extract_payload, ByteStorageError, LARGE_PAYLOAD};
513
514    #[derive(Deserialize)]
515    pub(super) struct StorageEnvelope<'a> {
516        #[serde(borrow, deserialize_with = "borrow_payload")]
517        pub(super) compressed_data: Cow<'a, [u8]>,
518        pub(super) checksum: [u8; 8],
519        pub(super) original_size: u32,
520        pub(super) format: String,
521    }
522
523    impl StorageEnvelope<'_> {
524        /// `StorageEnvelope::extract`, on the borrowed payload.
525        pub(super) fn extract(&self) -> Result<Vec<u8>, ByteStorageError> {
526            extract_payload(&self.compressed_data, &self.checksum, self.original_size)
527        }
528    }
529
530    /// serde_bytes' `ByteBuf` decode, which `StorageEnvelope::compressed_data`
531    /// uses, except that a `bin` or `str` payload under `LARGE_PAYLOAD` is
532    /// borrowed. Owned bytes and strings reach `visit_bytes` and `visit_str`
533    /// through serde's default methods. serde_bytes' own `Cow` decode borrows
534    /// too, but words its type error differently.
535    fn borrow_payload<'de, D: Deserializer<'de>>(
536        deserializer: D,
537    ) -> Result<Cow<'de, [u8]>, D::Error> {
538        struct PayloadVisitor;
539
540        impl<'de> Visitor<'de> for PayloadVisitor {
541            type Value = Cow<'de, [u8]>;
542
543            fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
544                formatter.write_str("byte array")
545            }
546
547            fn visit_borrowed_bytes<E: Error>(self, v: &'de [u8]) -> Result<Self::Value, E> {
548                Ok(borrow_below_large(v))
549            }
550
551            fn visit_borrowed_str<E: Error>(self, v: &'de str) -> Result<Self::Value, E> {
552                Ok(borrow_below_large(v.as_bytes()))
553            }
554
555            fn visit_bytes<E: Error>(self, v: &[u8]) -> Result<Self::Value, E> {
556                Ok(Cow::Owned(v.to_vec()))
557            }
558
559            fn visit_str<E: Error>(self, v: &str) -> Result<Self::Value, E> {
560                Ok(Cow::Owned(v.as_bytes().to_vec()))
561            }
562
563            // Legacy int-array payload, pre-allocation capped as serde_bytes caps it.
564            fn visit_seq<V: SeqAccess<'de>>(self, mut seq: V) -> Result<Self::Value, V::Error> {
565                let mut bytes = Vec::with_capacity(seq.size_hint().unwrap_or(0).min(4096));
566                while let Some(b) = seq.next_element()? {
567                    bytes.push(b);
568                }
569                Ok(Cow::Owned(bytes))
570            }
571        }
572
573        deserializer.deserialize_byte_buf(PayloadVisitor)
574    }
575
576    /// Borrow a payload shorter than `LARGE_PAYLOAD`; copy a longer one, as
577    /// the owned decode does.
578    fn borrow_below_large(v: &[u8]) -> Cow<'_, [u8]> {
579        if v.len() < LARGE_PAYLOAD {
580            Cow::Borrowed(v)
581        } else {
582            Cow::Owned(v.to_vec())
583        }
584    }
585}
586
587#[cfg(all(
588    test,
589    feature = "compression",
590    feature = "checksum",
591    feature = "messagepack"
592))]
593mod tests {
594    use super::*;
595
596    #[test]
597    fn envelope_embeds_canonical_checksum() {
598        let data = b"DRY-guard payload";
599        let envelope = StorageEnvelope::new(data, "test".to_string()).unwrap();
600        assert_eq!(envelope.checksum, crate::checksum::checksum(data));
601    }
602
603    #[test]
604    fn test_storage_envelope_roundtrip() {
605        let data = b"Hello, World! This is test data for compression.".to_vec();
606        let envelope = StorageEnvelope::new(&data, "test".to_string()).unwrap();
607        let extracted = envelope.extract().unwrap();
608        assert_eq!(data, extracted);
609    }
610
611    #[test]
612    fn test_compression_works() {
613        // At the threshold, so it takes the lz4_flex path, not the literal block
614        let data = vec![b'a'; LITERAL_BLOCK_THRESHOLD]; // Highly compressible
615        let envelope = StorageEnvelope::new(&data, "test".to_string()).unwrap();
616        assert!(envelope.compressed_data.len() < data.len());
617    }
618
619    #[test]
620    fn below_threshold_writes_a_literals_only_block() {
621        for len in [0, 1, 14, 15, LITERAL_BLOCK_THRESHOLD - 1] {
622            // Compressible, so a literals-only block is the writer's choice,
623            // not what lz4_flex would emit anyway
624            let data = vec![b'a'; len];
625            let envelope = StorageEnvelope::new(&data, "test".to_string()).unwrap();
626            let block = &envelope.compressed_data;
627
628            let token = block[0];
629            assert_eq!(token & 0x0F, 0, "len {len}: token has a match length");
630            let header = if len < 15 {
631                assert_eq!(usize::from(token >> 4), len, "len {len}: literal length");
632                1
633            } else {
634                assert_eq!(token >> 4, 15, "len {len}: literal length nibble");
635                assert_eq!(usize::from(block[1]) + 15, len, "len {len}: length byte");
636                2
637            };
638            assert_eq!(&block[header..], &data[..], "len {len}: literals");
639            assert_eq!(lz4_flex::decompress(block, len).unwrap(), data, "len {len}");
640        }
641    }
642
643    #[test]
644    fn at_threshold_writes_lz4_flex_output() {
645        for data in [
646            vec![b'a'; LITERAL_BLOCK_THRESHOLD],
647            (0..LITERAL_BLOCK_THRESHOLD as u8).collect::<Vec<u8>>(),
648        ] {
649            let envelope = StorageEnvelope::new(&data, "test".to_string()).unwrap();
650            assert_eq!(envelope.compressed_data, lz4_flex::compress(&data));
651        }
652    }
653
654    #[test]
655    fn below_threshold_incompressible_input_matches_lz4_flex() {
656        // Distinct bytes leave lz4_flex no match, so its output is a literals-only
657        // block too: the literal-block writer changes no bytes for such input
658        for len in 0..LITERAL_BLOCK_THRESHOLD {
659            let data: Vec<u8> = (0..len as u8).collect();
660            assert_eq!(
661                compress_block(&data),
662                lz4_flex::compress(&data),
663                "len {len}"
664            );
665        }
666    }
667
668    #[test]
669    fn store_roundtrips_around_and_above_threshold() {
670        // Repetitive, periodic and pseudo-random inputs either side of the
671        // threshold, so the lz4_flex encoder's match, literal and end-of-input
672        // paths all run
673        let storage = ByteStorage::new(None);
674        let mut state: u32 = 0x9E37_79B9;
675        let mut next = move || {
676            state ^= state << 13;
677            state ^= state >> 17;
678            state ^= state << 5;
679            state as u8
680        };
681        for len in [
682            LITERAL_BLOCK_THRESHOLD - 1,
683            LITERAL_BLOCK_THRESHOLD,
684            LITERAL_BLOCK_THRESHOLD + 1,
685            100,
686            1000,
687            4103,
688        ] {
689            let inputs = [
690                vec![b'z'; len],
691                (0..len).map(|i| (i % 251) as u8).collect::<Vec<u8>>(),
692                (0..len).map(|_| next()).collect::<Vec<u8>>(),
693            ];
694            for data in inputs {
695                let stored = storage.store(&data, None).unwrap();
696                assert_eq!(storage.retrieve(&stored).unwrap().0, data, "len {len}");
697            }
698        }
699    }
700
701    #[test]
702    fn estimate_compression_reports_what_store_writes() {
703        let storage = ByteStorage::new(None);
704        for len in [
705            0,
706            32,
707            LITERAL_BLOCK_THRESHOLD - 1,
708            LITERAL_BLOCK_THRESHOLD,
709            1024,
710        ] {
711            let data = vec![b'a'; len];
712            let envelope = StorageEnvelope::new(&data, "test".to_string()).unwrap();
713            let expected = len as f64 / envelope.compressed_data.len() as f64;
714            assert_eq!(
715                storage.estimate_compression(&data).unwrap(),
716                expected,
717                "len {len}"
718            );
719        }
720    }
721
722    #[test]
723    fn test_checksum_validation() {
724        let mut envelope = StorageEnvelope::new(b"test", "test".to_string()).unwrap();
725        // Corrupt the checksum
726        envelope.checksum[0] = !envelope.checksum[0];
727        // Fail-open guard: the DRY refactor must still surface the specific
728        // ChecksumMismatch variant (not silently succeed on a flipped checksum).
729        assert!(matches!(
730            envelope.extract(),
731            Err(ByteStorageError::ChecksumMismatch)
732        ));
733    }
734
735    #[test]
736    fn test_raw_persistence_roundtrip() {
737        let storage = ByteStorage::new(None);
738        let test_data = b"test data for persistence";
739
740        let stored = storage.store(test_data, None).unwrap();
741        let (retrieved_data, format) = storage.retrieve(&stored).unwrap();
742        assert_eq!(test_data, retrieved_data.as_slice());
743        assert_eq!("msgpack", format);
744    }
745
746    #[test]
747    fn test_size_limits_input() {
748        let storage = ByteStorage::new(None);
749
750        // Create data larger than MAX_UNCOMPRESSED_SIZE (512MB)
751        let large_data = vec![0u8; MAX_UNCOMPRESSED_SIZE + 1];
752
753        let result = storage.store(&large_data, None);
754        assert!(matches!(result, Err(ByteStorageError::InputTooLarge)));
755    }
756
757    #[test]
758    fn test_size_limits_envelope() {
759        // Create data exactly at the limit
760        let max_data = vec![0u8; MAX_UNCOMPRESSED_SIZE];
761        let envelope_result = StorageEnvelope::new(&max_data, "test".to_string());
762
763        // Should succeed at exactly the limit
764        assert!(envelope_result.is_ok());
765    }
766
767    #[test]
768    fn test_compression_ratio_bomb_protection() {
769        // Simulate a decompression bomb scenario
770        let malicious_envelope = StorageEnvelope {
771            compressed_data: vec![0u8; 1000], // Small compressed size
772            checksum: [0u8; 8],               // Fake checksum
773            original_size: 200 * 1024 * 1024, // Claims 200MB original (200x expansion)
774            format: "test".to_string(),
775        };
776
777        let result = malicious_envelope.extract();
778        assert!(matches!(result, Err(ByteStorageError::DecompressionBomb)));
779    }
780
781    // ============================================================================
782    // Decompression Bomb Edge Case Tests (Task 6.1)
783    // ============================================================================
784
785    #[test]
786    fn test_decompression_bomb_zero_compressed_size() {
787        // WHY: Empty compressed data claiming non-zero original is always a bomb
788        let malicious_envelope = StorageEnvelope {
789            compressed_data: vec![], // Zero compressed size
790            checksum: [0u8; 8],
791            original_size: 1000, // Claims 1KB original
792            format: "test".to_string(),
793        };
794
795        let result = malicious_envelope.extract();
796        assert!(
797            matches!(result, Err(ByteStorageError::DecompressionBomb)),
798            "Zero compressed size should be rejected as decompression bomb"
799        );
800    }
801
802    #[test]
803    fn test_decompression_bomb_extreme_ratio() {
804        // WHY: Test extreme ratio that exceeds 1000:1
805        // compressed_size=1, original_size=2000 → 2000:1 ratio exceeds limit
806        // Note: u32::MAX would be caught by InputTooLarge first (> MAX_UNCOMPRESSED_SIZE)
807        let malicious_envelope = StorageEnvelope {
808            compressed_data: vec![0u8; 1], // 1 byte compressed
809            checksum: [0u8; 8],
810            original_size: 2000, // 2000:1 ratio (exceeds 1000:1 limit)
811            format: "test".to_string(),
812        };
813
814        let result = malicious_envelope.extract();
815        assert!(
816            matches!(result, Err(ByteStorageError::DecompressionBomb)),
817            "Extreme ratio should be rejected as bomb: {:?}",
818            result
819        );
820    }
821
822    #[test]
823    fn test_decompression_u32_max_original_size() {
824        // WHY: u32::MAX original_size exceeds MAX_UNCOMPRESSED_SIZE
825        // Should fail with InputTooLarge, not DecompressionBomb
826        // This validates the check order (size limits before ratio)
827        let malicious_envelope = StorageEnvelope {
828            compressed_data: vec![0u8; 1000],
829            checksum: [0u8; 8],
830            original_size: u32::MAX, // ~4GB exceeds 512MB limit
831            format: "test".to_string(),
832        };
833
834        let result = malicious_envelope.extract();
835        assert!(
836            matches!(result, Err(ByteStorageError::InputTooLarge)),
837            "u32::MAX should be rejected as InputTooLarge (exceeds 512MB limit): {:?}",
838            result
839        );
840    }
841
842    #[test]
843    fn test_decompression_exactly_at_threshold() {
844        // WHY: Exactly 1000:1 ratio should be accepted (pass ratio check)
845        // The test verifies the ratio check passes - subsequent failures are expected
846        // (invalid LZ4 data will fail at decompression or checksum)
847        let envelope = StorageEnvelope {
848            compressed_data: vec![0u8; 100], // 100 bytes compressed (not valid LZ4)
849            checksum: [0u8; 8],
850            original_size: 100_000, // 100KB = exactly 1000:1 ratio
851            format: "test".to_string(),
852        };
853
854        let result = envelope.extract();
855        // KEY: Should NOT fail with DecompressionBomb (ratio check should pass)
856        // Will fail with DecompressionFailed, ChecksumMismatch, or SizeValidationFailed
857        assert!(
858            !matches!(result, Err(ByteStorageError::DecompressionBomb)),
859            "Exactly 1000:1 ratio should pass bomb check: {:?}",
860            result
861        );
862        assert!(
863            result.is_err(),
864            "Invalid data should still fail after ratio check"
865        );
866    }
867
868    #[test]
869    fn test_decompression_just_over_threshold() {
870        // WHY: 1001:1 ratio should be rejected
871        let malicious_envelope = StorageEnvelope {
872            compressed_data: vec![0u8; 100], // 100 bytes compressed
873            checksum: [0u8; 8],
874            original_size: 100_001, // 100.001KB = 1000.01:1 ratio (just over)
875            format: "test".to_string(),
876        };
877
878        let result = malicious_envelope.extract();
879        assert!(
880            matches!(result, Err(ByteStorageError::DecompressionBomb)),
881            "Just over 1000:1 ratio should be rejected as bomb"
882        );
883    }
884
885    #[test]
886    fn test_decompression_bomb_integer_boundary() {
887        // WHY: Test near u64 overflow boundary
888        // MAX_COMPRESSION_RATIO (1000) * compressed_size must not overflow
889        // u64::MAX / 1000 ≈ 18,446,744,073,709,551 is max safe compressed_size
890        // But we're constrained by MAX_COMPRESSED_SIZE (512MB), so overflow is unlikely
891        // This test verifies the check works at realistic boundary
892
893        let envelope = StorageEnvelope {
894            compressed_data: vec![0u8; 1_000_000], // 1MB compressed
895            checksum: [0u8; 8],
896            original_size: 1_000_000_000, // 1GB = exactly 1000:1 ratio
897            format: "test".to_string(),
898        };
899
900        // Should fail due to size limit (1GB > MAX_UNCOMPRESSED_SIZE)
901        let result = envelope.extract();
902        assert!(
903            matches!(result, Err(ByteStorageError::InputTooLarge)),
904            "Should fail size check before ratio check: {:?}",
905            result
906        );
907    }
908
909    #[test]
910    fn test_extract_rejects_oversized_compressed_data() {
911        // WHY: only the compressed-length cap rejects this envelope. original_size
912        // is under its cap and the ratio is far below 1000:1, and retrieve's
913        // envelope-length check is bypassed by calling extract directly.
914        let envelope = StorageEnvelope {
915            compressed_data: vec![0u8; MAX_COMPRESSED_SIZE + 1],
916            checksum: [0u8; 8],
917            original_size: 1,
918            format: "test".to_string(),
919        };
920
921        assert_eq!(envelope.extract(), Err(ByteStorageError::InputTooLarge));
922    }
923
924    #[test]
925    fn test_envelope_size_validation() {
926        let storage = ByteStorage::new(None);
927
928        // Create oversized envelope bytes
929        let oversized_envelope = vec![0u8; MAX_COMPRESSED_SIZE + 1];
930
931        let result = storage.retrieve(&oversized_envelope);
932        assert!(matches!(result, Err(ByteStorageError::InputTooLarge)));
933    }
934
935    #[test]
936    fn test_security_limits_getters() {
937        let storage = ByteStorage::new(None);
938
939        assert_eq!(storage.max_uncompressed_size(), MAX_UNCOMPRESSED_SIZE);
940        assert_eq!(storage.max_compressed_size(), MAX_COMPRESSED_SIZE);
941        assert_eq!(storage.max_compression_ratio(), 1000u64);
942    }
943
944    #[test]
945    fn test_compression_estimate_security() {
946        let storage = ByteStorage::new(None);
947
948        // Test with oversized data
949        let large_data = vec![0u8; MAX_UNCOMPRESSED_SIZE + 1];
950        let result = storage.estimate_compression(&large_data);
951        assert!(matches!(result, Err(ByteStorageError::InputTooLarge)));
952    }
953
954    #[test]
955    fn test_validate_security() {
956        let storage = ByteStorage::new(None);
957
958        // Test with oversized envelope
959        let large_envelope = vec![0u8; MAX_COMPRESSED_SIZE + 1];
960        let result = storage.validate(&large_envelope);
961        assert!(!result); // Should be invalid due to size
962    }
963
964    #[test]
965    fn test_edge_case_exactly_at_limits() {
966        // Test data exactly at 512MB
967        let storage = ByteStorage::new(None);
968        let max_size_data = vec![1u8; MAX_UNCOMPRESSED_SIZE]; // Fill with 1s to ensure it's compressible
969
970        // Should succeed at exactly the limit
971        let result = storage.store(&max_size_data, None);
972        assert!(result.is_ok());
973    }
974
975    /// Every envelope in the wire-format fixture (accepted, rejected and
976    /// payload-rejected), plus malformed shapes the fixture does not carry.
977    fn decode_corpus() -> Vec<(String, Vec<u8>)> {
978        let fixture: serde_json::Value =
979            serde_json::from_str(include_str!("../tests/vectors/wire-format.json")).unwrap();
980        let mut corpus = Vec::new();
981        for group in ["vectors", "reject_vectors", "payload_reject_vectors"] {
982            for v in fixture[group].as_array().unwrap() {
983                let hex = v["envelope_hex"].as_str().unwrap();
984                corpus.push((
985                    v["name"].as_str().unwrap().to_string(),
986                    hex::decode(hex).unwrap(),
987                ));
988            }
989        }
990        let checksum = [0x98, 0, 1, 2, 3, 4, 5, 6, 0xCC, 0xFF];
991        for (name, payload) in [
992            ("payload_str", &[0xA3, b'a', b'b', b'c'][..]),
993            ("payload_int", &[0x07][..]),
994            ("payload_nil", &[0xC0][..]),
995            ("payload_map", &[0x80][..]),
996            ("payload_bool", &[0xC3][..]),
997            ("payload_ext", &[0xD4, 0x01, 0x00][..]),
998            ("payload_float", &[0xCA, 0, 0, 0, 0][..]),
999            ("legacy_bad_element", &[0x92, 0x01, 0xA1, b'x'][..]),
1000            ("payload_truncated_bin", &[0xC4, 0x09, 0x00][..]),
1001        ] {
1002            let mut env = vec![0x94];
1003            env.extend_from_slice(payload);
1004            env.extend_from_slice(&checksum);
1005            env.extend_from_slice(&[0x03, 0xA4, b't', b'e', b's', b't']);
1006            corpus.push((name.to_string(), env));
1007        }
1008        // Map form with the field names, map form missing one, not an array.
1009        let mut named =
1010            rmp_serde::to_vec_named(&StorageEnvelope::new(b"named", "f".into()).unwrap()).unwrap();
1011        corpus.push(("map_form".into(), named.clone()));
1012        named[0] = 0x83;
1013        corpus.push(("map_form_missing_field".into(), named));
1014        corpus.push(("not_an_array".into(), vec![0xA2, b'h', b'i']));
1015        corpus.push(("empty".into(), vec![]));
1016        corpus
1017    }
1018
1019    #[test]
1020    fn borrowed_decode_matches_owned_decode() {
1021        // The borrowed decode accepts exactly what the owned one does, into
1022        // the same fields, and fails with the same message (bindings surface
1023        // that text: cachekit-py pins several of the fixture's messages).
1024        for (name, bytes) in decode_corpus() {
1025            let owned = rmp_serde::from_slice::<StorageEnvelope>(&bytes);
1026            let borrowed = rmp_serde::from_slice::<borrowed::StorageEnvelope>(&bytes);
1027            match (owned, borrowed) {
1028                (Ok(o), Ok(b)) => {
1029                    assert_eq!(o.compressed_data, &*b.compressed_data, "[{name}] payload");
1030                    assert_eq!(o.checksum, b.checksum, "[{name}] checksum");
1031                    assert_eq!(o.original_size, b.original_size, "[{name}] original_size");
1032                    assert_eq!(o.format, b.format, "[{name}] format");
1033                }
1034                (Err(o), Err(b)) => assert_eq!(o.to_string(), b.to_string(), "[{name}] error"),
1035                (o, b) => panic!("[{name}] owned {:?} vs borrowed {:?}", o.err(), b.err()),
1036            }
1037        }
1038    }
1039
1040    #[test]
1041    fn bin_payload_is_borrowed_and_legacy_payload_is_owned() {
1042        let fixture: serde_json::Value =
1043            serde_json::from_str(include_str!("../tests/vectors/wire-format.json")).unwrap();
1044        use std::borrow::Cow;
1045        let mut seen = (0, 0);
1046        for v in fixture["vectors"].as_array().unwrap() {
1047            let bytes = hex::decode(v["envelope_hex"].as_str().unwrap()).unwrap();
1048            let envelope = decode_envelope(&bytes).unwrap();
1049            let name = v["name"].as_str().unwrap();
1050            if name.ends_with("_bin") {
1051                seen.0 += 1;
1052                assert!(
1053                    matches!(envelope.compressed_data, Cow::Borrowed(_)),
1054                    "[{name}] bin payload copied"
1055                );
1056            } else {
1057                seen.1 += 1;
1058                assert!(
1059                    matches!(envelope.compressed_data, Cow::Owned(_)),
1060                    "[{name}] legacy payload"
1061                );
1062            }
1063        }
1064        assert!(
1065            seen.0 > 0 && seen.1 > 0,
1066            "fixture has both encodings: {seen:?}"
1067        );
1068
1069        // From LARGE_PAYLOAD the payload is copied, as the owned decode does
1070        let storage = ByteStorage::new(None);
1071        for (len, borrowed) in [(LARGE_PAYLOAD - 512, true), (LARGE_PAYLOAD + 512, false)] {
1072            let data = xorshift_bytes(len);
1073            let stored = storage.store(&data, None).unwrap();
1074            let envelope = decode_envelope(&stored).unwrap();
1075            let payload = envelope.compressed_data.len();
1076            assert_eq!(
1077                payload < LARGE_PAYLOAD,
1078                borrowed,
1079                "len {len}: payload {payload}"
1080            );
1081            assert_eq!(
1082                matches!(envelope.compressed_data, Cow::Borrowed(_)),
1083                borrowed,
1084                "len {len}: payload {payload}"
1085            );
1086            assert_eq!(storage.retrieve(&stored).unwrap().0, data, "len {len}");
1087        }
1088    }
1089
1090    /// Incompressible bytes, so the envelope and its payload track `len`.
1091    fn xorshift_bytes(len: usize) -> Vec<u8> {
1092        let mut state: u32 = 0x9E37_79B9;
1093        (0..len)
1094            .map(|_| {
1095                state ^= state << 13;
1096                state ^= state >> 17;
1097                state ^= state << 5;
1098                state as u8
1099            })
1100            .collect()
1101    }
1102
1103    /// An envelope with chosen field values, for the encoding width boundaries.
1104    fn envelope_with(
1105        payload_len: usize,
1106        checksum: [u8; 8],
1107        size: u32,
1108        format_len: usize,
1109    ) -> StorageEnvelope {
1110        StorageEnvelope {
1111            compressed_data: vec![0; payload_len],
1112            checksum,
1113            original_size: size,
1114            format: "f".repeat(format_len),
1115        }
1116    }
1117
1118    #[test]
1119    fn encoded_len_matches_rmp_serde() {
1120        // Each length and value either side of a MessagePack width change:
1121        // fixstr/str8/str16/str32, bin8/bin16/bin32, fixint/u8/u16/u32
1122        let lengths = [0, 1, 31, 32, 255, 256, 65_535, 65_536, 70_000];
1123        let sizes = [0, 127, 128, 255, 256, 65_535, 65_536, u32::MAX];
1124        let checksums = [
1125            [0x00; 8],
1126            [0x7F; 8],
1127            [0x80; 8],
1128            [0xFF; 8],
1129            [0, 0x80, 1, 0xFF, 0x7F, 2, 0xC0, 3],
1130        ];
1131        for payload_len in lengths {
1132            for format_len in lengths {
1133                let envelope = envelope_with(payload_len, checksums[4], 1, format_len);
1134                let expected = rmp_serde::to_vec(&envelope).unwrap().len();
1135                assert_eq!(
1136                    envelope.encoded_len(),
1137                    expected,
1138                    "payload {payload_len}, format {format_len}"
1139                );
1140            }
1141        }
1142        for size in sizes {
1143            for checksum in checksums {
1144                let envelope = envelope_with(3, checksum, size, 7);
1145                let expected = rmp_serde::to_vec(&envelope).unwrap().len();
1146                assert_eq!(
1147                    envelope.encoded_len(),
1148                    expected,
1149                    "size {size}, checksum {checksum:?}"
1150                );
1151            }
1152        }
1153    }
1154
1155    #[test]
1156    fn store_and_encode_into_write_rmp_serde_bytes() {
1157        let storage = ByteStorage::new(None);
1158        // (input, whether its envelope is at or above LARGE_PAYLOAD)
1159        for (data, large) in [
1160            (vec![], false),
1161            (vec![b'a'; 63], false),
1162            (vec![b'a'; 64], false),
1163            ((0..70_000).map(|i| (i * 7919 % 251) as u8).collect(), false),
1164            // either side of the gate: LZ4 adds ~257 length bytes to a literal run
1165            (xorshift_bytes(LARGE_PAYLOAD - 512), false),
1166            (xorshift_bytes(LARGE_PAYLOAD), true),
1167            (xorshift_bytes(300_000), true),
1168        ] {
1169            let envelope = StorageEnvelope::new(&data, "msgpack".to_string()).unwrap();
1170            let expected = rmp_serde::to_vec(&envelope).unwrap();
1171
1172            let stored = storage.store(&data, None).unwrap();
1173            assert_eq!(stored, expected, "store, len {}", data.len());
1174            assert_eq!(
1175                stored.len() >= LARGE_PAYLOAD,
1176                large,
1177                "gate side, len {}: envelope {}",
1178                data.len(),
1179                stored.len()
1180            );
1181            if stored.len() < LARGE_PAYLOAD {
1182                // allocated once at its final length, no reallocation
1183                assert_eq!(
1184                    stored.capacity(),
1185                    stored.len(),
1186                    "store capacity, len {}",
1187                    data.len()
1188                );
1189            } else {
1190                // a large envelope keeps rmp_serde's growing buffer
1191                assert_eq!(
1192                    stored.capacity(),
1193                    expected.capacity(),
1194                    "store capacity, len {}",
1195                    data.len()
1196                );
1197            }
1198
1199            let mut out = vec![0u8; envelope.encoded_len()];
1200            envelope.encode_into(&mut out).unwrap();
1201            assert_eq!(out, expected, "encode_into, len {}", data.len());
1202        }
1203    }
1204
1205    #[test]
1206    fn encode_into_rejects_a_wrong_sized_buffer() {
1207        let envelope = StorageEnvelope::new(b"payload", "msgpack".to_string()).unwrap();
1208        let len = envelope.encoded_len();
1209        for wrong in [0, len - 1, len + 1] {
1210            let mut out = vec![0u8; wrong];
1211            assert!(
1212                matches!(
1213                    envelope.encode_into(&mut out),
1214                    Err(ByteStorageError::SerializationFailed(_))
1215                ),
1216                "buffer of {wrong} bytes for a {len}-byte envelope"
1217            );
1218        }
1219    }
1220
1221    #[test]
1222    fn encode_into_applies_the_envelope_size_cap() {
1223        // Compressed data at the cap: the envelope headers push it over, and
1224        // the check runs before the (deliberately tiny) buffer is looked at
1225        let envelope = envelope_with(MAX_COMPRESSED_SIZE, [0; 8], 1, 7);
1226        assert!(envelope.encoded_len() > MAX_COMPRESSED_SIZE);
1227        assert_eq!(
1228            envelope.encode_into(&mut []),
1229            Err(ByteStorageError::InputTooLarge)
1230        );
1231        assert_eq!(
1232            envelope.encode().err(),
1233            Some(ByteStorageError::InputTooLarge)
1234        );
1235    }
1236
1237    #[test]
1238    fn test_zero_size_edge_case() {
1239        let storage = ByteStorage::new(None);
1240        let empty_data = vec![];
1241
1242        let stored = storage.store(&empty_data, None).unwrap();
1243        let (retrieved_data, format) = storage.retrieve(&stored).unwrap();
1244        assert_eq!(empty_data, retrieved_data);
1245        assert_eq!("msgpack", format);
1246    }
1247}
1248
1249// Kani Formal Verification Proofs
1250// These proofs use bounded model checking to verify critical security properties
1251// Bounded to complete within 10 minutes as per specification requirements
1252#[cfg(kani)]
1253mod kani_proofs {
1254    use super::*;
1255
1256    /// Verify checksum integrity (corruption detection)
1257    /// Property: Any single-bit checksum corruption is always detected
1258    #[kani::proof]
1259    #[kani::unwind(10)] // Need 8+ for memcmp of 8-byte xxHash3-64 checksum
1260    fn verify_checksum_detects_corruption() {
1261        // Create symbolic checksum (xxHash3-64 produces 8 bytes)
1262        let checksum_a: [u8; 8] = kani::any();
1263        let mut checksum_b = checksum_a;
1264
1265        // Flip exactly one bit
1266        let byte_index: usize = kani::any();
1267        let bit_index: usize = kani::any();
1268        kani::assume(byte_index < 8);
1269        kani::assume(bit_index < 8);
1270
1271        checksum_b[byte_index] ^= 1 << bit_index;
1272
1273        // Property: Corrupted checksum must differ from original
1274        assert_ne!(checksum_a, checksum_b);
1275    }
1276
1277    // The size/ratio proofs pin the limits as literals instead of reading the
1278    // constants, so changing a constant or inverting a comparison in
1279    // `check_decompression_bound` fails a proof.
1280    const PINNED_MAX_COMPRESSED: u64 = 512 * 1024 * 1024;
1281    const PINNED_MAX_UNCOMPRESSED: u64 = 512 * 1024 * 1024;
1282    const PINNED_MAX_RATIO: u64 = 1000;
1283
1284    /// Verify the size caps of the decompression bound
1285    /// Property: `InputTooLarge` exactly when either size exceeds 512 MiB, for
1286    /// every (compressed length, original_size) pair
1287    #[kani::proof]
1288    fn verify_decompression_bound_size_caps() {
1289        let compressed_len: usize = kani::any();
1290        let original_size: u32 = kani::any();
1291
1292        let over_cap = compressed_len as u64 > PINNED_MAX_COMPRESSED
1293            || original_size as u64 > PINNED_MAX_UNCOMPRESSED;
1294        let result = check_decompression_bound(compressed_len, original_size);
1295
1296        assert_eq!(
1297            over_cap,
1298            matches!(result, Err(ByteStorageError::InputTooLarge))
1299        );
1300    }
1301
1302    /// Verify the ratio limit of the decompression bound
1303    /// Property: with both sizes within their caps, `DecompressionBomb` exactly
1304    /// when the compressed length is zero or the ratio exceeds 1000:1, else Ok
1305    #[kani::proof]
1306    fn verify_decompression_bound_ratio() {
1307        let compressed_len: usize = kani::any();
1308        let original_size: u32 = kani::any();
1309        kani::assume(compressed_len as u64 <= PINNED_MAX_COMPRESSED);
1310        kani::assume(original_size as u64 <= PINNED_MAX_UNCOMPRESSED);
1311
1312        let is_bomb =
1313            compressed_len == 0 || original_size as u64 > PINNED_MAX_RATIO * compressed_len as u64;
1314        let result = check_decompression_bound(compressed_len, original_size);
1315
1316        if is_bomb {
1317            assert!(matches!(result, Err(ByteStorageError::DecompressionBomb)));
1318        } else {
1319            assert!(result.is_ok());
1320        }
1321    }
1322}