1#[cfg(feature = "compression")]
9use lz4_flex;
10use serde::{Deserialize, Serialize};
11use thiserror::Error;
12
13#[derive(Debug, Error, Clone, PartialEq)]
15pub enum ByteStorageError {
16 #[error("input exceeds maximum size")]
17 InputTooLarge,
18
19 #[error("decompression ratio exceeds safety limit")]
20 DecompressionBomb,
21
22 #[error("integrity check failed")]
23 ChecksumMismatch,
24
25 #[error("compression failed")]
26 CompressionFailed,
27
28 #[error("decompression failed")]
29 DecompressionFailed,
30
31 #[error("size validation failed")]
32 SizeValidationFailed,
33
34 #[error("serialization failed: {0}")]
35 SerializationFailed(String),
36
37 #[error("deserialization failed: {0}")]
38 DeserializationFailed(String),
39}
40
41const MAX_UNCOMPRESSED_SIZE: usize = 512 * 1024 * 1024; const MAX_COMPRESSED_SIZE: usize = 512 * 1024 * 1024; const MAX_COMPRESSION_RATIO: u64 = 1000;
47
48#[cfg(all(feature = "compression", feature = "checksum", feature = "messagepack"))]
69const LARGE_PAYLOAD: usize = 64 * 1024;
70
71#[cfg(feature = "compression")]
82const LITERAL_BLOCK_THRESHOLD: usize = 64;
83
84#[cfg(feature = "compression")]
87const _: () = assert!(LITERAL_BLOCK_THRESHOLD - 1 <= 15 + 254);
88
89#[cfg(feature = "compression")]
96fn compress_block(data: &[u8]) -> Vec<u8> {
97 if data.len() >= LITERAL_BLOCK_THRESHOLD {
98 return lz4_flex::compress(data);
99 }
100 let len = data.len();
101 let mut block = Vec::with_capacity(len + 2);
102 if len < 15 {
103 block.push((len as u8) << 4);
104 } else {
105 block.push(0xF0);
106 block.push((len - 15) as u8);
107 }
108 block.extend_from_slice(data);
109 block
110}
111
112#[derive(Serialize, Deserialize)]
117pub struct StorageEnvelope {
118 #[serde(with = "serde_bytes")]
124 pub compressed_data: Vec<u8>,
125 pub checksum: [u8; 8],
127 pub original_size: u32,
129 pub format: String,
131}
132
133impl StorageEnvelope {
134 #[cfg(all(feature = "compression", feature = "checksum"))]
140 pub fn new(data: &[u8], format: String) -> Result<Self, ByteStorageError> {
141 if data.len() > MAX_UNCOMPRESSED_SIZE {
143 return Err(ByteStorageError::InputTooLarge);
144 }
145
146 let original_size = data.len() as u32;
147
148 let compressed_data = compress_block(data);
150
151 if compressed_data.len() > MAX_COMPRESSED_SIZE {
153 return Err(ByteStorageError::InputTooLarge);
154 }
155
156 let checksum = crate::checksum::checksum(data);
158
159 Ok(StorageEnvelope {
160 compressed_data,
161 checksum,
162 original_size,
163 format,
164 })
165 }
166
167 #[cfg(all(feature = "compression", feature = "checksum"))]
169 pub fn extract(&self) -> Result<Vec<u8>, ByteStorageError> {
170 extract_payload(&self.compressed_data, &self.checksum, self.original_size)
171 }
172
173 #[cfg(feature = "messagepack")]
180 pub fn encoded_len(&self) -> usize {
181 fn uint_len(v: u32) -> usize {
185 match v {
186 0..=0x7F => 1,
187 0x80..=0xFF => 2,
188 0x100..=0xFFFF => 3,
189 _ => 5,
190 }
191 }
192 fn bin_header_len(len: usize) -> usize {
193 match len {
194 0..=0xFF => 2,
195 0x100..=0xFFFF => 3,
196 _ => 5,
197 }
198 }
199 fn str_header_len(len: usize) -> usize {
200 match len {
201 0..=31 => 1,
202 32..=0xFF => 2,
203 0x100..=0xFFFF => 3,
204 _ => 5,
205 }
206 }
207 let checksum_len = 1 + self
208 .checksum
209 .iter()
210 .map(|&b| uint_len(b.into()))
211 .sum::<usize>();
212 let fixed = 1 + bin_header_len(self.compressed_data.len())
214 + checksum_len
215 + uint_len(self.original_size)
216 + str_header_len(self.format.len());
217 fixed
218 .saturating_add(self.compressed_data.len())
219 .saturating_add(self.format.len())
220 }
221
222 #[cfg(feature = "messagepack")]
239 pub fn encode_into(&self, out: &mut [u8]) -> Result<(), ByteStorageError> {
240 let len = self.capped_encoded_len()?;
241 if out.len() != len {
242 return Err(ByteStorageError::SerializationFailed(format!(
243 "output buffer is {} bytes, the envelope encodes to {len}",
244 out.len()
245 )));
246 }
247 let mut rest = &mut out[..];
248 rmp_serde::encode::write(&mut rest, self)
249 .map_err(|e| ByteStorageError::SerializationFailed(e.to_string()))?;
250 if !rest.is_empty() {
252 return Err(ByteStorageError::SerializationFailed(format!(
253 "envelope encoded {} bytes short of its computed length",
254 rest.len()
255 )));
256 }
257 Ok(())
258 }
259
260 #[cfg(feature = "messagepack")]
263 fn capped_encoded_len(&self) -> Result<usize, ByteStorageError> {
264 let len = self.encoded_len();
265 if len > MAX_COMPRESSED_SIZE {
266 return Err(ByteStorageError::InputTooLarge);
267 }
268 Ok(len)
269 }
270
271 #[cfg(all(feature = "compression", feature = "checksum", feature = "messagepack"))]
278 fn encode(&self) -> Result<Vec<u8>, ByteStorageError> {
279 let len = self.capped_encoded_len()?;
280 if len >= LARGE_PAYLOAD {
281 return rmp_serde::to_vec(self)
282 .map_err(|e| ByteStorageError::SerializationFailed(e.to_string()));
283 }
284 let mut out = Vec::with_capacity(len);
285 rmp_serde::encode::write(&mut out, self)
286 .map_err(|e| ByteStorageError::SerializationFailed(e.to_string()))?;
287 debug_assert_eq!(out.len(), len, "encoded_len must be exact");
288 Ok(out)
289 }
290}
291
292#[cfg(all(feature = "compression", feature = "checksum"))]
296fn extract_payload(
297 compressed_data: &[u8],
298 checksum: &[u8; 8],
299 original_size: u32,
300) -> Result<Vec<u8>, ByteStorageError> {
301 #[cfg(test)]
303 crate::read_allocation_probe::reserve_first_control(compressed_data, original_size);
304
305 check_decompression_bound(compressed_data.len(), original_size)?;
306
307 let decompressed = lz4_flex::decompress(compressed_data, original_size as usize)
309 .map_err(|_| ByteStorageError::DecompressionFailed)?;
310
311 if !crate::checksum::verify_checksum(&decompressed, checksum) {
314 return Err(ByteStorageError::ChecksumMismatch);
315 }
316
317 if decompressed.len() != original_size as usize {
319 return Err(ByteStorageError::SizeValidationFailed);
320 }
321
322 Ok(decompressed)
323}
324
325#[cfg(all(feature = "compression", feature = "checksum"))]
331fn check_decompression_bound(
332 compressed_len: usize,
333 original_size: u32,
334) -> Result<(), ByteStorageError> {
335 if compressed_len > MAX_COMPRESSED_SIZE {
337 return Err(ByteStorageError::InputTooLarge);
338 }
339
340 if original_size as usize > MAX_UNCOMPRESSED_SIZE {
341 return Err(ByteStorageError::InputTooLarge);
342 }
343
344 let compressed_size = compressed_len as u64;
347
348 if compressed_size == 0 {
350 return Err(ByteStorageError::DecompressionBomb);
351 }
352
353 let max_allowed_original = MAX_COMPRESSION_RATIO
355 .checked_mul(compressed_size)
356 .ok_or(ByteStorageError::DecompressionBomb)?;
357
358 if (original_size as u64) > max_allowed_original {
360 return Err(ByteStorageError::DecompressionBomb);
361 }
362
363 Ok(())
364}
365
366pub struct ByteStorage {
369 default_format: String,
370}
371
372impl ByteStorage {
373 pub fn new(default_format: Option<String>) -> Self {
375 ByteStorage {
376 default_format: default_format.unwrap_or_else(|| "msgpack".to_string()),
377 }
378 }
379
380 #[cfg(all(feature = "compression", feature = "checksum", feature = "messagepack"))]
384 pub fn store(&self, data: &[u8], format: Option<String>) -> Result<Vec<u8>, ByteStorageError> {
385 if data.len() > MAX_UNCOMPRESSED_SIZE {
387 return Err(ByteStorageError::InputTooLarge);
388 }
389
390 let format = format.unwrap_or_else(|| self.default_format.clone());
391
392 let envelope = StorageEnvelope::new(data, format)?;
393
394 envelope.encode()
397 }
398
399 #[cfg(all(feature = "compression", feature = "checksum", feature = "messagepack"))]
412 pub fn retrieve(&self, envelope_bytes: &[u8]) -> Result<(Vec<u8>, String), ByteStorageError> {
413 if envelope_bytes.len() > MAX_COMPRESSED_SIZE {
415 return Err(ByteStorageError::InputTooLarge);
416 }
417
418 let envelope = decode_envelope(envelope_bytes)?;
419
420 let data = envelope.extract()?;
422
423 Ok((data, envelope.format))
424 }
425
426 #[cfg(feature = "compression")]
429 pub fn estimate_compression(&self, data: &[u8]) -> Result<f64, ByteStorageError> {
430 if data.len() > MAX_UNCOMPRESSED_SIZE {
432 return Err(ByteStorageError::InputTooLarge);
433 }
434
435 let compressed = compress_block(data);
436
437 Ok(data.len() as f64 / compressed.len() as f64)
438 }
439
440 #[cfg(all(feature = "compression", feature = "checksum", feature = "messagepack"))]
446 pub fn validate(&self, envelope_bytes: &[u8]) -> bool {
447 if envelope_bytes.len() > MAX_COMPRESSED_SIZE {
449 return false; }
451
452 match decode_envelope(envelope_bytes) {
453 Ok(envelope) => envelope.extract().is_ok(),
454 Err(_) => false,
455 }
456 }
457
458 pub fn max_uncompressed_size(&self) -> usize {
460 MAX_UNCOMPRESSED_SIZE
461 }
462
463 pub fn max_compressed_size(&self) -> usize {
464 MAX_COMPRESSED_SIZE
465 }
466
467 pub fn max_compression_ratio(&self) -> u64 {
468 MAX_COMPRESSION_RATIO
469 }
470}
471
472impl Default for ByteStorage {
473 fn default() -> Self {
474 Self::new(None)
475 }
476}
477
478#[cfg(all(feature = "compression", feature = "checksum", feature = "messagepack"))]
482fn decode_envelope(
483 envelope_bytes: &[u8],
484) -> Result<borrowed::StorageEnvelope<'_>, ByteStorageError> {
485 const MAX_DEPTH: usize = 100;
488 crate::check_msgpack_structure(envelope_bytes, MAX_DEPTH).map_err(|what| {
489 ByteStorageError::DeserializationFailed(format!("decode pre-scan: {what}"))
490 })?;
491 rmp_serde::from_slice(envelope_bytes)
492 .map_err(|e| ByteStorageError::DeserializationFailed(e.to_string()))
493}
494
495#[cfg(all(feature = "compression", feature = "checksum", feature = "messagepack"))]
506mod borrowed {
507 use std::borrow::Cow;
508
509 use serde::de::{Deserializer, Error, SeqAccess, Visitor};
510 use serde::Deserialize;
511
512 use super::{extract_payload, ByteStorageError, LARGE_PAYLOAD};
513
514 #[derive(Deserialize)]
515 pub(super) struct StorageEnvelope<'a> {
516 #[serde(borrow, deserialize_with = "borrow_payload")]
517 pub(super) compressed_data: Cow<'a, [u8]>,
518 pub(super) checksum: [u8; 8],
519 pub(super) original_size: u32,
520 pub(super) format: String,
521 }
522
523 impl StorageEnvelope<'_> {
524 pub(super) fn extract(&self) -> Result<Vec<u8>, ByteStorageError> {
526 extract_payload(&self.compressed_data, &self.checksum, self.original_size)
527 }
528 }
529
530 fn borrow_payload<'de, D: Deserializer<'de>>(
536 deserializer: D,
537 ) -> Result<Cow<'de, [u8]>, D::Error> {
538 struct PayloadVisitor;
539
540 impl<'de> Visitor<'de> for PayloadVisitor {
541 type Value = Cow<'de, [u8]>;
542
543 fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
544 formatter.write_str("byte array")
545 }
546
547 fn visit_borrowed_bytes<E: Error>(self, v: &'de [u8]) -> Result<Self::Value, E> {
548 Ok(borrow_below_large(v))
549 }
550
551 fn visit_borrowed_str<E: Error>(self, v: &'de str) -> Result<Self::Value, E> {
552 Ok(borrow_below_large(v.as_bytes()))
553 }
554
555 fn visit_bytes<E: Error>(self, v: &[u8]) -> Result<Self::Value, E> {
556 Ok(Cow::Owned(v.to_vec()))
557 }
558
559 fn visit_str<E: Error>(self, v: &str) -> Result<Self::Value, E> {
560 Ok(Cow::Owned(v.as_bytes().to_vec()))
561 }
562
563 fn visit_seq<V: SeqAccess<'de>>(self, mut seq: V) -> Result<Self::Value, V::Error> {
565 let mut bytes = Vec::with_capacity(seq.size_hint().unwrap_or(0).min(4096));
566 while let Some(b) = seq.next_element()? {
567 bytes.push(b);
568 }
569 Ok(Cow::Owned(bytes))
570 }
571 }
572
573 deserializer.deserialize_byte_buf(PayloadVisitor)
574 }
575
576 fn borrow_below_large(v: &[u8]) -> Cow<'_, [u8]> {
579 if v.len() < LARGE_PAYLOAD {
580 Cow::Borrowed(v)
581 } else {
582 Cow::Owned(v.to_vec())
583 }
584 }
585}
586
587#[cfg(all(
588 test,
589 feature = "compression",
590 feature = "checksum",
591 feature = "messagepack"
592))]
593mod tests {
594 use super::*;
595
596 #[test]
597 fn envelope_embeds_canonical_checksum() {
598 let data = b"DRY-guard payload";
599 let envelope = StorageEnvelope::new(data, "test".to_string()).unwrap();
600 assert_eq!(envelope.checksum, crate::checksum::checksum(data));
601 }
602
603 #[test]
604 fn test_storage_envelope_roundtrip() {
605 let data = b"Hello, World! This is test data for compression.".to_vec();
606 let envelope = StorageEnvelope::new(&data, "test".to_string()).unwrap();
607 let extracted = envelope.extract().unwrap();
608 assert_eq!(data, extracted);
609 }
610
611 #[test]
612 fn test_compression_works() {
613 let data = vec![b'a'; LITERAL_BLOCK_THRESHOLD]; let envelope = StorageEnvelope::new(&data, "test".to_string()).unwrap();
616 assert!(envelope.compressed_data.len() < data.len());
617 }
618
619 #[test]
620 fn below_threshold_writes_a_literals_only_block() {
621 for len in [0, 1, 14, 15, LITERAL_BLOCK_THRESHOLD - 1] {
622 let data = vec![b'a'; len];
625 let envelope = StorageEnvelope::new(&data, "test".to_string()).unwrap();
626 let block = &envelope.compressed_data;
627
628 let token = block[0];
629 assert_eq!(token & 0x0F, 0, "len {len}: token has a match length");
630 let header = if len < 15 {
631 assert_eq!(usize::from(token >> 4), len, "len {len}: literal length");
632 1
633 } else {
634 assert_eq!(token >> 4, 15, "len {len}: literal length nibble");
635 assert_eq!(usize::from(block[1]) + 15, len, "len {len}: length byte");
636 2
637 };
638 assert_eq!(&block[header..], &data[..], "len {len}: literals");
639 assert_eq!(lz4_flex::decompress(block, len).unwrap(), data, "len {len}");
640 }
641 }
642
643 #[test]
644 fn at_threshold_writes_lz4_flex_output() {
645 for data in [
646 vec![b'a'; LITERAL_BLOCK_THRESHOLD],
647 (0..LITERAL_BLOCK_THRESHOLD as u8).collect::<Vec<u8>>(),
648 ] {
649 let envelope = StorageEnvelope::new(&data, "test".to_string()).unwrap();
650 assert_eq!(envelope.compressed_data, lz4_flex::compress(&data));
651 }
652 }
653
654 #[test]
655 fn below_threshold_incompressible_input_matches_lz4_flex() {
656 for len in 0..LITERAL_BLOCK_THRESHOLD {
659 let data: Vec<u8> = (0..len as u8).collect();
660 assert_eq!(
661 compress_block(&data),
662 lz4_flex::compress(&data),
663 "len {len}"
664 );
665 }
666 }
667
668 #[test]
669 fn store_roundtrips_around_and_above_threshold() {
670 let storage = ByteStorage::new(None);
674 let mut state: u32 = 0x9E37_79B9;
675 let mut next = move || {
676 state ^= state << 13;
677 state ^= state >> 17;
678 state ^= state << 5;
679 state as u8
680 };
681 for len in [
682 LITERAL_BLOCK_THRESHOLD - 1,
683 LITERAL_BLOCK_THRESHOLD,
684 LITERAL_BLOCK_THRESHOLD + 1,
685 100,
686 1000,
687 4103,
688 ] {
689 let inputs = [
690 vec![b'z'; len],
691 (0..len).map(|i| (i % 251) as u8).collect::<Vec<u8>>(),
692 (0..len).map(|_| next()).collect::<Vec<u8>>(),
693 ];
694 for data in inputs {
695 let stored = storage.store(&data, None).unwrap();
696 assert_eq!(storage.retrieve(&stored).unwrap().0, data, "len {len}");
697 }
698 }
699 }
700
701 #[test]
702 fn estimate_compression_reports_what_store_writes() {
703 let storage = ByteStorage::new(None);
704 for len in [
705 0,
706 32,
707 LITERAL_BLOCK_THRESHOLD - 1,
708 LITERAL_BLOCK_THRESHOLD,
709 1024,
710 ] {
711 let data = vec![b'a'; len];
712 let envelope = StorageEnvelope::new(&data, "test".to_string()).unwrap();
713 let expected = len as f64 / envelope.compressed_data.len() as f64;
714 assert_eq!(
715 storage.estimate_compression(&data).unwrap(),
716 expected,
717 "len {len}"
718 );
719 }
720 }
721
722 #[test]
723 fn test_checksum_validation() {
724 let mut envelope = StorageEnvelope::new(b"test", "test".to_string()).unwrap();
725 envelope.checksum[0] = !envelope.checksum[0];
727 assert!(matches!(
730 envelope.extract(),
731 Err(ByteStorageError::ChecksumMismatch)
732 ));
733 }
734
735 #[test]
736 fn test_raw_persistence_roundtrip() {
737 let storage = ByteStorage::new(None);
738 let test_data = b"test data for persistence";
739
740 let stored = storage.store(test_data, None).unwrap();
741 let (retrieved_data, format) = storage.retrieve(&stored).unwrap();
742 assert_eq!(test_data, retrieved_data.as_slice());
743 assert_eq!("msgpack", format);
744 }
745
746 #[test]
747 fn test_size_limits_input() {
748 let storage = ByteStorage::new(None);
749
750 let large_data = vec![0u8; MAX_UNCOMPRESSED_SIZE + 1];
752
753 let result = storage.store(&large_data, None);
754 assert!(matches!(result, Err(ByteStorageError::InputTooLarge)));
755 }
756
757 #[test]
758 fn test_size_limits_envelope() {
759 let max_data = vec![0u8; MAX_UNCOMPRESSED_SIZE];
761 let envelope_result = StorageEnvelope::new(&max_data, "test".to_string());
762
763 assert!(envelope_result.is_ok());
765 }
766
767 #[test]
768 fn test_compression_ratio_bomb_protection() {
769 let malicious_envelope = StorageEnvelope {
771 compressed_data: vec![0u8; 1000], checksum: [0u8; 8], original_size: 200 * 1024 * 1024, format: "test".to_string(),
775 };
776
777 let result = malicious_envelope.extract();
778 assert!(matches!(result, Err(ByteStorageError::DecompressionBomb)));
779 }
780
781 #[test]
786 fn test_decompression_bomb_zero_compressed_size() {
787 let malicious_envelope = StorageEnvelope {
789 compressed_data: vec![], checksum: [0u8; 8],
791 original_size: 1000, format: "test".to_string(),
793 };
794
795 let result = malicious_envelope.extract();
796 assert!(
797 matches!(result, Err(ByteStorageError::DecompressionBomb)),
798 "Zero compressed size should be rejected as decompression bomb"
799 );
800 }
801
802 #[test]
803 fn test_decompression_bomb_extreme_ratio() {
804 let malicious_envelope = StorageEnvelope {
808 compressed_data: vec![0u8; 1], checksum: [0u8; 8],
810 original_size: 2000, format: "test".to_string(),
812 };
813
814 let result = malicious_envelope.extract();
815 assert!(
816 matches!(result, Err(ByteStorageError::DecompressionBomb)),
817 "Extreme ratio should be rejected as bomb: {:?}",
818 result
819 );
820 }
821
822 #[test]
823 fn test_decompression_u32_max_original_size() {
824 let malicious_envelope = StorageEnvelope {
828 compressed_data: vec![0u8; 1000],
829 checksum: [0u8; 8],
830 original_size: u32::MAX, format: "test".to_string(),
832 };
833
834 let result = malicious_envelope.extract();
835 assert!(
836 matches!(result, Err(ByteStorageError::InputTooLarge)),
837 "u32::MAX should be rejected as InputTooLarge (exceeds 512MB limit): {:?}",
838 result
839 );
840 }
841
842 #[test]
843 fn test_decompression_exactly_at_threshold() {
844 let envelope = StorageEnvelope {
848 compressed_data: vec![0u8; 100], checksum: [0u8; 8],
850 original_size: 100_000, format: "test".to_string(),
852 };
853
854 let result = envelope.extract();
855 assert!(
858 !matches!(result, Err(ByteStorageError::DecompressionBomb)),
859 "Exactly 1000:1 ratio should pass bomb check: {:?}",
860 result
861 );
862 assert!(
863 result.is_err(),
864 "Invalid data should still fail after ratio check"
865 );
866 }
867
868 #[test]
869 fn test_decompression_just_over_threshold() {
870 let malicious_envelope = StorageEnvelope {
872 compressed_data: vec![0u8; 100], checksum: [0u8; 8],
874 original_size: 100_001, format: "test".to_string(),
876 };
877
878 let result = malicious_envelope.extract();
879 assert!(
880 matches!(result, Err(ByteStorageError::DecompressionBomb)),
881 "Just over 1000:1 ratio should be rejected as bomb"
882 );
883 }
884
885 #[test]
886 fn test_decompression_bomb_integer_boundary() {
887 let envelope = StorageEnvelope {
894 compressed_data: vec![0u8; 1_000_000], checksum: [0u8; 8],
896 original_size: 1_000_000_000, format: "test".to_string(),
898 };
899
900 let result = envelope.extract();
902 assert!(
903 matches!(result, Err(ByteStorageError::InputTooLarge)),
904 "Should fail size check before ratio check: {:?}",
905 result
906 );
907 }
908
909 #[test]
910 fn test_extract_rejects_oversized_compressed_data() {
911 let envelope = StorageEnvelope {
915 compressed_data: vec![0u8; MAX_COMPRESSED_SIZE + 1],
916 checksum: [0u8; 8],
917 original_size: 1,
918 format: "test".to_string(),
919 };
920
921 assert_eq!(envelope.extract(), Err(ByteStorageError::InputTooLarge));
922 }
923
924 #[test]
925 fn test_envelope_size_validation() {
926 let storage = ByteStorage::new(None);
927
928 let oversized_envelope = vec![0u8; MAX_COMPRESSED_SIZE + 1];
930
931 let result = storage.retrieve(&oversized_envelope);
932 assert!(matches!(result, Err(ByteStorageError::InputTooLarge)));
933 }
934
935 #[test]
936 fn test_security_limits_getters() {
937 let storage = ByteStorage::new(None);
938
939 assert_eq!(storage.max_uncompressed_size(), MAX_UNCOMPRESSED_SIZE);
940 assert_eq!(storage.max_compressed_size(), MAX_COMPRESSED_SIZE);
941 assert_eq!(storage.max_compression_ratio(), 1000u64);
942 }
943
944 #[test]
945 fn test_compression_estimate_security() {
946 let storage = ByteStorage::new(None);
947
948 let large_data = vec![0u8; MAX_UNCOMPRESSED_SIZE + 1];
950 let result = storage.estimate_compression(&large_data);
951 assert!(matches!(result, Err(ByteStorageError::InputTooLarge)));
952 }
953
954 #[test]
955 fn test_validate_security() {
956 let storage = ByteStorage::new(None);
957
958 let large_envelope = vec![0u8; MAX_COMPRESSED_SIZE + 1];
960 let result = storage.validate(&large_envelope);
961 assert!(!result); }
963
964 #[test]
965 fn test_edge_case_exactly_at_limits() {
966 let storage = ByteStorage::new(None);
968 let max_size_data = vec![1u8; MAX_UNCOMPRESSED_SIZE]; let result = storage.store(&max_size_data, None);
972 assert!(result.is_ok());
973 }
974
975 fn decode_corpus() -> Vec<(String, Vec<u8>)> {
978 let fixture: serde_json::Value =
979 serde_json::from_str(include_str!("../tests/vectors/wire-format.json")).unwrap();
980 let mut corpus = Vec::new();
981 for group in ["vectors", "reject_vectors", "payload_reject_vectors"] {
982 for v in fixture[group].as_array().unwrap() {
983 let hex = v["envelope_hex"].as_str().unwrap();
984 corpus.push((
985 v["name"].as_str().unwrap().to_string(),
986 hex::decode(hex).unwrap(),
987 ));
988 }
989 }
990 let checksum = [0x98, 0, 1, 2, 3, 4, 5, 6, 0xCC, 0xFF];
991 for (name, payload) in [
992 ("payload_str", &[0xA3, b'a', b'b', b'c'][..]),
993 ("payload_int", &[0x07][..]),
994 ("payload_nil", &[0xC0][..]),
995 ("payload_map", &[0x80][..]),
996 ("payload_bool", &[0xC3][..]),
997 ("payload_ext", &[0xD4, 0x01, 0x00][..]),
998 ("payload_float", &[0xCA, 0, 0, 0, 0][..]),
999 ("legacy_bad_element", &[0x92, 0x01, 0xA1, b'x'][..]),
1000 ("payload_truncated_bin", &[0xC4, 0x09, 0x00][..]),
1001 ] {
1002 let mut env = vec![0x94];
1003 env.extend_from_slice(payload);
1004 env.extend_from_slice(&checksum);
1005 env.extend_from_slice(&[0x03, 0xA4, b't', b'e', b's', b't']);
1006 corpus.push((name.to_string(), env));
1007 }
1008 let mut named =
1010 rmp_serde::to_vec_named(&StorageEnvelope::new(b"named", "f".into()).unwrap()).unwrap();
1011 corpus.push(("map_form".into(), named.clone()));
1012 named[0] = 0x83;
1013 corpus.push(("map_form_missing_field".into(), named));
1014 corpus.push(("not_an_array".into(), vec![0xA2, b'h', b'i']));
1015 corpus.push(("empty".into(), vec![]));
1016 corpus
1017 }
1018
1019 #[test]
1020 fn borrowed_decode_matches_owned_decode() {
1021 for (name, bytes) in decode_corpus() {
1025 let owned = rmp_serde::from_slice::<StorageEnvelope>(&bytes);
1026 let borrowed = rmp_serde::from_slice::<borrowed::StorageEnvelope>(&bytes);
1027 match (owned, borrowed) {
1028 (Ok(o), Ok(b)) => {
1029 assert_eq!(o.compressed_data, &*b.compressed_data, "[{name}] payload");
1030 assert_eq!(o.checksum, b.checksum, "[{name}] checksum");
1031 assert_eq!(o.original_size, b.original_size, "[{name}] original_size");
1032 assert_eq!(o.format, b.format, "[{name}] format");
1033 }
1034 (Err(o), Err(b)) => assert_eq!(o.to_string(), b.to_string(), "[{name}] error"),
1035 (o, b) => panic!("[{name}] owned {:?} vs borrowed {:?}", o.err(), b.err()),
1036 }
1037 }
1038 }
1039
1040 #[test]
1041 fn bin_payload_is_borrowed_and_legacy_payload_is_owned() {
1042 let fixture: serde_json::Value =
1043 serde_json::from_str(include_str!("../tests/vectors/wire-format.json")).unwrap();
1044 use std::borrow::Cow;
1045 let mut seen = (0, 0);
1046 for v in fixture["vectors"].as_array().unwrap() {
1047 let bytes = hex::decode(v["envelope_hex"].as_str().unwrap()).unwrap();
1048 let envelope = decode_envelope(&bytes).unwrap();
1049 let name = v["name"].as_str().unwrap();
1050 if name.ends_with("_bin") {
1051 seen.0 += 1;
1052 assert!(
1053 matches!(envelope.compressed_data, Cow::Borrowed(_)),
1054 "[{name}] bin payload copied"
1055 );
1056 } else {
1057 seen.1 += 1;
1058 assert!(
1059 matches!(envelope.compressed_data, Cow::Owned(_)),
1060 "[{name}] legacy payload"
1061 );
1062 }
1063 }
1064 assert!(
1065 seen.0 > 0 && seen.1 > 0,
1066 "fixture has both encodings: {seen:?}"
1067 );
1068
1069 let storage = ByteStorage::new(None);
1071 for (len, borrowed) in [(LARGE_PAYLOAD - 512, true), (LARGE_PAYLOAD + 512, false)] {
1072 let data = xorshift_bytes(len);
1073 let stored = storage.store(&data, None).unwrap();
1074 let envelope = decode_envelope(&stored).unwrap();
1075 let payload = envelope.compressed_data.len();
1076 assert_eq!(
1077 payload < LARGE_PAYLOAD,
1078 borrowed,
1079 "len {len}: payload {payload}"
1080 );
1081 assert_eq!(
1082 matches!(envelope.compressed_data, Cow::Borrowed(_)),
1083 borrowed,
1084 "len {len}: payload {payload}"
1085 );
1086 assert_eq!(storage.retrieve(&stored).unwrap().0, data, "len {len}");
1087 }
1088 }
1089
1090 fn xorshift_bytes(len: usize) -> Vec<u8> {
1092 let mut state: u32 = 0x9E37_79B9;
1093 (0..len)
1094 .map(|_| {
1095 state ^= state << 13;
1096 state ^= state >> 17;
1097 state ^= state << 5;
1098 state as u8
1099 })
1100 .collect()
1101 }
1102
1103 fn envelope_with(
1105 payload_len: usize,
1106 checksum: [u8; 8],
1107 size: u32,
1108 format_len: usize,
1109 ) -> StorageEnvelope {
1110 StorageEnvelope {
1111 compressed_data: vec![0; payload_len],
1112 checksum,
1113 original_size: size,
1114 format: "f".repeat(format_len),
1115 }
1116 }
1117
1118 #[test]
1119 fn encoded_len_matches_rmp_serde() {
1120 let lengths = [0, 1, 31, 32, 255, 256, 65_535, 65_536, 70_000];
1123 let sizes = [0, 127, 128, 255, 256, 65_535, 65_536, u32::MAX];
1124 let checksums = [
1125 [0x00; 8],
1126 [0x7F; 8],
1127 [0x80; 8],
1128 [0xFF; 8],
1129 [0, 0x80, 1, 0xFF, 0x7F, 2, 0xC0, 3],
1130 ];
1131 for payload_len in lengths {
1132 for format_len in lengths {
1133 let envelope = envelope_with(payload_len, checksums[4], 1, format_len);
1134 let expected = rmp_serde::to_vec(&envelope).unwrap().len();
1135 assert_eq!(
1136 envelope.encoded_len(),
1137 expected,
1138 "payload {payload_len}, format {format_len}"
1139 );
1140 }
1141 }
1142 for size in sizes {
1143 for checksum in checksums {
1144 let envelope = envelope_with(3, checksum, size, 7);
1145 let expected = rmp_serde::to_vec(&envelope).unwrap().len();
1146 assert_eq!(
1147 envelope.encoded_len(),
1148 expected,
1149 "size {size}, checksum {checksum:?}"
1150 );
1151 }
1152 }
1153 }
1154
1155 #[test]
1156 fn store_and_encode_into_write_rmp_serde_bytes() {
1157 let storage = ByteStorage::new(None);
1158 for (data, large) in [
1160 (vec![], false),
1161 (vec![b'a'; 63], false),
1162 (vec![b'a'; 64], false),
1163 ((0..70_000).map(|i| (i * 7919 % 251) as u8).collect(), false),
1164 (xorshift_bytes(LARGE_PAYLOAD - 512), false),
1166 (xorshift_bytes(LARGE_PAYLOAD), true),
1167 (xorshift_bytes(300_000), true),
1168 ] {
1169 let envelope = StorageEnvelope::new(&data, "msgpack".to_string()).unwrap();
1170 let expected = rmp_serde::to_vec(&envelope).unwrap();
1171
1172 let stored = storage.store(&data, None).unwrap();
1173 assert_eq!(stored, expected, "store, len {}", data.len());
1174 assert_eq!(
1175 stored.len() >= LARGE_PAYLOAD,
1176 large,
1177 "gate side, len {}: envelope {}",
1178 data.len(),
1179 stored.len()
1180 );
1181 if stored.len() < LARGE_PAYLOAD {
1182 assert_eq!(
1184 stored.capacity(),
1185 stored.len(),
1186 "store capacity, len {}",
1187 data.len()
1188 );
1189 } else {
1190 assert_eq!(
1192 stored.capacity(),
1193 expected.capacity(),
1194 "store capacity, len {}",
1195 data.len()
1196 );
1197 }
1198
1199 let mut out = vec![0u8; envelope.encoded_len()];
1200 envelope.encode_into(&mut out).unwrap();
1201 assert_eq!(out, expected, "encode_into, len {}", data.len());
1202 }
1203 }
1204
1205 #[test]
1206 fn encode_into_rejects_a_wrong_sized_buffer() {
1207 let envelope = StorageEnvelope::new(b"payload", "msgpack".to_string()).unwrap();
1208 let len = envelope.encoded_len();
1209 for wrong in [0, len - 1, len + 1] {
1210 let mut out = vec![0u8; wrong];
1211 assert!(
1212 matches!(
1213 envelope.encode_into(&mut out),
1214 Err(ByteStorageError::SerializationFailed(_))
1215 ),
1216 "buffer of {wrong} bytes for a {len}-byte envelope"
1217 );
1218 }
1219 }
1220
1221 #[test]
1222 fn encode_into_applies_the_envelope_size_cap() {
1223 let envelope = envelope_with(MAX_COMPRESSED_SIZE, [0; 8], 1, 7);
1226 assert!(envelope.encoded_len() > MAX_COMPRESSED_SIZE);
1227 assert_eq!(
1228 envelope.encode_into(&mut []),
1229 Err(ByteStorageError::InputTooLarge)
1230 );
1231 assert_eq!(
1232 envelope.encode().err(),
1233 Some(ByteStorageError::InputTooLarge)
1234 );
1235 }
1236
1237 #[test]
1238 fn test_zero_size_edge_case() {
1239 let storage = ByteStorage::new(None);
1240 let empty_data = vec![];
1241
1242 let stored = storage.store(&empty_data, None).unwrap();
1243 let (retrieved_data, format) = storage.retrieve(&stored).unwrap();
1244 assert_eq!(empty_data, retrieved_data);
1245 assert_eq!("msgpack", format);
1246 }
1247}
1248
1249#[cfg(kani)]
1253mod kani_proofs {
1254 use super::*;
1255
1256 #[kani::proof]
1259 #[kani::unwind(10)] fn verify_checksum_detects_corruption() {
1261 let checksum_a: [u8; 8] = kani::any();
1263 let mut checksum_b = checksum_a;
1264
1265 let byte_index: usize = kani::any();
1267 let bit_index: usize = kani::any();
1268 kani::assume(byte_index < 8);
1269 kani::assume(bit_index < 8);
1270
1271 checksum_b[byte_index] ^= 1 << bit_index;
1272
1273 assert_ne!(checksum_a, checksum_b);
1275 }
1276
1277 const PINNED_MAX_COMPRESSED: u64 = 512 * 1024 * 1024;
1281 const PINNED_MAX_UNCOMPRESSED: u64 = 512 * 1024 * 1024;
1282 const PINNED_MAX_RATIO: u64 = 1000;
1283
1284 #[kani::proof]
1288 fn verify_decompression_bound_size_caps() {
1289 let compressed_len: usize = kani::any();
1290 let original_size: u32 = kani::any();
1291
1292 let over_cap = compressed_len as u64 > PINNED_MAX_COMPRESSED
1293 || original_size as u64 > PINNED_MAX_UNCOMPRESSED;
1294 let result = check_decompression_bound(compressed_len, original_size);
1295
1296 assert_eq!(
1297 over_cap,
1298 matches!(result, Err(ByteStorageError::InputTooLarge))
1299 );
1300 }
1301
1302 #[kani::proof]
1306 fn verify_decompression_bound_ratio() {
1307 let compressed_len: usize = kani::any();
1308 let original_size: u32 = kani::any();
1309 kani::assume(compressed_len as u64 <= PINNED_MAX_COMPRESSED);
1310 kani::assume(original_size as u64 <= PINNED_MAX_UNCOMPRESSED);
1311
1312 let is_bomb =
1313 compressed_len == 0 || original_size as u64 > PINNED_MAX_RATIO * compressed_len as u64;
1314 let result = check_decompression_bound(compressed_len, original_size);
1315
1316 if is_bomb {
1317 assert!(matches!(result, Err(ByteStorageError::DecompressionBomb)));
1318 } else {
1319 assert!(result.is_ok());
1320 }
1321 }
1322}