1use std::io::Write;
37
38use crate::error::{CompressError, DecompressError};
39use crate::{CHUNK_SIZE, STREAM_BUF_SIZE};
40
41#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
43#[serde(rename_all = "lowercase")]
44pub enum CompressionFormat {
45 #[serde(rename = "identity")]
47 None,
48 Zrip,
50}
51
52impl CompressionFormat {
53 pub fn as_str(self) -> &'static str {
56 match self {
57 CompressionFormat::None => "identity",
58 CompressionFormat::Zrip => "zrip",
59 }
60 }
61
62 pub fn parse_header(s: &str) -> Option<Self> {
64 match s.trim().to_ascii_lowercase().as_str() {
65 "" | "identity" | "none" => Some(CompressionFormat::None),
66 "zrip" | "zstd" => Some(CompressionFormat::Zrip),
67 _ => None,
68 }
69 }
70}
71
72pub const MAX_FRAME_OUTPUT: usize = CHUNK_SIZE as usize;
77
78pub const MAX_PENDING_FRAME: usize = CHUNK_SIZE as usize + STREAM_BUF_SIZE;
83
84pub const MAX_OUTPUT_PER_CALL: usize = MAX_FRAME_OUTPUT.saturating_mul(8);
94
95pub const ZRIP_DEFAULT_LEVEL: i32 = 1;
97
98pub const ZRIP_MIN_LEVEL: i32 = -8;
100
101pub const ZRIP_MAX_LEVEL: i32 = 4;
103
104pub fn is_valid_zrip_level(level: i32) -> bool {
106 (ZRIP_MIN_LEVEL..=ZRIP_MAX_LEVEL).contains(&level)
107}
108
109pub fn negotiate_level(req: Option<i32>, min: i32, max: i32, default: i32) -> i32 {
115 match req {
116 None => default,
117 Some(l) => l.clamp(min, max),
118 }
119}
120
121pub trait Compressor: Send {
124 fn format(&self) -> CompressionFormat;
126
127 fn compress(&mut self, input: &[u8], out: &mut Vec<u8>) -> Result<(), CompressError>;
133
134 fn finish(&mut self, out: &mut Vec<u8>) -> Result<(), CompressError>;
138}
139
140pub trait Decompressor: Send {
143 fn format(&self) -> CompressionFormat;
145
146 fn decompress(&mut self, input: &[u8], out: &mut Vec<u8>) -> Result<(), DecompressError>;
151
152 fn finish(&mut self, out: &mut Vec<u8>) -> Result<(), DecompressError>;
157}
158
159pub fn compressor(format: CompressionFormat) -> Result<Box<dyn Compressor>, CompressError> {
161 compressor_with_level(format, ZRIP_DEFAULT_LEVEL)
162}
163
164pub fn compressor_with_level(
172 format: CompressionFormat,
173 level: i32,
174) -> Result<Box<dyn Compressor>, CompressError> {
175 match format {
176 CompressionFormat::None => Ok(Box::new(PassthroughCompressor)),
177 CompressionFormat::Zrip => Ok(Box::new(ZripCompressor::new(level)?)),
178 }
179}
180
181pub fn decompressor(format: CompressionFormat) -> Box<dyn Decompressor> {
183 decompressor_with_limit(format, MAX_OUTPUT_PER_CALL)
184}
185
186pub fn decompressor_with_limit(
193 format: CompressionFormat,
194 max_output_per_call: usize,
195) -> Box<dyn Decompressor> {
196 match format {
197 CompressionFormat::None => Box::new(PassthroughDecompressor),
198 CompressionFormat::Zrip => Box::new(ZripDecompressor::with_max_output(max_output_per_call)),
199 }
200}
201
202struct PassthroughCompressor;
207
208impl Compressor for PassthroughCompressor {
209 fn format(&self) -> CompressionFormat {
210 CompressionFormat::None
211 }
212
213 fn compress(&mut self, input: &[u8], out: &mut Vec<u8>) -> Result<(), CompressError> {
214 out.extend_from_slice(input);
215 Ok(())
216 }
217
218 fn finish(&mut self, _out: &mut Vec<u8>) -> Result<(), CompressError> {
219 Ok(())
220 }
221}
222
223struct PassthroughDecompressor;
224
225impl Decompressor for PassthroughDecompressor {
226 fn format(&self) -> CompressionFormat {
227 CompressionFormat::None
228 }
229
230 fn decompress(&mut self, input: &[u8], out: &mut Vec<u8>) -> Result<(), DecompressError> {
231 out.extend_from_slice(input);
232 Ok(())
233 }
234
235 fn finish(&mut self, _out: &mut Vec<u8>) -> Result<(), DecompressError> {
236 Ok(())
237 }
238}
239
240pub struct ZripCompressor {
246 encoder: Option<zrip::FrameEncoder<Vec<u8>>>,
247 dirty: bool,
249}
250
251impl ZripCompressor {
252 pub fn new(level: i32) -> Result<Self, CompressError> {
254 Ok(ZripCompressor {
255 encoder: Some(zrip::FrameEncoder::new(Vec::new(), level)?),
256 dirty: false,
257 })
258 }
259}
260
261impl Compressor for ZripCompressor {
262 fn format(&self) -> CompressionFormat {
263 CompressionFormat::Zrip
264 }
265
266 fn compress(&mut self, input: &[u8], out: &mut Vec<u8>) -> Result<(), CompressError> {
267 if input.is_empty() {
268 return Ok(());
269 }
270 let encoder = self
271 .encoder
272 .as_mut()
273 .ok_or_else(|| std::io::Error::other("compressor already finished"))?;
274 if self.dirty {
276 let finished = encoder.reset(Vec::new())?;
277 out.extend_from_slice(&finished);
278 }
279 encoder.write_all(input)?;
280 self.dirty = true;
281 Ok(())
282 }
283
284 fn finish(&mut self, out: &mut Vec<u8>) -> Result<(), CompressError> {
285 let encoder = self
286 .encoder
287 .take()
288 .ok_or_else(|| std::io::Error::other("compressor already finished"))?;
289 let tail = encoder.finish()?;
290 out.extend_from_slice(&tail);
291 self.dirty = false;
292 Ok(())
293 }
294}
295
296pub struct ZripDecompressor {
298 pending: Vec<u8>,
300 finished: bool,
301 max_output_per_call: usize,
304}
305
306impl ZripDecompressor {
307 pub fn new() -> Self {
310 ZripDecompressor::with_max_output(MAX_OUTPUT_PER_CALL)
311 }
312
313 pub fn with_max_output(max_output_per_call: usize) -> Self {
315 ZripDecompressor {
316 pending: Vec::with_capacity(STREAM_BUF_SIZE),
317 finished: false,
318 max_output_per_call,
319 }
320 }
321
322 fn drain_frames(&mut self, out: &mut Vec<u8>, max_add: usize) -> Result<(), DecompressError> {
325 let mut added = 0usize;
326 loop {
327 let boundary = frame_boundary(&self.pending);
328 match boundary {
329 FrameBoundary::Complete { len, content_size } => {
330 if content_size.is_some_and(|cs| cs > MAX_FRAME_OUTPUT as u64) {
331 return Err(DecompressError::TooLarge {
332 limit: MAX_FRAME_OUTPUT,
333 });
334 }
335 let decoded = {
336 let frame = &self.pending[..len];
337 zrip::decompress_with_limit(frame, MAX_FRAME_OUTPUT).map_err(|e| {
338 DecompressError::Io(std::io::Error::new(
339 std::io::ErrorKind::InvalidData,
340 e,
341 ))
342 })?
343 };
344 if added.saturating_add(decoded.len()) > max_add {
347 return Err(DecompressError::TooLarge { limit: max_add });
348 }
349 out.extend_from_slice(&decoded);
350 added = added.saturating_add(decoded.len());
351 self.pending.drain(..len);
352 }
353 FrameBoundary::Incomplete => return Ok(()),
354 FrameBoundary::Invalid => {
355 return Err(DecompressError::Io(std::io::Error::new(
356 std::io::ErrorKind::InvalidData,
357 "invalid zstd frame data",
358 )))
359 }
360 }
361 }
362 }
363}
364
365impl Default for ZripDecompressor {
366 fn default() -> Self {
367 ZripDecompressor::new()
368 }
369}
370
371impl Decompressor for ZripDecompressor {
372 fn format(&self) -> CompressionFormat {
373 CompressionFormat::Zrip
374 }
375
376 fn decompress(&mut self, input: &[u8], out: &mut Vec<u8>) -> Result<(), DecompressError> {
377 if self.finished {
378 return Err(DecompressError::Io(std::io::Error::other(
379 "decompressor already finished",
380 )));
381 }
382 if !input.is_empty() {
383 self.pending.extend_from_slice(input);
384 }
385 self.drain_frames(out, self.max_output_per_call)?;
386 if self.pending.len() > MAX_PENDING_FRAME {
387 return Err(DecompressError::TooLarge {
388 limit: MAX_PENDING_FRAME,
389 });
390 }
391 Ok(())
392 }
393
394 fn finish(&mut self, out: &mut Vec<u8>) -> Result<(), DecompressError> {
395 if self.finished {
396 return Ok(());
397 }
398 self.finished = true;
399 self.drain_frames(out, self.max_output_per_call)?;
400 if !self.pending.is_empty() {
401 return Err(DecompressError::Truncated(std::io::Error::new(
402 std::io::ErrorKind::UnexpectedEof,
403 format!("{} trailing compressed bytes", self.pending.len()),
404 )));
405 }
406 Ok(())
407 }
408}
409
410const ZSTD_MAGIC: u32 = 0xFD2F_B528;
415const SKIPPABLE_MASK: u32 = 0xFFFF_FFF0;
416const SKIPPABLE_MAGIC: u32 = 0x184D_2A50;
417
418enum FrameBoundary {
419 Complete { len: usize, content_size: Option<u64> },
422 Incomplete,
424 Invalid,
426}
427
428fn frame_boundary(buf: &[u8]) -> FrameBoundary {
430 if buf.len() < 4 {
431 return FrameBoundary::Incomplete;
432 }
433 let magic = u32::from_le_bytes([buf[0], buf[1], buf[2], buf[3]]);
434 if magic == ZSTD_MAGIC {
435 zstd_frame_boundary(buf)
436 } else if (magic & SKIPPABLE_MASK) == SKIPPABLE_MAGIC {
437 skippable_frame_boundary(buf)
438 } else {
439 FrameBoundary::Invalid
440 }
441}
442
443fn skippable_frame_boundary(buf: &[u8]) -> FrameBoundary {
445 if buf.len() < 8 {
446 return FrameBoundary::Incomplete;
447 }
448 let skip = u32::from_le_bytes([buf[4], buf[5], buf[6], buf[7]]) as usize;
449 let total = 8usize.saturating_add(skip);
450 if buf.len() >= total {
451 FrameBoundary::Complete {
452 len: total,
453 content_size: Some(0),
454 }
455 } else {
456 FrameBoundary::Incomplete
457 }
458}
459
460fn zstd_frame_boundary(buf: &[u8]) -> FrameBoundary {
465 if buf.len() < 5 {
466 return FrameBoundary::Incomplete;
467 }
468 let descriptor = buf[4];
469 if descriptor & 0x18 != 0 {
471 return FrameBoundary::Invalid;
472 }
473 let single_segment = descriptor & 0x20 != 0;
474 let checksum = descriptor & 0x04 != 0;
475 let dict_id_flag = descriptor & 0x03;
476 let fcs_flag = (descriptor >> 6) & 0x03;
477
478 let mut hdr_len = 5usize;
479 if !single_segment {
480 hdr_len += 1; }
482 let dict_id_size = match dict_id_flag {
483 0 => 0,
484 1 => 1,
485 2 => 2,
486 3 => 4,
487 _ => return FrameBoundary::Invalid,
488 };
489 hdr_len += dict_id_size;
490 let fcs_size: usize = match fcs_flag {
491 0 if single_segment => 1,
492 0 => 0,
493 1 => 2,
494 2 => 4,
495 3 => 8,
496 _ => return FrameBoundary::Invalid,
497 };
498 hdr_len += fcs_size;
499
500 if buf.len() < hdr_len {
501 return FrameBoundary::Incomplete;
502 }
503 let content_size = if fcs_size > 0 {
504 let mut v = 0u64;
505 for (i, &b) in buf[5..5 + fcs_size].iter().enumerate() {
506 v |= (b as u64) << (8 * i);
507 }
508 Some(v)
509 } else {
510 None
511 };
512
513 let mut off = hdr_len;
515 loop {
516 if buf.len() < off + 3 {
517 return FrameBoundary::Incomplete;
518 }
519 let block_header = u32::from_le_bytes([buf[off], buf[off + 1], buf[off + 2], 0]);
522 let last = block_header & 0x01 != 0;
523 let block_type = (block_header >> 1) & 0x03;
524 let block_size = (block_header >> 3) as usize;
525 if block_type == 3 {
526 return FrameBoundary::Invalid;
528 }
529 off += 3 + block_size;
530 if off > MAX_PENDING_FRAME {
531 return FrameBoundary::Invalid;
532 }
533 if last {
534 break;
535 }
536 }
537 if checksum {
538 off += 4;
539 }
540 if buf.len() < off {
541 return FrameBoundary::Incomplete;
542 }
543 FrameBoundary::Complete {
544 len: off,
545 content_size,
546 }
547}
548
549#[cfg(test)]
550mod tests {
551 use super::*;
552
553 fn roundtrip_chunks(data: &[u8], feed: &[usize]) {
554 let mut c = ZripCompressor::new(ZRIP_DEFAULT_LEVEL).unwrap();
555 let mut compressed = Vec::new();
556 let mut off = 0;
557 for &n in feed {
558 let end = (off + n).min(data.len());
559 if end > off {
560 c.compress(&data[off..end], &mut compressed).unwrap();
561 }
562 off = end;
563 }
564 c.finish(&mut compressed).unwrap();
565 assert!(!compressed.is_empty());
566
567 let mut d = ZripDecompressor::new();
569 let mut plain = Vec::new();
570 let mut step = 1;
571 let mut i = 0;
572 while i < compressed.len() {
573 let end = (i + step).min(compressed.len());
574 d.decompress(&compressed[i..end], &mut plain).unwrap();
575 i = end;
576 step = step % 5 + 1; }
578 d.finish(&mut plain).unwrap();
579 assert_eq!(plain, data, "roundtrip mismatch with feed {feed:?}");
580 }
581
582 #[test]
583 fn roundtrip_single_chunk() {
584 let data: Vec<u8> = (0..100_000u32).map(|i| (i % 251) as u8).collect();
585 roundtrip_chunks(&data, &[data.len()]);
586 }
587
588 #[test]
589 fn roundtrip_multi_chunk_64k_windows() {
590 let data: Vec<u8> = (0..300_000u32).map(|i| (i / 7) as u8).collect();
592 let feed: Vec<usize> = std::iter::repeat(STREAM_BUF_SIZE).take(5).collect();
593 roundtrip_chunks(&data, &feed);
594 }
595
596 #[test]
597 fn roundtrip_highly_compressible() {
598 let data = b"libfw streaming compression test. ".repeat(10_000);
599 let mut feed = Vec::new();
601 let mut consumed = 0;
602 for (_, &size) in [1024usize, 2048, 4096, 8192, 16384].iter().cycle().enumerate() {
603 feed.push(size);
604 consumed += size;
605 if consumed >= data.len() {
606 break;
607 }
608 }
609 roundtrip_chunks(&data, &feed);
610 }
611
612 #[test]
613 fn roundtrip_empty_stream() {
614 let mut c = ZripCompressor::new(ZRIP_DEFAULT_LEVEL).unwrap();
615 let mut compressed = Vec::new();
616 c.finish(&mut compressed).unwrap();
617
618 let mut d = ZripDecompressor::new();
619 let mut plain = Vec::new();
620 d.decompress(&compressed, &mut plain).unwrap();
621 d.finish(&mut plain).unwrap();
622 assert!(plain.is_empty());
623 }
624
625 #[test]
626 fn malformed_zstd_headers_are_rejected_without_panicking() {
627 let bad = [0xFD, 0x2F, 0xB5, 0x28, 0xFF];
631 assert!(matches!(frame_boundary(&bad), FrameBoundary::Invalid));
632
633 let bad2 = [0xFD, 0x2F, 0xB5, 0x28, 0x00, 0x00, 0x00, 0x00, 0x00];
634 assert!(matches!(frame_boundary(&bad2), FrameBoundary::Invalid));
635 }
636
637 #[test]
638 fn empty_input_chunks_are_noops() {
639 let mut c = ZripCompressor::new(ZRIP_DEFAULT_LEVEL).unwrap();
640 let mut out = Vec::new();
641 c.compress(&[], &mut out).unwrap();
642 c.compress(b"hello", &mut out).unwrap();
643 c.finish(&mut out).unwrap();
644
645 let mut d = ZripDecompressor::new();
646 let mut plain = Vec::new();
647 d.decompress(&out, &mut plain).unwrap();
648 d.finish(&mut plain).unwrap();
649 assert_eq!(plain, b"hello");
650 }
651
652 #[test]
653 fn format_header_roundtrip() {
654 assert_eq!(CompressionFormat::parse_header("zrip"), Some(CompressionFormat::Zrip));
655 assert_eq!(CompressionFormat::parse_header("ZSTD"), Some(CompressionFormat::Zrip));
656 assert_eq!(CompressionFormat::parse_header("identity"), Some(CompressionFormat::None));
657 assert_eq!(CompressionFormat::parse_header(""), Some(CompressionFormat::None));
658 assert_eq!(CompressionFormat::parse_header("br"), None);
659 assert_eq!(CompressionFormat::Zrip.as_str(), "zrip");
660 assert_eq!(CompressionFormat::None.as_str(), "identity");
661 }
662
663 #[test]
664 fn passthrough_roundtrip() {
665 let mut c = compressor(CompressionFormat::None).unwrap();
666 let mut d = decompressor(CompressionFormat::None);
667 let mut compressed = Vec::new();
668 let mut plain = Vec::new();
669 c.compress(b"abc", &mut compressed).unwrap();
670 c.finish(&mut compressed).unwrap();
671 d.decompress(&compressed, &mut plain).unwrap();
672 d.finish(&mut plain).unwrap();
673 assert_eq!(plain, b"abc");
674 }
675
676 #[test]
677 fn truncated_stream_is_detected() {
678 let mut c = ZripCompressor::new(ZRIP_DEFAULT_LEVEL).unwrap();
679 let mut compressed = Vec::new();
680 c.compress(&vec![7u8; 5000], &mut compressed).unwrap();
681 c.finish(&mut compressed).unwrap();
682 compressed.truncate(compressed.len() - 1); let mut d = ZripDecompressor::new();
685 let mut plain = Vec::new();
686 d.decompress(&compressed, &mut plain).unwrap();
687 assert!(matches!(d.finish(&mut plain), Err(DecompressError::Truncated(_))));
688 }
689
690 #[test]
691 fn corrupt_stream_is_detected() {
692 let mut d = ZripDecompressor::new();
693 let mut plain = Vec::new();
694 let err = d.decompress(b"this is not a zstd frame at all", &mut plain);
695 assert!(err.is_err());
696 }
697
698 #[test]
699 fn per_call_output_budget_rejects_multi_frame_bomb() {
700 let mut compressed = Vec::new();
704 for _ in 0..64 {
705 let mut c = ZripCompressor::new(ZRIP_DEFAULT_LEVEL).unwrap();
706 c.compress(&vec![7u8; STREAM_BUF_SIZE], &mut compressed)
707 .unwrap();
708 c.finish(&mut compressed).unwrap();
709 }
710 let mut d = ZripDecompressor::with_max_output(MAX_FRAME_OUTPUT);
711 let mut plain = Vec::new();
712 let err = d.decompress(&compressed, &mut plain);
713 assert!(
714 matches!(err, Err(DecompressError::TooLarge { .. })),
715 "expected TooLarge, got {err:?}"
716 );
717 assert!(plain.len() <= MAX_FRAME_OUTPUT);
719 }
720
721 #[test]
722 fn generous_default_budget_allows_coalesced_frames() {
723 let mut compressed = Vec::new();
726 for _ in 0..8 {
727 let mut c = ZripCompressor::new(ZRIP_DEFAULT_LEVEL).unwrap();
728 c.compress(&vec![7u8; STREAM_BUF_SIZE], &mut compressed)
729 .unwrap();
730 c.finish(&mut compressed).unwrap();
731 }
732 let mut d = ZripDecompressor::new();
733 let mut plain = Vec::new();
734 d.decompress(&compressed, &mut plain).unwrap();
735 d.finish(&mut plain).unwrap();
736 assert_eq!(plain.len(), 8 * STREAM_BUF_SIZE);
737 }
738
739 #[test]
740 fn zstd_compat_interop() {
741 let data: Vec<u8> = (0..50_000u32).map(|i| (i % 31) as u8).collect();
744 let mut c = ZripCompressor::new(ZRIP_DEFAULT_LEVEL).unwrap();
745 let mut compressed = Vec::new();
746 c.compress(&data, &mut compressed).unwrap();
747 c.finish(&mut compressed).unwrap();
748 let decoded = zstd::stream::decode_all(&compressed[..]).unwrap();
749 assert_eq!(decoded, data);
750
751 let zstd_enc = zstd::stream::encode_all(&data[..], 1).unwrap();
752 let mut d = ZripDecompressor::new();
753 let mut plain = Vec::new();
754 d.decompress(&zstd_enc, &mut plain).unwrap();
755 d.finish(&mut plain).unwrap();
756 assert_eq!(plain, data);
757 }
758}