Skip to main content

pbrs_grpc/compression/
mod.rs

1//! Wire message compression: the `grpc-encoding` registry (RX-06).
2//!
3//! gRPC compresses each message independently and names the coding once per
4//! RPC in `grpc-encoding`. Supported codings are `gzip` (RFC 1952) and
5//! `deflate` (zlib, RFC 1950), subject to the RPC's compression settings.
6//! Enabling the `zstd` feature adds `zstd` (RFC 8878).
7//! `identity` is the absence of a coding, not a [`CompressionAlgorithm`].
8//!
9//! Backend: `miniz_oxide` through `flate2` (`rust_backend` +
10//! `runtime_detection`), measured faster than `zlib-rs` at the kernel
11//! default level on the dev host. See
12//! `docs/decisions/compression.md` for the shootout numbers and the
13//! optional zstd decision.
14//!
15//! [`CompressionAlgorithm::Gzip`] is the default outbound coding. Configs can
16//! select another coding. Compressed inbound frames require a supported
17//! `grpc-encoding` header; omitted or identity encodings require flag zero.
18
19mod deflate;
20mod gzip;
21#[cfg(feature = "zstd")]
22mod zstd;
23
24#[cfg(feature = "tonic")]
25pub(crate) use gzip::GzipMembers;
26pub use gzip::{decode, decode_limited, encode, encode_level};
27
28use crate::limits::MessageLimits;
29use crate::status::Status;
30
31/// Decoded bytes examined before the bounded reader enters its growth loop.
32/// Cold stream messages within this lookahead keep the ordinary codec path.
33pub(crate) const DECODE_LOOKAHEAD_BYTES: usize = 8192;
34
35/// Read at most the decoded message budget, probing one more byte for overflow.
36///
37/// Decode a bounded chunk before allocating. A short final chunk reserves only
38/// the bytes it produced; full chunks retain geometric growth for large output.
39/// The buffered compressed length is an initial hint capped at 256 KiB. Neither
40/// gzip trailers nor other untrusted decoded-size fields drive reservations.
41#[cfg(test)]
42fn read_bounded(
43    decoder: impl std::io::Read,
44    budget: usize,
45    compressed_length: usize,
46    coding: &str,
47) -> Result<Vec<u8>, Status> {
48    read_bounded_into(decoder, budget, compressed_length, coding, Vec::new())
49}
50
51/// Reuse an exclusively owned output allocation. Existing spare capacity does
52/// not increase the decoded-byte allowance or drive a new reservation.
53fn read_bounded_into(
54    mut decoder: impl std::io::Read,
55    budget: usize,
56    compressed_length: usize,
57    coding: &str,
58    mut out: Vec<u8>,
59) -> Result<Vec<u8>, Status> {
60    out.clear();
61    fn read_retry(
62        decoder: &mut impl std::io::Read,
63        target: &mut [u8],
64        coding: &str,
65    ) -> Result<usize, Status> {
66        loop {
67            match decoder.read(target) {
68                Ok(n) if n <= target.len() => return Ok(n),
69                Ok(_) => return Err(Status::internal("compression decode: invalid read length")),
70                Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue,
71                Err(error) => return Err(Status::internal(format!("{coding} decode: {error}"))),
72            }
73        }
74    }
75    // A short Read is not EOF. Fill the bounded lookahead so adversarial short
76    // reads cannot cause one reallocation per byte, or bypass trailer checks.
77    fn read_chunk(
78        decoder: &mut impl std::io::Read,
79        target: &mut [u8],
80        coding: &str,
81    ) -> Result<(usize, bool), Status> {
82        let mut used = 0;
83        while used < target.len() {
84            let tail = target
85                .get_mut(used..)
86                .ok_or_else(|| Status::internal("compression decode: short buffer"))?;
87            let n = read_retry(decoder, tail, coding)?;
88            if n == 0 {
89                return Ok((used, true));
90            }
91            used += n;
92        }
93        Ok((used, false))
94    }
95    fn reserve(out: &mut Vec<u8>, capacity: usize, budget: usize) -> Result<(), Status> {
96        if capacity <= out.capacity() {
97            return Ok(());
98        }
99        out.try_reserve_exact(capacity - out.len())
100            .map_err(|_| Status::resource_exhausted("decompressed message allocation failed"))?;
101        if out.capacity() > budget {
102            return Err(Status::resource_exhausted(
103                "decompressed message allocation exceeds limit",
104            ));
105        }
106        Ok(())
107    }
108    let exceeded =
109        || Status::resource_exhausted(format!("decompressed message exceeds limit {budget}"));
110    let mut chunk = [0u8; DECODE_LOOKAHEAD_BYTES];
111    let want = budget.saturating_add(1).min(chunk.len());
112    let target = chunk
113        .get_mut(..want)
114        .ok_or_else(|| Status::internal("compression decode: short buffer"))?;
115    let (n, eof) = read_chunk(&mut decoder, target, coding)?;
116    if n > budget {
117        return Err(exceeded());
118    }
119    if n == 0 {
120        return Ok(out);
121    }
122    let capacity = if eof {
123        n
124    } else {
125        n.max(compressed_length.min(256 * 1024)).min(budget)
126    };
127    reserve(&mut out, capacity, budget)?;
128    out.extend_from_slice(
129        chunk
130            .get(..n)
131            .ok_or_else(|| Status::internal("compression decode: short buffer"))?,
132    );
133    if eof {
134        return Ok(out);
135    }
136    // Keep used length separately and initialize each unread tail once. Reads
137    // between growths go directly into output, without per-chunk heap copies.
138    out.resize(capacity, 0);
139    let mut used = n;
140    loop {
141        if used == out.len() {
142            // Observe the final tail before growing. For output just beyond a
143            // capacity boundary (for example a 64 KiB protobuf plus headers),
144            // EOF gives an exact reservation instead of doubling the buffer.
145            // At the budget this is exactly the required one-byte probe.
146            let want = (budget - used).saturating_add(1).min(chunk.len());
147            let target = chunk
148                .get_mut(..want)
149                .ok_or_else(|| Status::internal("compression decode: short buffer"))?;
150            let (n, eof) = read_chunk(&mut decoder, target, coding)?;
151            if n > budget - used {
152                return Err(exceeded());
153            }
154            if n == 0 {
155                return Ok(out);
156            }
157            let decoded = used + n;
158            let capacity = if eof {
159                decoded
160            } else {
161                used.saturating_mul(2).max(decoded).min(budget)
162            };
163            reserve(&mut out, capacity, budget)?;
164            out.extend_from_slice(
165                chunk
166                    .get(..n)
167                    .ok_or_else(|| Status::internal("compression decode: short buffer"))?,
168            );
169            if eof {
170                return Ok(out);
171            }
172            out.resize(capacity, 0);
173            used = decoded;
174            // A clipped budget may now be full. Loop back for the overflow /
175            // EOF probe; an empty read target would falsely prove completion.
176            continue;
177        }
178        let target = out
179            .get_mut(used..)
180            .ok_or_else(|| Status::internal("compression decode: short output"))?;
181        let n = read_retry(&mut decoder, target, coding)?;
182        if n == 0 {
183            out.truncate(used);
184            return Ok(out);
185        }
186        used += n;
187    }
188}
189
190/// The published gzip/deflate compression selector.
191///
192/// This enum stays exhaustive so existing callers can match both variants.
193/// Use [`CompressionAlgorithm`] for the extensible wire-coding registry,
194/// including optional Zstd support.
195#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)]
196pub enum Codec {
197    /// `gzip`: RFC 1952 member with header, CRC, and length trailer.
198    #[default]
199    Gzip,
200    /// `deflate`: zlib wrapper (RFC 1950) around a DEFLATE stream.
201    Deflate,
202}
203
204impl Codec {
205    /// Parse a gzip or deflate token, ignoring case, whitespace and parameters.
206    /// `identity` and optional codings are not published `Codec` variants.
207    #[must_use]
208    pub fn parse(token: &str) -> Option<Self> {
209        CompressionAlgorithm::parse(token).and_then(CompressionAlgorithm::legacy_codec)
210    }
211
212    /// The wire token: `gzip` or `deflate`.
213    #[must_use]
214    pub fn name(self) -> &'static str {
215        CompressionAlgorithm::from(self).name()
216    }
217
218    /// Encode `payload` at the kernel default level for this coding.
219    pub fn encode(self, payload: &[u8]) -> Result<Vec<u8>, Status> {
220        CompressionAlgorithm::from(self).encode(payload)
221    }
222
223    /// Encode `payload` at DEFLATE effort `level` (clamped to 9).
224    pub fn encode_level(self, payload: &[u8], level: u32) -> Result<Vec<u8>, Status> {
225        CompressionAlgorithm::from(self).encode_level(payload, level)
226    }
227
228    /// Inflate `payload` with no cap. Only use with a trusted peer;
229    /// prefer [`Self::decode_limited`].
230    pub fn decode(self, payload: &[u8]) -> Result<Vec<u8>, Status> {
231        CompressionAlgorithm::from(self).decode(payload)
232    }
233
234    /// Inflate `payload`, refusing to allocate past the inbound cap.
235    pub fn decode_limited(self, payload: &[u8], limits: MessageLimits) -> Result<Vec<u8>, Status> {
236        CompressionAlgorithm::from(self).decode_limited(payload, limits)
237    }
238}
239
240/// A `grpc-encoding` wire coding this kernel can inflate and emit.
241///
242/// Use this extensible registry for optional codings such as Zstd. The
243/// published [`Codec`] remains exhaustive over gzip and deflate.
244#[non_exhaustive]
245#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)]
246pub enum CompressionAlgorithm {
247    /// `gzip`: RFC 1952 member with header, CRC, and length trailer.
248    #[default]
249    Gzip,
250    /// `deflate`: zlib wrapper (RFC 1950) around a DEFLATE stream.
251    Deflate,
252    /// `zstd`: Zstandard frame (RFC 8878). Requires the `zstd` feature.
253    #[cfg(feature = "zstd")]
254    Zstd,
255}
256
257impl From<Codec> for CompressionAlgorithm {
258    fn from(codec: Codec) -> Self {
259        match codec {
260            Codec::Gzip => Self::Gzip,
261            Codec::Deflate => Self::Deflate,
262        }
263    }
264}
265
266impl CompressionAlgorithm {
267    /// The corresponding published codec, when this is gzip or deflate.
268    #[must_use]
269    pub fn legacy_codec(self) -> Option<Codec> {
270        match self {
271            Self::Gzip => Some(Codec::Gzip),
272            Self::Deflate => Some(Codec::Deflate),
273            #[cfg(feature = "zstd")]
274            Self::Zstd => None,
275        }
276    }
277
278    /// Parse a `grpc-encoding` token. Case-insensitive; surrounding
279    /// whitespace and a trailing `;parameter` are ignored, mirroring
280    /// the wire-layer encoding-token parser. `identity` is not a coding.
281    #[must_use]
282    pub fn parse(token: &str) -> Option<Self> {
283        let coding = match token.split_once(';') {
284            Some((coding, _)) => coding.trim(),
285            None => token.trim(),
286        };
287        if coding.eq_ignore_ascii_case("gzip") {
288            Some(Self::Gzip)
289        } else if coding.eq_ignore_ascii_case("deflate") {
290            Some(Self::Deflate)
291        } else if cfg!(feature = "zstd") && coding.eq_ignore_ascii_case("zstd") {
292            #[cfg(feature = "zstd")]
293            {
294                Some(Self::Zstd)
295            }
296            #[cfg(not(feature = "zstd"))]
297            {
298                None
299            }
300        } else {
301            None
302        }
303    }
304
305    /// The wire token: `gzip`, `deflate`, or optional `zstd`.
306    #[must_use]
307    pub fn name(self) -> &'static str {
308        match self {
309            Self::Gzip => "gzip",
310            Self::Deflate => "deflate",
311            #[cfg(feature = "zstd")]
312            Self::Zstd => "zstd",
313        }
314    }
315
316    /// Encode `payload` at the kernel default level for this coding.
317    pub fn encode(self, payload: &[u8]) -> Result<Vec<u8>, Status> {
318        match self {
319            Self::Gzip => gzip::encode(payload),
320            Self::Deflate => deflate::encode(payload),
321            #[cfg(feature = "zstd")]
322            Self::Zstd => zstd::encode(payload),
323        }
324    }
325
326    /// Encode `payload` at `level` (0 stores, 1 is fast, 9 is best;
327    /// clamped to 9). The 0-9 scale is the same DEFLATE effort for gzip and
328    /// deflate; only the framing differs. Zstd currently uses the backend's
329    /// fastest level for every requested level.
330    pub fn encode_level(self, payload: &[u8], level: u32) -> Result<Vec<u8>, Status> {
331        match self {
332            Self::Gzip => gzip::encode_level(payload, level),
333            Self::Deflate => deflate::encode_level(payload, level),
334            #[cfg(feature = "zstd")]
335            Self::Zstd => zstd::encode_level(payload, level),
336        }
337    }
338
339    /// Encode `payload` at `level`, appending to `out`.
340    ///
341    /// Byte-identical to [`Self::encode_level`]; the framing layer uses
342    /// this to compress straight into the frame buffer.
343    pub(crate) fn encode_into(
344        self,
345        payload: &[u8],
346        level: u32,
347        out: &mut impl std::io::Write,
348    ) -> Result<(), Status> {
349        match self {
350            Self::Gzip => gzip::encode_into(payload, level, out),
351            Self::Deflate => deflate::encode_into(payload, level, out),
352            #[cfg(feature = "zstd")]
353            Self::Zstd => zstd::encode_into(payload, level, out),
354        }
355    }
356
357    /// Inflate `payload` with no cap.
358    ///
359    /// Prefer [`Self::decode_limited`]. This is only safe against a
360    /// trusted peer.
361    pub fn decode(self, payload: &[u8]) -> Result<Vec<u8>, Status> {
362        match self {
363            Self::Gzip => gzip::decode(payload),
364            Self::Deflate => deflate::decode(payload),
365            #[cfg(feature = "zstd")]
366            Self::Zstd => zstd::decode(payload),
367        }
368    }
369
370    /// Inflate `payload`, refusing to allocate past the inbound cap.
371    pub fn decode_limited(self, payload: &[u8], limits: MessageLimits) -> Result<Vec<u8>, Status> {
372        self.decode_limited_into(payload, limits, Vec::new())
373    }
374
375    /// Decode into a previously reclaimed, exclusively owned allocation.
376    pub(crate) fn decode_limited_into(
377        self,
378        payload: &[u8],
379        limits: MessageLimits,
380        out: Vec<u8>,
381    ) -> Result<Vec<u8>, Status> {
382        match self {
383            Self::Gzip => gzip::decode_limited_into(payload, limits, out),
384            Self::Deflate => deflate::decode_limited_into(payload, limits, out),
385            #[cfg(feature = "zstd")]
386            Self::Zstd => zstd::decode_limited_into(payload, limits, out),
387        }
388    }
389}
390
391#[cfg(test)]
392mod tests {
393    use super::{CompressionAlgorithm, read_bounded, read_bounded_into};
394    use crate::{Code, MessageLimits};
395    use std::io::Read;
396
397    // Input-buffering wrappers provide an independent I/O wrapper oracle.
398    // Full member/stream consumption follows the new native format policy.
399    fn buffered_decode(
400        codec: CompressionAlgorithm,
401        payload: &[u8],
402        limits: MessageLimits,
403    ) -> Result<Vec<u8>, Code> {
404        let budget = limits.inflate_budget();
405        let read_cap = u64::try_from(budget.saturating_add(1)).unwrap_or(u64::MAX);
406        let mut out =
407            Vec::with_capacity(payload.len().saturating_mul(4).min(256 * 1024).min(budget));
408        let (result, consumed) = match codec {
409            CompressionAlgorithm::Gzip => (
410                flate2::read::MultiGzDecoder::new(payload)
411                    .take(read_cap)
412                    .read_to_end(&mut out),
413                None,
414            ),
415            CompressionAlgorithm::Deflate => {
416                let mut decoder = flate2::read::ZlibDecoder::new(payload);
417                let result = (&mut decoder).take(read_cap).read_to_end(&mut out);
418                (result, Some(decoder.total_in()))
419            }
420            #[cfg(feature = "zstd")]
421            CompressionAlgorithm::Zstd => return Err(Code::Unimplemented),
422        };
423        result.map_err(|_| Code::Internal)?;
424        if out.len() > budget {
425            return Err(Code::ResourceExhausted);
426        }
427        if consumed.is_some_and(|n| u64::try_from(payload.len()).ok() != Some(n)) {
428            return Err(Code::Internal);
429        }
430        Ok(out)
431    }
432
433    #[test]
434    fn reused_output_preserves_decode_limits_and_complete_input_validation() {
435        for codec in codecs() {
436            for length in [0, 1, 8192, 8193, 65539] {
437                let payload = vec![7; length];
438                let encoded = codec.encode(&payload).expect("encode");
439                let mut truncated = encoded.clone();
440                truncated.pop();
441                let mut trailing = encoded.clone();
442                trailing.extend_from_slice(b"invalid trailing data");
443                let mut corrupt = encoded.clone();
444                if let Some(last) = corrupt.last_mut() {
445                    *last ^= 0xff;
446                }
447                for bytes in [&encoded, &truncated, &trailing, &corrupt] {
448                    for budget in [0, length.saturating_sub(1), length, length + 1] {
449                        let limits = MessageLimits::unlimited().with_max_decoding(budget);
450                        let fresh = codec.decode_limited(bytes, limits).map_err(|e| e.code());
451                        let reused = codec
452                            .decode_limited_into(bytes, limits, Vec::with_capacity(131072))
453                            .map_err(|e| e.code());
454                        assert_eq!(reused, fresh, "{codec:?}, length {length}, budget {budget}");
455                    }
456                }
457                assert_eq!(
458                    codec
459                        .decode_limited_into(
460                            &encoded,
461                            MessageLimits::unlimited().with_max_compression_frames(0),
462                            Vec::with_capacity(131072),
463                        )
464                        .expect_err("frame count applies to reused output")
465                        .code(),
466                    Code::ResourceExhausted
467                );
468            }
469        }
470    }
471
472    #[test]
473    fn reused_large_output_handles_smaller_allowances_without_reallocation() {
474        for codec in codecs() {
475            let mut output = Vec::with_capacity(131072);
476            let pointer = output.as_ptr();
477            let capacity = output.capacity();
478            for length in [65539, 1, 8193, 0, 8192, 65539] {
479                let payload = vec![7; length];
480                let encoded = codec.encode(&payload).expect("encode");
481                output = codec
482                    .decode_limited_into(
483                        &encoded,
484                        MessageLimits::unlimited().with_max_decoding(length),
485                        output,
486                    )
487                    .expect("decode with retained allocation larger than allowance");
488                assert_eq!(output, payload);
489                assert_eq!(output.as_ptr(), pointer, "{codec:?}, length {length}");
490                assert_eq!(output.capacity(), capacity);
491            }
492        }
493    }
494
495    #[test]
496    fn retained_capacity_does_not_expand_read_spans_for_a_modest_message() {
497        struct ReadSpans<'a> {
498            remaining: &'a [u8],
499            largest_target: usize,
500        }
501        impl Read for ReadSpans<'_> {
502            fn read(&mut self, target: &mut [u8]) -> std::io::Result<usize> {
503                self.largest_target = self.largest_target.max(target.len());
504                self.remaining.read(target)
505            }
506        }
507
508        let payload = vec![7; 8193];
509        let retained = Vec::with_capacity(1024 * 1024);
510        let pointer = retained.as_ptr();
511        let capacity = retained.capacity();
512        let mut reader = ReadSpans {
513            remaining: &payload,
514            largest_target: 0,
515        };
516        let decoded = read_bounded_into(
517            &mut reader,
518            2 * 1024 * 1024,
519            16,
520            "read-span control",
521            retained,
522        )
523        .expect("small output with a large retained allocation");
524        assert_eq!(decoded, payload);
525        assert_eq!(
526            decoded.as_ptr(),
527            pointer,
528            "the retained allocation stays owned"
529        );
530        assert_eq!(decoded.capacity(), capacity);
531        // A large allocation retained by an earlier message must not become
532        // the next message's initialized output/read span. Keep the bound
533        // proportional to this output rather than to a particular chunk size.
534        assert!(
535            reader.largest_target <= 2 * payload.len(),
536            "a {}-byte message requested a {}-byte read target from retained capacity {}",
537            payload.len(),
538            reader.largest_target,
539            capacity,
540        );
541    }
542
543    #[test]
544    fn bounded_decode_handles_short_reads_interrupts_and_clipped_growth() {
545        struct ShortReader<'a> {
546            remaining: &'a [u8],
547            interrupt: bool,
548            calls: usize,
549        }
550        impl Read for ShortReader<'_> {
551            fn read(&mut self, target: &mut [u8]) -> std::io::Result<usize> {
552                assert!(!target.is_empty(), "an empty read cannot prove decoder EOF");
553                self.calls += 1;
554                self.interrupt = !self.interrupt;
555                if self.interrupt {
556                    return Err(std::io::ErrorKind::Interrupted.into());
557                }
558                if let Some(&byte) = self.remaining.first() {
559                    *target.first_mut().expect("nonempty read target") = byte;
560                    self.remaining = self.remaining.get(1..).unwrap_or_default();
561                    Ok(1)
562                } else {
563                    Ok(0)
564                }
565            }
566        }
567        let payload = b"short reads cross several buffer growth boundaries";
568        for budget in [0, 1, 2, 5, payload.len() - 1, payload.len()] {
569            let mut reader = ShortReader {
570                remaining: payload,
571                interrupt: false,
572                calls: 0,
573            };
574            let result = read_bounded(&mut reader, budget, 0, "test");
575            if budget == payload.len() {
576                assert_eq!(result.expect("exact fit"), payload);
577            } else {
578                assert_eq!(
579                    result.expect_err("over limit").code(),
580                    Code::ResourceExhausted
581                );
582            }
583            assert!(reader.calls <= 2 * (payload.len() + 1));
584        }
585    }
586
587    #[test]
588    fn short_final_chunks_do_not_double_output_capacity() {
589        let budget = 4 * 1024 * 1024;
590        for codec in codecs() {
591            for length in [8191, 8192, 8193, 16385, 32769, 65536, 65539, 73727] {
592                let payload = vec![7; length];
593                let encoded = codec.encode(&payload).expect("encode");
594                let decoded = codec
595                    .decode_limited(
596                        &encoded,
597                        MessageLimits::unlimited().with_max_decoding(budget),
598                    )
599                    .expect("decode");
600                assert_eq!(decoded, payload);
601                assert_eq!(
602                    decoded.capacity(),
603                    length,
604                    "{codec:?}: a final partial chunk must reserve only decoded bytes"
605                );
606            }
607        }
608    }
609
610    #[test]
611    fn bounded_lookahead_handles_short_reads_at_large_growth_boundaries() {
612        struct ShortReader<'a>(&'a [u8]);
613        impl Read for ShortReader<'_> {
614            fn read(&mut self, target: &mut [u8]) -> std::io::Result<usize> {
615                assert!(!target.is_empty());
616                let n = self.0.len().min(target.len()).min(31);
617                target[..n].copy_from_slice(&self.0[..n]);
618                self.0 = &self.0[n..];
619                Ok(n)
620            }
621        }
622        let payload = vec![7; 65539];
623        let decoded =
624            read_bounded(ShortReader(&payload), 4 * 1024 * 1024, 0, "test").expect("short reads");
625        assert_eq!(decoded, payload);
626        assert_eq!(decoded.capacity(), payload.len());
627        for budget in [8192, 8193, 16385, 32769, 65538] {
628            assert_eq!(
629                read_bounded(ShortReader(&payload), budget, 0, "test")
630                    .expect_err("over limit")
631                    .code(),
632                Code::ResourceExhausted
633            );
634        }
635    }
636
637    #[test]
638    fn final_lookahead_preserves_decoder_errors_and_limit_precedence() {
639        struct ErrorAfter<'a>(&'a [u8]);
640        impl Read for ErrorAfter<'_> {
641            fn read(&mut self, target: &mut [u8]) -> std::io::Result<usize> {
642                assert!(!target.is_empty());
643                if self.0.is_empty() {
644                    return Err(std::io::ErrorKind::InvalidData.into());
645                }
646                self.0.read(target)
647            }
648        }
649        let payload = vec![7; 65539];
650        for budget in [payload.len(), payload.len() + 1, 4 * 1024 * 1024] {
651            assert_eq!(
652                read_bounded(ErrorAfter(&payload), budget, 0, "test")
653                    .expect_err("decoder error after final bytes")
654                    .code(),
655                Code::Internal
656            );
657        }
658        assert_eq!(
659            read_bounded(ErrorAfter(&payload), payload.len() - 1, 0, "test")
660                .expect_err("overflow before decoder trailer")
661                .code(),
662            Code::ResourceExhausted
663        );
664    }
665
666    #[test]
667    fn bounded_decode_rejects_invalid_reader_lengths() {
668        struct InvalidReader;
669        impl Read for InvalidReader {
670            fn read(&mut self, target: &mut [u8]) -> std::io::Result<usize> {
671                Ok(target.len() + 1)
672            }
673        }
674        assert_eq!(
675            read_bounded(InvalidReader, 100, 0, "test")
676                .expect_err("reader exceeded target length")
677                .code(),
678            Code::Internal
679        );
680    }
681
682    #[test]
683    fn growth_clipped_to_budget_still_checks_the_compressed_trailer() {
684        for codec in codecs() {
685            let mut encoded = codec.encode(&vec![7; 8193]).expect("compress");
686            encoded.truncate(encoded.len() - 1);
687            let limits = MessageLimits::unlimited().with_max_decoding(8193);
688            assert_eq!(
689                codec
690                    .decode_limited(&encoded, limits)
691                    .expect_err("truncated trailer")
692                    .code(),
693                Code::Internal
694            );
695        }
696    }
697
698    #[test]
699    fn compressed_length_hint_is_bounded_and_waits_for_decoded_bytes() {
700        let budget = 4 * 1024 * 1024;
701        for payload in [b"".as_slice(), b"x".as_slice()] {
702            let decoded = read_bounded(payload, budget, budget, "hint").expect("decode");
703            assert_eq!(decoded, payload);
704            assert_eq!(
705                decoded.capacity(),
706                payload.len(),
707                "observed EOF must override a large compressed-size hint"
708            );
709        }
710    }
711
712    #[test]
713    fn gzip_consumes_all_members_and_rejects_nonmember_padding() {
714        let first = CompressionAlgorithm::Gzip.encode(b"hello").expect("first");
715        let second = CompressionAlgorithm::Gzip
716            .encode(b" world")
717            .expect("second");
718        let joined = [first.as_slice(), second.as_slice()].concat();
719        assert_eq!(
720            CompressionAlgorithm::Gzip
721                .decode_limited(&joined, MessageLimits::new().with_max_decoding(11))
722                .expect("members"),
723            b"hello world"
724        );
725        assert_eq!(
726            CompressionAlgorithm::Gzip
727                .decode_limited(&joined, MessageLimits::new().with_max_decoding(10))
728                .expect_err("aggregate cap")
729                .code(),
730            Code::ResourceExhausted
731        );
732        for tail in [
733            b"garbage".as_slice(),
734            &[0],
735            second.get(..second.len() - 1).expect("truncated"),
736        ] {
737            let joined = [first.as_slice(), tail].concat();
738            assert_eq!(
739                CompressionAlgorithm::Gzip
740                    .decode(&joined)
741                    .expect_err("invalid tail")
742                    .code(),
743                Code::Internal
744            );
745        }
746    }
747
748    #[test]
749    fn deflate_consumes_exactly_one_zlib_stream() {
750        let first = CompressionAlgorithm::Deflate
751            .encode(b"hello")
752            .expect("first");
753        for tail in [
754            vec![0],
755            b"garbage".to_vec(),
756            CompressionAlgorithm::Deflate
757                .encode(b" world")
758                .expect("second"),
759        ] {
760            let joined = [first.as_slice(), tail.as_slice()].concat();
761            assert_eq!(
762                CompressionAlgorithm::Deflate
763                    .decode(&joined)
764                    .expect_err("trailing bytes")
765                    .code(),
766                Code::Internal
767            );
768        }
769    }
770
771    #[test]
772    fn decoded_output_capacity_never_grows_past_the_limit() {
773        // Non-power-of-two limits catch geometric Vec growth past the cap.
774        // Highly compressed input also exercises growth far beyond a size guess.
775        for codec in codecs() {
776            for length in [0, 1, 1001, 8191, 8192, 8193, 8194, 32769, 32770, 65539] {
777                let payload = vec![7; length];
778                let encoded = codec.encode(&payload).expect("encode");
779                for budget in [length, length + 1] {
780                    let limits = MessageLimits::unlimited().with_max_decoding(budget);
781                    let decoded = codec.decode_limited(&encoded, limits).expect("decode");
782                    assert_eq!(decoded, payload);
783                    assert!(
784                        decoded.capacity() <= budget,
785                        "{codec:?}: output capacity {} exceeds budget {budget}",
786                        decoded.capacity()
787                    );
788                }
789                if length != 0 {
790                    let limits = MessageLimits::unlimited().with_max_decoding(length - 1);
791                    assert_eq!(
792                        codec
793                            .decode_limited(&encoded, limits)
794                            .expect_err("over limit")
795                            .code(),
796                        Code::ResourceExhausted
797                    );
798                }
799            }
800        }
801    }
802
803    #[test]
804    fn slice_decoding_matches_buffered_input_on_malformed_data_and_limits() {
805        for codec in [CompressionAlgorithm::Gzip, CompressionAlgorithm::Deflate] {
806            let payload = b"protobuf compression wrapper compatibility".repeat(100);
807            let compressed = codec.encode(&payload).expect("encode");
808            let mut inputs = vec![compressed.clone(), Vec::new(), vec![0xff; 32]];
809            // Every truncation, and a bit mutation in every header/body/trailer byte.
810            for end in 0..compressed.len() {
811                inputs.push(compressed[..end].to_vec());
812                let mut changed = compressed.clone();
813                changed[end] ^= 0x80;
814                inputs.push(changed);
815            }
816            for input in inputs {
817                for budget in [0, 1, payload.len() - 1, payload.len(), payload.len() + 1] {
818                    let limits = MessageLimits::unlimited().with_max_decoding(budget);
819                    let actual = codec.decode_limited(&input, limits).map_err(|e| e.code());
820                    assert_eq!(
821                        actual,
822                        buffered_decode(codec, &input, limits),
823                        "codec {codec:?}, input {input:?}, budget {budget}"
824                    );
825                }
826            }
827        }
828    }
829
830    fn codecs() -> Vec<CompressionAlgorithm> {
831        #[cfg(feature = "zstd")]
832        {
833            vec![
834                CompressionAlgorithm::Gzip,
835                CompressionAlgorithm::Deflate,
836                CompressionAlgorithm::Zstd,
837            ]
838        }
839        #[cfg(not(feature = "zstd"))]
840        {
841            vec![CompressionAlgorithm::Gzip, CompressionAlgorithm::Deflate]
842        }
843    }
844
845    #[test]
846    fn parse_tokens() {
847        assert_eq!(
848            CompressionAlgorithm::parse("gzip"),
849            Some(CompressionAlgorithm::Gzip)
850        );
851        assert_eq!(
852            CompressionAlgorithm::parse("GZIP"),
853            Some(CompressionAlgorithm::Gzip)
854        );
855        assert_eq!(
856            CompressionAlgorithm::parse("  deflate;q=0.5 "),
857            Some(CompressionAlgorithm::Deflate)
858        );
859        #[cfg(feature = "zstd")]
860        assert_eq!(
861            CompressionAlgorithm::parse(" ZSTD;q=0.5 "),
862            Some(CompressionAlgorithm::Zstd)
863        );
864        #[cfg(not(feature = "zstd"))]
865        assert_eq!(CompressionAlgorithm::parse("zstd"), None);
866        assert_eq!(CompressionAlgorithm::parse("identity"), None);
867        assert_eq!(CompressionAlgorithm::parse(""), None);
868        assert_eq!(CompressionAlgorithm::parse("snappy"), None);
869        assert_eq!(CompressionAlgorithm::Gzip.name(), "gzip");
870        assert_eq!(CompressionAlgorithm::Deflate.name(), "deflate");
871        #[cfg(feature = "zstd")]
872        assert_eq!(CompressionAlgorithm::Zstd.name(), "zstd");
873        assert_eq!(CompressionAlgorithm::default(), CompressionAlgorithm::Gzip);
874    }
875
876    #[test]
877    fn both_codecs_round_trip() {
878        let payload = b"the quick brown fox jumps over the lazy dog".repeat(16);
879        for codec in codecs() {
880            let enc = codec.encode(&payload).expect("encode");
881            let dec = codec
882                .decode_limited(&enc, crate::limits::MessageLimits::unlimited())
883                .expect("decode");
884            assert_eq!(dec, payload, "codec {codec:?}");
885        }
886    }
887
888    #[test]
889    fn encode_into_matches_encode() {
890        let payload = b"the quick brown fox jumps over the lazy dog".repeat(16);
891        for codec in codecs() {
892            let mut into = Vec::new();
893            codec
894                .encode_into(&payload, 1, &mut into)
895                .expect("encode_into");
896            assert_eq!(into, codec.encode(&payload).expect("encode"));
897        }
898    }
899
900    #[test]
901    fn codings_are_not_cross_decodable() {
902        // A gzip member is not a zlib stream and vice versa: the
903        // decoder must follow the RPC's coding, not guess.
904        let payload = b"the quick brown fox jumps over the lazy dog".repeat(16);
905        let gz = CompressionAlgorithm::Gzip.encode(&payload).expect("gzip");
906        let df = CompressionAlgorithm::Deflate
907            .encode(&payload)
908            .expect("deflate");
909        let limits = crate::limits::MessageLimits::unlimited();
910        assert!(
911            CompressionAlgorithm::Deflate
912                .decode_limited(&gz, limits)
913                .is_err()
914        );
915        assert!(
916            CompressionAlgorithm::Gzip
917                .decode_limited(&df, limits)
918                .is_err()
919        );
920        #[cfg(feature = "zstd")]
921        {
922            let zst = CompressionAlgorithm::Zstd.encode(&payload).expect("zstd");
923            assert!(
924                CompressionAlgorithm::Gzip
925                    .decode_limited(&zst, limits)
926                    .is_err()
927            );
928            assert!(
929                CompressionAlgorithm::Deflate
930                    .decode_limited(&zst, limits)
931                    .is_err()
932            );
933            assert!(
934                CompressionAlgorithm::Zstd
935                    .decode_limited(&gz, limits)
936                    .is_err()
937            );
938            assert!(
939                CompressionAlgorithm::Zstd
940                    .decode_limited(&df, limits)
941                    .is_err()
942            );
943        }
944    }
945}