1use crate::{Error, Result};
4pub fn decode(encoding: &str, input: &[u8], output: &mut Vec<u8>, limit: usize) -> Result<()> {
7 output.clear();
8 let mut decoder = StreamingDecoder::new(encoding.trim(), limit)?;
9 let mut pos = 0;
10 loop {
11 let mut bytes = [0; 8192];
12 let step = decoder.process(&input[pos..], &mut bytes, true)?;
13 pos += step.consumed;
14 output.extend_from_slice(&bytes[..step.written]);
15 if step.finished {
16 return Ok(());
17 }
18 if step.needs_input {
20 return Err(corrupt());
21 }
22 }
23}
24
25#[derive(Debug)]
42pub struct DecodeStep {
43 pub consumed: usize,
44 pub written: usize,
45 pub finished: bool,
46 pub needs_input: bool,
48}
49#[derive(Clone, Copy, PartialEq, Eq)]
50enum Stage {
51 Header,
52 Body,
53 Trailer,
54 Done,
55}
56type BytePool = crate::recycling::Pool<u8>;
57type IntPool = crate::recycling::Pool<u32>;
58type CodePool = crate::recycling::Pool<brotli_decompressor::HuffmanCode>;
59type BrotliState = brotli_decompressor::BrotliState<BytePool, IntPool, CodePool>;
60#[allow(clippy::large_enum_variant)]
62enum Engine {
63 Identity,
64 Deflate {
65 decoder: flate2::Decompress,
66 gzip: bool,
67 detect: bool,
68 stage: Stage,
69 crc: flate2::Crc,
70 members: usize,
71 },
72 Brotli {
73 state: Box<BrotliState>,
74 bytes: BytePool,
75 ints: IntPool,
76 codes: CodePool,
77 },
78 #[cfg(not(any(target_arch = "wasm32", feature = "pure-rust-zstd")))]
79 Zstd {
80 decoder: zstd::stream::raw::Decoder<'static>,
81 boundary: bool,
82 },
83 #[cfg(any(target_arch = "wasm32", feature = "pure-rust-zstd"))]
84 Zstd {
85 decoder: turnloop_zstd_decoder::decoding::FrameDecoder,
86 reset: bool,
87 boundary: bool,
88 },
89}
90pub const MAX_CODINGS: usize = 5;
93const STAGE_BYTES: usize = 16 * 1024;
96pub struct StreamingDecoder {
99 codec: Codec,
102 outer: Vec<Staged>,
106 limit: usize,
107 failed: bool,
108}
109struct Codec {
111 engine: Engine,
112 total: usize,
113 done: bool,
114}
115struct Staged {
117 codec: Codec,
118 buffer: Box<[u8]>,
119 start: usize,
120 end: usize,
121}
122fn unsupported() -> Error {
123 Error::new("UND_ERR_NOT_SUPPORTED", "unsupported content encoding")
124}
125fn corrupt() -> Error {
126 Error::new("UND_ERR_SOCKET", "invalid or incomplete compressed body")
127}
128impl Codec {
129 fn new(token: &[u8]) -> Result<Self> {
130 let name = std::str::from_utf8(token)
132 .map_err(|_| unsupported())?
133 .to_ascii_lowercase();
134 let engine = match name.as_str() {
135 "" | "identity" => Engine::Identity,
136 "gzip" | "x-gzip" => Engine::Deflate {
137 decoder: flate2::Decompress::new(false),
138 gzip: true,
139 detect: false,
140 stage: Stage::Header,
141 crc: flate2::Crc::new(),
142 members: 0,
143 },
144 "deflate" => Engine::Deflate {
145 decoder: flate2::Decompress::new(false),
146 gzip: false,
147 detect: true,
148 stage: Stage::Body,
149 crc: flate2::Crc::new(),
150 members: 0,
151 },
152 "br" => {
153 let bytes = BytePool::default();
154 let ints = IntPool::default();
155 let codes = CodePool::default();
156 let state = Box::new(BrotliState::new_strict(
157 bytes.clone(),
158 ints.clone(),
159 codes.clone(),
160 ));
161 Engine::Brotli {
162 state,
163 bytes,
164 ints,
165 codes,
166 }
167 }
168 #[cfg(not(any(target_arch = "wasm32", feature = "pure-rust-zstd")))]
169 "zstd" => {
170 let mut decoder = zstd::stream::raw::Decoder::new().map_err(|_| corrupt())?;
171 decoder
172 .set_parameter(zstd::stream::raw::DParameter::WindowLogMax(27))
173 .map_err(|_| corrupt())?;
174 Engine::Zstd {
175 decoder,
176 boundary: false,
177 }
178 }
179 #[cfg(any(target_arch = "wasm32", feature = "pure-rust-zstd"))]
180 "zstd" => Engine::Zstd {
181 decoder: turnloop_zstd_decoder::decoding::FrameDecoder::new(),
182 reset: true,
183 boundary: false,
184 },
185 _ => return Err(unsupported()),
186 };
187 Ok(Self {
188 engine,
189 total: 0,
190 done: false,
191 })
192 }
193 fn reset(&mut self) -> Result<()> {
194 match &mut self.engine {
195 Engine::Identity => {}
196 Engine::Deflate {
197 decoder,
198 gzip,
199 detect,
200 stage,
201 crc,
202 members,
203 } => {
204 decoder.reset(false);
205 *detect = !*gzip;
206 *stage = if *gzip { Stage::Header } else { Stage::Body };
207 crc.reset();
208 *members = 0;
209 }
210 Engine::Brotli {
211 state,
212 bytes,
213 ints,
214 codes,
215 } => {
216 **state = BrotliState::new_strict(bytes.clone(), ints.clone(), codes.clone());
217 }
218 #[cfg(not(any(target_arch = "wasm32", feature = "pure-rust-zstd")))]
219 Engine::Zstd { decoder, boundary } => {
220 use zstd::stream::raw::Operation;
221 decoder.reinit().map_err(|_| corrupt())?;
222 *boundary = false;
223 }
224 #[cfg(any(target_arch = "wasm32", feature = "pure-rust-zstd"))]
225 Engine::Zstd {
226 reset, boundary, ..
227 } => {
228 *reset = true;
229 *boundary = false;
230 }
231 }
232 self.total = 0;
233 self.done = false;
234 Ok(())
235 }
236 fn run(
237 &mut self,
238 input: &[u8],
239 output: &mut [u8],
240 end: bool,
241 limit: usize,
242 ) -> Result<DecodeStep> {
243 let mut consumed = 0;
246 let mut written = 0;
247 loop {
248 let step = self.step_once(&input[consumed..], &mut output[written..], end, limit)?;
249 consumed += step.consumed;
250 written += step.written;
251 let stalled = step.consumed == 0 && step.written == 0;
252 if step.finished || written == output.len() || stalled {
253 return Ok(DecodeStep {
254 consumed,
255 written,
256 finished: step.finished,
257 needs_input: !step.finished && written < output.len(),
258 });
259 }
260 }
261 }
262 fn step_once(
265 &mut self,
266 input: &[u8],
267 output: &mut [u8],
268 end: bool,
269 limit: usize,
270 ) -> Result<DecodeStep> {
271 if self.done {
272 if !input.is_empty() {
273 return Err(corrupt());
274 }
275 return Ok(DecodeStep {
276 consumed: 0,
277 written: 0,
278 finished: true,
279 needs_input: false,
280 });
281 }
282 if output.is_empty() {
283 return Ok(DecodeStep {
284 consumed: 0,
285 written: 0,
286 finished: false,
287 needs_input: false,
288 });
289 }
290 let mut consumed = 0;
291 let mut written = 0;
292 let mut finished = false;
293 match &mut self.engine {
294 Engine::Identity => {
295 consumed = input.len().min(output.len());
296 output[..consumed].copy_from_slice(&input[..consumed]);
297 written = consumed;
298 finished = end && consumed == input.len();
299 }
300 Engine::Deflate {
301 decoder,
302 gzip,
303 detect,
304 stage,
305 crc,
306 members,
307 } => {
308 if *stage == Stage::Header {
309 if input.is_empty() && end && *members > 0 {
310 finished = true;
311 } else if let Some(n) = gzip_header(input)? {
312 consumed = n;
313 decoder.reset(false);
314 crc.reset();
315 *stage = Stage::Body;
316 } else if end {
317 return Err(corrupt());
318 }
319 } else if *stage == Stage::Trailer {
320 if input.len() >= 8 {
321 if u32::from_le_bytes(input[..4].try_into().unwrap()) != crc.sum()
322 || u32::from_le_bytes(input[4..8].try_into().unwrap()) != crc.amount()
323 {
324 return Err(corrupt());
325 }
326 consumed = 8;
327 *members += 1;
328 *stage = Stage::Header;
329 finished = end && input.len() == 8;
330 } else if end {
331 return Err(corrupt());
332 }
333 } else if *stage == Stage::Done {
334 if !input.is_empty() {
335 return Err(corrupt());
336 }
337 finished = end;
338 } else {
339 if *detect {
340 if input.len() < 2 {
341 if end {
342 return Err(corrupt());
343 }
344 return Ok(DecodeStep {
345 consumed: 0,
346 written: 0,
347 finished: false,
348 needs_input: false,
349 });
350 }
351 let zlib = input[0] & 15 == 8
352 && u16::from_be_bytes([input[0], input[1]]).is_multiple_of(31);
353 decoder.reset(zlib);
354 *detect = false;
355 }
356 let before_in = decoder.total_in();
357 let before_out = decoder.total_out();
358 let status = decoder
359 .decompress(input, output, flate2::FlushDecompress::None)
360 .map_err(|_| corrupt())?;
361 consumed = (decoder.total_in() - before_in) as usize;
362 written = (decoder.total_out() - before_out) as usize;
363 if *gzip {
364 crc.update(&output[..written]);
365 }
366 if status == flate2::Status::StreamEnd {
367 *stage = if *gzip { Stage::Trailer } else { Stage::Done };
368 finished = !*gzip && end && consumed == input.len();
369 } else if end && consumed == input.len() && written < output.len() {
370 return Err(corrupt());
371 }
372 }
373 }
374 Engine::Brotli { state, .. } => {
375 let mut available_in = input.len();
376 let mut available_out = output.len();
377 let mut total = self.total;
378 let result = brotli_decompressor::BrotliDecompressStream(
379 &mut available_in,
380 &mut consumed,
381 input,
382 &mut available_out,
383 &mut written,
384 output,
385 &mut total,
386 state,
387 );
388 match result {
389 brotli_decompressor::BrotliResult::ResultSuccess => {
390 if consumed != input.len() {
391 return Err(corrupt());
392 }
393 finished = true;
394 }
395 brotli_decompressor::BrotliResult::ResultFailure => return Err(corrupt()),
396 brotli_decompressor::BrotliResult::NeedsMoreInput if end => {
397 return Err(corrupt());
398 }
399 _ => {}
400 }
401 }
402 #[cfg(not(any(target_arch = "wasm32", feature = "pure-rust-zstd")))]
403 Engine::Zstd { decoder, boundary } => {
404 use zstd::stream::raw::Operation;
405 if input.is_empty() && end && *boundary {
406 self.done = true;
407 return Ok(DecodeStep {
408 consumed: 0,
409 written: 0,
410 finished: true,
411 needs_input: false,
412 });
413 }
414 let result = decoder
415 .run_on_buffers(input, output)
416 .map_err(|_| corrupt())?;
417 consumed = result.bytes_read;
418 written = result.bytes_written;
419 if consumed > 0 || written > 0 {
423 *boundary = result.remaining == 0;
424 }
425 finished = *boundary && end && consumed == input.len();
426 if !*boundary && end && consumed == input.len() && written < output.len() {
427 return Err(corrupt());
428 }
429 }
430 #[cfg(any(target_arch = "wasm32", feature = "pure-rust-zstd"))]
431 Engine::Zstd {
432 decoder,
433 reset,
434 boundary,
435 } => {
436 if input.is_empty() && end && *boundary {
437 self.done = true;
438 return Ok(DecodeStep {
439 consumed: 0,
440 written: 0,
441 finished: true,
442 needs_input: false,
443 });
444 }
445 let mut source = input;
446 if *reset {
447 if source.len() < 18 && !end {
448 return Ok(DecodeStep {
449 consumed: 0,
450 written: 0,
451 finished: false,
452 needs_input: false,
453 });
454 }
455 decoder.reset(&mut source).map_err(|_| corrupt())?;
456 consumed = input.len() - source.len();
457 *reset = false;
458 *boundary = false;
459 }
460 let result = decoder
461 .decode_from_to(source, output)
462 .map_err(|_| corrupt())?;
463 consumed += result.0;
464 written = result.1;
465 if decoder.is_finished() && decoder.can_collect() == 0 {
466 if let Some(expected) = decoder.get_checksum_from_data()
467 && decoder.get_calculated_checksum() != Some(expected)
468 {
469 return Err(corrupt());
470 }
471 *boundary = true;
472 *reset = true;
473 finished = end && consumed == input.len();
474 } else if end && consumed == 0 && written == 0 {
475 return Err(corrupt());
476 }
477 }
478 }
479 if written > limit.saturating_sub(self.total) {
480 return Err(Error::new(
481 "UND_ERR_RES_EXCEEDED_MAX_SIZE",
482 "decoded body exceeds limit",
483 ));
484 }
485 self.total += written;
486 self.done = finished;
487 Ok(DecodeStep {
488 consumed,
489 written,
490 finished,
491 needs_input: false,
492 })
493 }
494}
495impl Staged {
496 fn compact(&mut self) {
498 if self.start > 0 {
499 self.buffer.copy_within(self.start..self.end, 0);
500 self.end -= self.start;
501 self.start = 0;
502 }
503 }
504}
505impl StreamingDecoder {
506 pub fn new(encoding: &str, limit: usize) -> Result<Self> {
510 Self::from_codings([encoding], limit)
511 }
512 pub fn from_codings<I>(values: I, limit: usize) -> Result<Self>
530 where
531 I: IntoIterator,
532 I::Item: AsRef<[u8]>,
533 {
534 let mut codecs = Vec::new();
535 for value in values {
536 for token in value.as_ref().split(|b| *b == b',') {
537 let token = token.trim_ascii();
538 if token.is_empty() || token.eq_ignore_ascii_case(b"identity") {
539 continue;
540 }
541 if codecs.len() == MAX_CODINGS {
542 return Err(Error::new(
543 "UND_ERR_NOT_SUPPORTED",
544 "too many content codings",
545 ));
546 }
547 codecs.push(Codec::new(token)?);
548 }
549 }
550 let mut codecs = codecs.into_iter();
551 let codec = match codecs.next() {
552 Some(codec) => codec,
553 None => Codec::new(b"identity")?,
554 };
555 let outer = codecs
556 .rev()
557 .map(|codec| Staged {
558 codec,
559 buffer: vec![0; STAGE_BYTES].into_boxed_slice(),
560 start: 0,
561 end: 0,
562 })
563 .collect();
564 Ok(Self {
565 codec,
566 outer,
567 limit,
568 failed: false,
569 })
570 }
571 pub fn reset(&mut self, limit: usize) -> Result<()> {
574 self.codec.reset()?;
575 for stage in &mut self.outer {
576 stage.codec.reset()?;
577 stage.start = 0;
578 stage.end = 0;
579 }
580 self.limit = limit;
581 self.failed = false;
582 Ok(())
583 }
584 pub fn process(&mut self, input: &[u8], output: &mut [u8], end: bool) -> Result<DecodeStep> {
592 if self.failed {
593 return Err(corrupt());
594 }
595 let result = self.process_inner(input, output, end);
596 if result.is_err() {
597 self.failed = true;
598 }
599 result
600 }
601 fn process_inner(&mut self, input: &[u8], output: &mut [u8], end: bool) -> Result<DecodeStep> {
602 let limit = self.limit;
603 let Some((last, earlier)) = self.outer.split_last_mut() else {
604 return self.codec.run(input, output, end, limit);
605 };
606 if self.codec.done {
607 return self.codec.run(input, output, end, limit);
609 }
610 let mut consumed = 0;
611 let mut written = 0;
612 loop {
613 let mut progress = false;
614 for i in 0..=earlier.len() {
618 let (before, rest) = earlier.split_at_mut(i);
619 let stage = match rest.first_mut() {
620 Some(stage) => stage,
621 None => &mut *last,
622 };
623 stage.compact();
624 if stage.end == stage.buffer.len() {
625 continue;
626 }
627 let step = match before.last_mut() {
628 None => {
629 let step = stage.codec.run(
630 &input[consumed..],
631 &mut stage.buffer[stage.end..],
632 end,
633 limit,
634 )?;
635 consumed += step.consumed;
636 step
637 }
638 Some(feed) => {
639 let step = stage.codec.run(
640 &feed.buffer[feed.start..feed.end],
641 &mut stage.buffer[stage.end..],
642 feed.codec.done,
643 limit,
644 )?;
645 feed.start += step.consumed;
646 step
647 }
648 };
649 stage.end += step.written;
650 progress |= step.consumed > 0 || step.written > 0;
651 }
652 let step = self.codec.run(
654 &last.buffer[last.start..last.end],
655 &mut output[written..],
656 last.codec.done,
657 limit,
658 )?;
659 last.start += step.consumed;
660 written += step.written;
661 progress |= step.consumed > 0 || step.written > 0;
662 if step.finished || written == output.len() || !progress {
663 return Ok(DecodeStep {
664 consumed,
665 written,
666 finished: step.finished,
667 needs_input: !step.finished && written < output.len(),
668 });
669 }
670 }
671 }
672}
673fn gzip_header(input: &[u8]) -> Result<Option<usize>> {
674 if input.len() < 10 {
675 return Ok(None);
676 }
677 if input[..3] != [31, 139, 8] || input[3] & 0xe0 != 0 {
678 return Err(corrupt());
679 }
680 let flags = input[3];
681 let mut pos = 10;
682 if flags & 4 != 0 {
683 if input.len() < 12 {
684 return Ok(None);
685 }
686 pos = 12 + u16::from_le_bytes(input[10..12].try_into().unwrap()) as usize;
687 }
688 for flag in [8, 16] {
689 if flags & flag != 0 {
690 if pos > input.len() {
691 break;
692 }
693 if let Some(n) = input[pos..].iter().position(|b| *b == 0) {
694 pos += n + 1;
695 } else {
696 if input.len() > 8192 {
697 return Err(corrupt());
698 }
699 return Ok(None);
700 }
701 }
702 }
703 if pos > 8192 {
704 return Err(corrupt());
705 }
706 if input.len() < pos {
707 return Ok(None);
708 }
709 if flags & 2 != 0 {
710 if input.len() < pos + 2 {
711 return Ok(None);
712 }
713 let mut crc = flate2::Crc::new();
714 crc.update(&input[..pos]);
715 if crc.sum() as u16 != u16::from_le_bytes(input[pos..pos + 2].try_into().unwrap()) {
716 return Err(corrupt());
717 }
718 pos += 2;
719 }
720 Ok(Some(pos))
721}