1use crate::{Error, Result};
4pub fn decode(encoding: &str, input: &[u8], output: &mut Vec<u8>, limit: usize) -> Result<()> {
7 output.clear();
8 let mut decoder = StreamingDecoder::new(encoding.trim(), limit)?;
9 let mut pos = 0;
10 loop {
11 let mut bytes = [0; 8192];
12 let step = decoder.process(&input[pos..], &mut bytes, true)?;
13 pos += step.consumed;
14 output.extend_from_slice(&bytes[..step.written]);
15 if step.finished {
16 return Ok(());
17 }
18 if step.needs_input {
20 return Err(corrupt());
21 }
22 }
23}
24
25#[derive(Debug)]
42pub struct DecodeStep {
43 pub consumed: usize,
44 pub written: usize,
45 pub finished: bool,
46 pub needs_input: bool,
48}
49#[derive(Clone, Copy, PartialEq, Eq)]
50enum Stage {
51 Header,
52 Body,
53 Trailer,
54 Done,
55}
56type BytePool = crate::recycling::Pool<u8>;
57type IntPool = crate::recycling::Pool<u32>;
58type CodePool = crate::recycling::Pool<brotli::HuffmanCode>;
59type BrotliState = brotli::BrotliState<BytePool, IntPool, CodePool>;
60#[allow(clippy::large_enum_variant)]
62enum Engine {
63 Identity,
64 Deflate {
65 decoder: flate2::Decompress,
66 gzip: bool,
67 detect: bool,
68 stage: Stage,
69 crc: flate2::Crc,
70 members: usize,
71 },
72 Brotli {
73 state: Box<BrotliState>,
74 bytes: BytePool,
75 ints: IntPool,
76 codes: CodePool,
77 },
78 #[cfg(not(any(target_arch = "wasm32", feature = "pure-rust-zstd")))]
79 Zstd {
80 decoder: zstd::stream::raw::Decoder<'static>,
81 boundary: bool,
82 },
83 #[cfg(any(target_arch = "wasm32", feature = "pure-rust-zstd"))]
84 Zstd {
85 decoder: turnloop_zstd_decoder::decoding::FrameDecoder,
86 reset: bool,
87 boundary: bool,
88 },
89}
90pub const MAX_CODINGS: usize = 5;
93const STAGE_BYTES: usize = 16 * 1024;
96pub struct StreamingDecoder {
99 codec: Codec,
102 outer: Vec<Staged>,
106 limit: usize,
107 failed: bool,
108}
109struct Codec {
111 engine: Engine,
112 total: usize,
113 done: bool,
114}
115struct Staged {
117 codec: Codec,
118 buffer: Box<[u8]>,
119 start: usize,
120 end: usize,
121}
122fn unsupported() -> Error {
123 Error::new("UND_ERR_NOT_SUPPORTED", "unsupported content encoding")
124}
125fn corrupt() -> Error {
126 Error::new("UND_ERR_SOCKET", "invalid or incomplete compressed body")
127}
128impl Codec {
129 fn new(token: &[u8]) -> Result<Self> {
130 let name = std::str::from_utf8(token)
132 .map_err(|_| unsupported())?
133 .to_ascii_lowercase();
134 let engine = match name.as_str() {
135 "" | "identity" => Engine::Identity,
136 "gzip" | "x-gzip" => Engine::Deflate {
137 decoder: flate2::Decompress::new(false),
138 gzip: true,
139 detect: false,
140 stage: Stage::Header,
141 crc: flate2::Crc::new(),
142 members: 0,
143 },
144 "deflate" => Engine::Deflate {
145 decoder: flate2::Decompress::new(false),
146 gzip: false,
147 detect: true,
148 stage: Stage::Body,
149 crc: flate2::Crc::new(),
150 members: 0,
151 },
152 "br" => {
153 let bytes = BytePool::default();
154 let ints = IntPool::default();
155 let codes = CodePool::default();
156 let state = Box::new(BrotliState::new(bytes.clone(), ints.clone(), codes.clone()));
157 Engine::Brotli {
158 state,
159 bytes,
160 ints,
161 codes,
162 }
163 }
164 #[cfg(not(any(target_arch = "wasm32", feature = "pure-rust-zstd")))]
165 "zstd" => {
166 let mut decoder = zstd::stream::raw::Decoder::new().map_err(|_| corrupt())?;
167 decoder
168 .set_parameter(zstd::stream::raw::DParameter::WindowLogMax(27))
169 .map_err(|_| corrupt())?;
170 Engine::Zstd {
171 decoder,
172 boundary: false,
173 }
174 }
175 #[cfg(any(target_arch = "wasm32", feature = "pure-rust-zstd"))]
176 "zstd" => Engine::Zstd {
177 decoder: turnloop_zstd_decoder::decoding::FrameDecoder::new(),
178 reset: true,
179 boundary: false,
180 },
181 _ => return Err(unsupported()),
182 };
183 Ok(Self {
184 engine,
185 total: 0,
186 done: false,
187 })
188 }
189 fn reset(&mut self) -> Result<()> {
190 match &mut self.engine {
191 Engine::Identity => {}
192 Engine::Deflate {
193 decoder,
194 gzip,
195 detect,
196 stage,
197 crc,
198 members,
199 } => {
200 decoder.reset(false);
201 *detect = !*gzip;
202 *stage = if *gzip { Stage::Header } else { Stage::Body };
203 crc.reset();
204 *members = 0;
205 }
206 Engine::Brotli {
207 state,
208 bytes,
209 ints,
210 codes,
211 } => {
212 **state = BrotliState::new(bytes.clone(), ints.clone(), codes.clone());
213 }
214 #[cfg(not(any(target_arch = "wasm32", feature = "pure-rust-zstd")))]
215 Engine::Zstd { decoder, boundary } => {
216 use zstd::stream::raw::Operation;
217 decoder.reinit().map_err(|_| corrupt())?;
218 *boundary = false;
219 }
220 #[cfg(any(target_arch = "wasm32", feature = "pure-rust-zstd"))]
221 Engine::Zstd {
222 reset, boundary, ..
223 } => {
224 *reset = true;
225 *boundary = false;
226 }
227 }
228 self.total = 0;
229 self.done = false;
230 Ok(())
231 }
232 fn run(
233 &mut self,
234 input: &[u8],
235 output: &mut [u8],
236 end: bool,
237 limit: usize,
238 ) -> Result<DecodeStep> {
239 let mut consumed = 0;
242 let mut written = 0;
243 loop {
244 let step = self.step_once(&input[consumed..], &mut output[written..], end, limit)?;
245 consumed += step.consumed;
246 written += step.written;
247 let stalled = step.consumed == 0 && step.written == 0;
248 if step.finished || written == output.len() || stalled {
249 return Ok(DecodeStep {
250 consumed,
251 written,
252 finished: step.finished,
253 needs_input: !step.finished && written < output.len(),
254 });
255 }
256 }
257 }
258 fn step_once(
261 &mut self,
262 input: &[u8],
263 output: &mut [u8],
264 end: bool,
265 limit: usize,
266 ) -> Result<DecodeStep> {
267 if self.done {
268 if !input.is_empty() {
269 return Err(corrupt());
270 }
271 return Ok(DecodeStep {
272 consumed: 0,
273 written: 0,
274 finished: true,
275 needs_input: false,
276 });
277 }
278 if output.is_empty() {
279 return Ok(DecodeStep {
280 consumed: 0,
281 written: 0,
282 finished: false,
283 needs_input: false,
284 });
285 }
286 let mut consumed = 0;
287 let mut written = 0;
288 let mut finished = false;
289 match &mut self.engine {
290 Engine::Identity => {
291 consumed = input.len().min(output.len());
292 output[..consumed].copy_from_slice(&input[..consumed]);
293 written = consumed;
294 finished = end && consumed == input.len();
295 }
296 Engine::Deflate {
297 decoder,
298 gzip,
299 detect,
300 stage,
301 crc,
302 members,
303 } => {
304 if *stage == Stage::Header {
305 if input.is_empty() && end && *members > 0 {
306 finished = true;
307 } else if let Some(n) = gzip_header(input)? {
308 consumed = n;
309 decoder.reset(false);
310 crc.reset();
311 *stage = Stage::Body;
312 } else if end {
313 return Err(corrupt());
314 }
315 } else if *stage == Stage::Trailer {
316 if input.len() >= 8 {
317 if u32::from_le_bytes(input[..4].try_into().unwrap()) != crc.sum()
318 || u32::from_le_bytes(input[4..8].try_into().unwrap()) != crc.amount()
319 {
320 return Err(corrupt());
321 }
322 consumed = 8;
323 *members += 1;
324 *stage = Stage::Header;
325 finished = end && input.len() == 8;
326 } else if end {
327 return Err(corrupt());
328 }
329 } else if *stage == Stage::Done {
330 if !input.is_empty() {
331 return Err(corrupt());
332 }
333 finished = end;
334 } else {
335 if *detect {
336 if input.len() < 2 {
337 if end {
338 return Err(corrupt());
339 }
340 return Ok(DecodeStep {
341 consumed: 0,
342 written: 0,
343 finished: false,
344 needs_input: false,
345 });
346 }
347 let zlib = input[0] & 15 == 8
348 && u16::from_be_bytes([input[0], input[1]]).is_multiple_of(31);
349 decoder.reset(zlib);
350 *detect = false;
351 }
352 let before_in = decoder.total_in();
353 let before_out = decoder.total_out();
354 let status = decoder
355 .decompress(input, output, flate2::FlushDecompress::None)
356 .map_err(|_| corrupt())?;
357 consumed = (decoder.total_in() - before_in) as usize;
358 written = (decoder.total_out() - before_out) as usize;
359 if *gzip {
360 crc.update(&output[..written]);
361 }
362 if status == flate2::Status::StreamEnd {
363 *stage = if *gzip { Stage::Trailer } else { Stage::Done };
364 finished = !*gzip && end && consumed == input.len();
365 } else if end && consumed == input.len() && written < output.len() {
366 return Err(corrupt());
367 }
368 }
369 }
370 Engine::Brotli { state, .. } => {
371 let mut available_in = input.len();
372 let mut available_out = output.len();
373 let mut total = self.total;
374 let result = brotli::BrotliDecompressStream(
375 &mut available_in,
376 &mut consumed,
377 input,
378 &mut available_out,
379 &mut written,
380 output,
381 &mut total,
382 state,
383 );
384 match result {
385 brotli::BrotliResult::ResultSuccess => {
386 if consumed != input.len() {
387 return Err(corrupt());
388 }
389 finished = true;
390 }
391 brotli::BrotliResult::ResultFailure => return Err(corrupt()),
392 brotli::BrotliResult::NeedsMoreInput if end => return Err(corrupt()),
393 _ => {}
394 }
395 }
396 #[cfg(not(any(target_arch = "wasm32", feature = "pure-rust-zstd")))]
397 Engine::Zstd { decoder, boundary } => {
398 use zstd::stream::raw::Operation;
399 if input.is_empty() && end && *boundary {
400 self.done = true;
401 return Ok(DecodeStep {
402 consumed: 0,
403 written: 0,
404 finished: true,
405 needs_input: false,
406 });
407 }
408 let result = decoder
409 .run_on_buffers(input, output)
410 .map_err(|_| corrupt())?;
411 consumed = result.bytes_read;
412 written = result.bytes_written;
413 if consumed > 0 || written > 0 {
417 *boundary = result.remaining == 0;
418 }
419 finished = *boundary && end && consumed == input.len();
420 if !*boundary && end && consumed == input.len() && written < output.len() {
421 return Err(corrupt());
422 }
423 }
424 #[cfg(any(target_arch = "wasm32", feature = "pure-rust-zstd"))]
425 Engine::Zstd {
426 decoder,
427 reset,
428 boundary,
429 } => {
430 if input.is_empty() && end && *boundary {
431 self.done = true;
432 return Ok(DecodeStep {
433 consumed: 0,
434 written: 0,
435 finished: true,
436 needs_input: false,
437 });
438 }
439 let mut source = input;
440 if *reset {
441 if source.len() < 18 && !end {
442 return Ok(DecodeStep {
443 consumed: 0,
444 written: 0,
445 finished: false,
446 needs_input: false,
447 });
448 }
449 decoder.reset(&mut source).map_err(|_| corrupt())?;
450 consumed = input.len() - source.len();
451 *reset = false;
452 *boundary = false;
453 }
454 let result = decoder
455 .decode_from_to(source, output)
456 .map_err(|_| corrupt())?;
457 consumed += result.0;
458 written = result.1;
459 if decoder.is_finished() && decoder.can_collect() == 0 {
460 if let Some(expected) = decoder.get_checksum_from_data()
461 && decoder.get_calculated_checksum() != Some(expected)
462 {
463 return Err(corrupt());
464 }
465 *boundary = true;
466 *reset = true;
467 finished = end && consumed == input.len();
468 } else if end && consumed == 0 && written == 0 {
469 return Err(corrupt());
470 }
471 }
472 }
473 if written > limit.saturating_sub(self.total) {
474 return Err(Error::new(
475 "UND_ERR_RES_EXCEEDED_MAX_SIZE",
476 "decoded body exceeds limit",
477 ));
478 }
479 self.total += written;
480 self.done = finished;
481 Ok(DecodeStep {
482 consumed,
483 written,
484 finished,
485 needs_input: false,
486 })
487 }
488}
489impl Staged {
490 fn compact(&mut self) {
492 if self.start > 0 {
493 self.buffer.copy_within(self.start..self.end, 0);
494 self.end -= self.start;
495 self.start = 0;
496 }
497 }
498}
499impl StreamingDecoder {
500 pub fn new(encoding: &str, limit: usize) -> Result<Self> {
504 Self::from_codings([encoding], limit)
505 }
506 pub fn from_codings<I>(values: I, limit: usize) -> Result<Self>
524 where
525 I: IntoIterator,
526 I::Item: AsRef<[u8]>,
527 {
528 let mut codecs = Vec::new();
529 for value in values {
530 for token in value.as_ref().split(|b| *b == b',') {
531 let token = token.trim_ascii();
532 if token.is_empty() || token.eq_ignore_ascii_case(b"identity") {
533 continue;
534 }
535 if codecs.len() == MAX_CODINGS {
536 return Err(Error::new(
537 "UND_ERR_NOT_SUPPORTED",
538 "too many content codings",
539 ));
540 }
541 codecs.push(Codec::new(token)?);
542 }
543 }
544 let mut codecs = codecs.into_iter();
545 let codec = match codecs.next() {
546 Some(codec) => codec,
547 None => Codec::new(b"identity")?,
548 };
549 let outer = codecs
550 .rev()
551 .map(|codec| Staged {
552 codec,
553 buffer: vec![0; STAGE_BYTES].into_boxed_slice(),
554 start: 0,
555 end: 0,
556 })
557 .collect();
558 Ok(Self {
559 codec,
560 outer,
561 limit,
562 failed: false,
563 })
564 }
565 pub fn reset(&mut self, limit: usize) -> Result<()> {
568 self.codec.reset()?;
569 for stage in &mut self.outer {
570 stage.codec.reset()?;
571 stage.start = 0;
572 stage.end = 0;
573 }
574 self.limit = limit;
575 self.failed = false;
576 Ok(())
577 }
578 pub fn process(&mut self, input: &[u8], output: &mut [u8], end: bool) -> Result<DecodeStep> {
586 if self.failed {
587 return Err(corrupt());
588 }
589 let result = self.process_inner(input, output, end);
590 if result.is_err() {
591 self.failed = true;
592 }
593 result
594 }
595 fn process_inner(&mut self, input: &[u8], output: &mut [u8], end: bool) -> Result<DecodeStep> {
596 let limit = self.limit;
597 let Some((last, earlier)) = self.outer.split_last_mut() else {
598 return self.codec.run(input, output, end, limit);
599 };
600 if self.codec.done {
601 return self.codec.run(input, output, end, limit);
603 }
604 let mut consumed = 0;
605 let mut written = 0;
606 loop {
607 let mut progress = false;
608 for i in 0..=earlier.len() {
612 let (before, rest) = earlier.split_at_mut(i);
613 let stage = match rest.first_mut() {
614 Some(stage) => stage,
615 None => &mut *last,
616 };
617 stage.compact();
618 if stage.end == stage.buffer.len() {
619 continue;
620 }
621 let step = match before.last_mut() {
622 None => {
623 let step = stage.codec.run(
624 &input[consumed..],
625 &mut stage.buffer[stage.end..],
626 end,
627 limit,
628 )?;
629 consumed += step.consumed;
630 step
631 }
632 Some(feed) => {
633 let step = stage.codec.run(
634 &feed.buffer[feed.start..feed.end],
635 &mut stage.buffer[stage.end..],
636 feed.codec.done,
637 limit,
638 )?;
639 feed.start += step.consumed;
640 step
641 }
642 };
643 stage.end += step.written;
644 progress |= step.consumed > 0 || step.written > 0;
645 }
646 let step = self.codec.run(
648 &last.buffer[last.start..last.end],
649 &mut output[written..],
650 last.codec.done,
651 limit,
652 )?;
653 last.start += step.consumed;
654 written += step.written;
655 progress |= step.consumed > 0 || step.written > 0;
656 if step.finished || written == output.len() || !progress {
657 return Ok(DecodeStep {
658 consumed,
659 written,
660 finished: step.finished,
661 needs_input: !step.finished && written < output.len(),
662 });
663 }
664 }
665 }
666}
667fn gzip_header(input: &[u8]) -> Result<Option<usize>> {
668 if input.len() < 10 {
669 return Ok(None);
670 }
671 if input[..3] != [31, 139, 8] || input[3] & 0xe0 != 0 {
672 return Err(corrupt());
673 }
674 let flags = input[3];
675 let mut pos = 10;
676 if flags & 4 != 0 {
677 if input.len() < 12 {
678 return Ok(None);
679 }
680 pos = 12 + u16::from_le_bytes(input[10..12].try_into().unwrap()) as usize;
681 }
682 for flag in [8, 16] {
683 if flags & flag != 0 {
684 if pos > input.len() {
685 break;
686 }
687 if let Some(n) = input[pos..].iter().position(|b| *b == 0) {
688 pos += n + 1;
689 } else {
690 if input.len() > 8192 {
691 return Err(corrupt());
692 }
693 return Ok(None);
694 }
695 }
696 }
697 if pos > 8192 {
698 return Err(corrupt());
699 }
700 if input.len() < pos {
701 return Ok(None);
702 }
703 if flags & 2 != 0 {
704 if input.len() < pos + 2 {
705 return Ok(None);
706 }
707 let mut crc = flate2::Crc::new();
708 crc.update(&input[..pos]);
709 if crc.sum() as u16 != u16::from_le_bytes(input[pos..pos + 2].try_into().unwrap()) {
710 return Err(corrupt());
711 }
712 pos += 2;
713 }
714 Ok(Some(pos))
715}