compress_lib/gzip.rs
1//! gzip framing (RFC 1952): a header, raw DEFLATE, and a CRC-32 and length
2//! of the data; several members may follow one another.
3//!
4//! ```
5//! use compress_lib::gzip;
6//!
7//! let packed = gzip::compress(b"hello, hello, hello");
8//! assert_eq!(gzip::decompress(&packed)?, b"hello, hello, hello");
9//! # Ok::<(), compress_lib::Error>(())
10//! ```
11
12use alloc::vec::Vec;
13
14use crate::{
15 Error, ErrorKind, Level, Limits, Progress, Status,
16 checksum::Crc32,
17 deflate::{Deflater, Flush, Inflater},
18 zlib::Sink,
19};
20
21const ID1: u8 = 0x1F;
22const ID2: u8 = 0x8B;
23const FTEXT: u8 = 1;
24const FHCRC: u8 = 2;
25const FEXTRA: u8 = 4;
26const FNAME: u8 = 8;
27const FCOMMENT: u8 = 16;
28/// Operating system "unknown": the same bytes on every platform.
29const OS_UNKNOWN: u8 = 255;
30
31/// Compresses `data` into a gzip member at level 6, with a header that has
32/// no name, no time stamp, and OS "unknown" (the same bytes everywhere).
33///
34/// # Examples
35///
36/// ```
37/// let packed = compress_lib::gzip::compress(b"data");
38/// assert_eq!(&packed[..3], &[0x1F, 0x8B, 0x08]);
39/// ```
40#[must_use]
41pub fn compress(data: &[u8]) -> Vec<u8> {
42 compress_with(data, Level::DEFAULT)
43}
44
45/// Compresses `data` into a gzip member at `level`.
46///
47/// # Examples
48///
49/// ```
50/// use compress_lib::{Level, gzip};
51///
52/// let packed = gzip::compress_with(b"data", Level::FAST);
53/// assert_eq!(gzip::decompress(&packed)?, b"data");
54/// # Ok::<(), compress_lib::Error>(())
55/// ```
56#[must_use]
57pub fn compress_with(data: &[u8], level: Level) -> Vec<u8> {
58 let mut encoder = Encoder::new(level);
59 let mut out = Vec::with_capacity(data.len() / 2 + 64);
60 // Fails only when the vector cannot grow, which aborts first.
61 let _ = encoder.compress_vec(data, &mut out, Flush::Finish);
62 out
63}
64
65/// Decompresses a complete gzip stream (all members) with the default
66/// [`Limits`].
67///
68/// # Errors
69///
70/// Any decoding error; `InvalidHeader`, `ChecksumMismatch`,
71/// `LengthMismatch`, `HeaderTooLarge`, `TooManyMembers`; `UnexpectedEnd`,
72/// `TrailingData`, `OutputLimit` as for raw DEFLATE.
73///
74/// # Examples
75///
76/// ```
77/// use compress_lib::gzip;
78///
79/// let mut two = gzip::compress(b"first ");
80/// two.extend_from_slice(&gzip::compress(b"second"));
81/// assert_eq!(gzip::decompress(&two)?, b"first second");
82/// # Ok::<(), compress_lib::Error>(())
83/// ```
84pub fn decompress(data: &[u8]) -> Result<Vec<u8>, Error> {
85 decompress_with(data, Limits::new())
86}
87
88/// Decompresses a complete gzip stream with the given limits.
89///
90/// # Errors
91///
92/// As [`decompress`].
93///
94/// # Examples
95///
96/// ```
97/// use compress_lib::{ErrorKind, Limits, gzip};
98///
99/// let packed = gzip::compress(&[0u8; 10_000]);
100/// let tight = Limits::new().with_max_output(100);
101/// assert_eq!(gzip::decompress_with(&packed, tight).unwrap_err().kind(), ErrorKind::OutputLimit);
102/// ```
103pub fn decompress_with(data: &[u8], limits: Limits) -> Result<Vec<u8>, Error> {
104 let mut decoder = Decoder::one_shot(limits);
105 let mut out = Vec::new();
106 let reserve = data.len().saturating_mul(4).clamp(64, 1 << 20);
107 let cap = usize::try_from(limits.max_output()).unwrap_or(usize::MAX);
108 if out.try_reserve(reserve.min(cap)).is_err() {
109 return Err(Error::config(ErrorKind::AllocationFailed));
110 }
111 let p = decoder.finish_vec(data, &mut out)?;
112 if p.consumed != data.len() {
113 return Err(Error::new(ErrorKind::TrailingData, p.consumed as u64));
114 }
115 Ok(out)
116}
117
118/// The fields of a gzip header.
119///
120/// Decoders fill it only when asked ([`Decoder::with_keep_header`]); the
121/// optional fields are bounded by [`Limits::max_header`].
122///
123/// # Examples
124///
125/// ```
126/// use compress_lib::{Level, deflate::Flush, gzip::{self, GzipHeader}};
127///
128/// let header = GzipHeader::new().with_name(b"report.csv").with_mtime(1_700_000_000);
129/// let mut enc = gzip::Encoder::new(Level::DEFAULT).with_header(header);
130/// let mut packed = Vec::new();
131/// enc.compress_vec(b"a,b\n1,2\n", &mut packed, Flush::Finish)?;
132///
133/// let mut dec = gzip::Decoder::new().with_keep_header(true);
134/// let mut out = Vec::new();
135/// dec.finish_vec(&packed, &mut out)?;
136/// assert_eq!(dec.header().and_then(|h| h.name()), Some(&b"report.csv"[..]));
137/// # Ok::<(), compress_lib::Error>(())
138/// ```
139#[derive(Clone, Debug, Default, PartialEq, Eq)]
140#[non_exhaustive]
141pub struct GzipHeader {
142 mtime: u32,
143 os: u8,
144 extra_flags: u8,
145 text: bool,
146 name: Option<Vec<u8>>,
147 comment: Option<Vec<u8>>,
148 extra: Option<Vec<u8>>,
149}
150
151impl GzipHeader {
152 /// No name, comment, or extra field; time stamp 0; OS "unknown".
153 ///
154 /// # Examples
155 ///
156 /// ```
157 /// assert_eq!(compress_lib::gzip::GzipHeader::new().os(), 255);
158 /// ```
159 #[must_use]
160 pub fn new() -> Self {
161 Self {
162 os: OS_UNKNOWN,
163 ..Self::default()
164 }
165 }
166
167 /// Sets the file name (written without its terminating zero; it must
168 /// not contain one).
169 ///
170 /// # Examples
171 ///
172 /// ```
173 /// let h = compress_lib::gzip::GzipHeader::new().with_name(b"a.txt");
174 /// assert_eq!(h.name(), Some(&b"a.txt"[..]));
175 /// ```
176 #[must_use]
177 pub fn with_name(mut self, name: &[u8]) -> Self {
178 self.name = Some(name.iter().copied().take_while(|&b| b != 0).collect());
179 self
180 }
181
182 /// Sets the comment (it must not contain a zero byte).
183 ///
184 /// # Examples
185 ///
186 /// ```
187 /// let h = compress_lib::gzip::GzipHeader::new().with_comment(b"nightly export");
188 /// assert_eq!(h.comment(), Some(&b"nightly export"[..]));
189 /// ```
190 #[must_use]
191 pub fn with_comment(mut self, comment: &[u8]) -> Self {
192 self.comment = Some(comment.iter().copied().take_while(|&b| b != 0).collect());
193 self
194 }
195
196 /// Sets the extra field (at most 65,535 bytes; longer is cut).
197 ///
198 /// # Examples
199 ///
200 /// ```
201 /// let h = compress_lib::gzip::GzipHeader::new().with_extra(b"AB\x02\x00hi");
202 /// assert_eq!(h.extra().map(<[u8]>::len), Some(6));
203 /// ```
204 #[must_use]
205 pub fn with_extra(mut self, extra: &[u8]) -> Self {
206 self.extra = Some(extra[..extra.len().min(0xFFFF)].to_vec());
207 self
208 }
209
210 /// Sets the modification time (Unix seconds; 0 means none).
211 ///
212 /// # Examples
213 ///
214 /// ```
215 /// assert_eq!(compress_lib::gzip::GzipHeader::new().with_mtime(7).mtime(), 7);
216 /// ```
217 #[must_use]
218 pub fn with_mtime(mut self, mtime: u32) -> Self {
219 self.mtime = mtime;
220 self
221 }
222
223 /// Sets the operating system byte (RFC 1952 section 2.3.1).
224 ///
225 /// # Examples
226 ///
227 /// ```
228 /// assert_eq!(compress_lib::gzip::GzipHeader::new().with_os(3).os(), 3);
229 /// ```
230 #[must_use]
231 pub fn with_os(mut self, os: u8) -> Self {
232 self.os = os;
233 self
234 }
235
236 /// Marks the data as probably text (FTEXT).
237 ///
238 /// # Examples
239 ///
240 /// ```
241 /// assert!(compress_lib::gzip::GzipHeader::new().with_text(true).text());
242 /// ```
243 #[must_use]
244 pub fn with_text(mut self, text: bool) -> Self {
245 self.text = text;
246 self
247 }
248
249 /// The file name, if any.
250 ///
251 /// # Examples
252 ///
253 /// ```
254 /// assert_eq!(compress_lib::gzip::GzipHeader::new().name(), None);
255 /// ```
256 #[must_use]
257 pub fn name(&self) -> Option<&[u8]> {
258 self.name.as_deref()
259 }
260
261 /// The comment, if any.
262 ///
263 /// # Examples
264 ///
265 /// ```
266 /// assert_eq!(compress_lib::gzip::GzipHeader::new().comment(), None);
267 /// ```
268 #[must_use]
269 pub fn comment(&self) -> Option<&[u8]> {
270 self.comment.as_deref()
271 }
272
273 /// The extra field, if any.
274 ///
275 /// # Examples
276 ///
277 /// ```
278 /// assert_eq!(compress_lib::gzip::GzipHeader::new().extra(), None);
279 /// ```
280 #[must_use]
281 pub fn extra(&self) -> Option<&[u8]> {
282 self.extra.as_deref()
283 }
284
285 /// The modification time.
286 ///
287 /// # Examples
288 ///
289 /// ```
290 /// assert_eq!(compress_lib::gzip::GzipHeader::new().mtime(), 0);
291 /// ```
292 #[must_use]
293 pub fn mtime(&self) -> u32 {
294 self.mtime
295 }
296
297 /// The operating system byte.
298 ///
299 /// # Examples
300 ///
301 /// ```
302 /// assert_eq!(compress_lib::gzip::GzipHeader::new().os(), 255);
303 /// ```
304 #[must_use]
305 pub fn os(&self) -> u8 {
306 self.os
307 }
308
309 /// The extra flags byte (2: best compression, 4: fastest).
310 ///
311 /// # Examples
312 ///
313 /// ```
314 /// assert_eq!(compress_lib::gzip::GzipHeader::new().extra_flags(), 0);
315 /// ```
316 #[must_use]
317 pub fn extra_flags(&self) -> u8 {
318 self.extra_flags
319 }
320
321 /// Whether FTEXT is set.
322 ///
323 /// # Examples
324 ///
325 /// ```
326 /// assert!(!compress_lib::gzip::GzipHeader::new().text());
327 /// ```
328 #[must_use]
329 pub fn text(&self) -> bool {
330 self.text
331 }
332
333 /// The encoded header (with `extra_flags` from the level).
334 fn encode(&self, xfl: u8) -> Vec<u8> {
335 let mut flg = 0;
336 if self.text {
337 flg |= FTEXT;
338 }
339 if self.extra.is_some() {
340 flg |= FEXTRA;
341 }
342 if self.name.is_some() {
343 flg |= FNAME;
344 }
345 if self.comment.is_some() {
346 flg |= FCOMMENT;
347 }
348 let mut h = Vec::with_capacity(10);
349 h.extend_from_slice(&[ID1, ID2, 8, flg]);
350 h.extend_from_slice(&self.mtime.to_le_bytes());
351 h.extend_from_slice(&[xfl, self.os]);
352 if let Some(extra) = &self.extra {
353 h.extend_from_slice(&(extra.len() as u16).to_le_bytes());
354 h.extend_from_slice(extra);
355 }
356 for field in [&self.name, &self.comment].into_iter().flatten() {
357 h.extend_from_slice(field);
358 h.push(0);
359 }
360 h
361 }
362}
363
364#[derive(Clone, Copy, Debug, PartialEq, Eq)]
365enum State {
366 /// Fixed ten-byte header: `index` bytes read.
367 Fixed,
368 ExtraLength,
369 Extra,
370 Name,
371 Comment,
372 HeaderCrc,
373 Body,
374 Trailer,
375 /// A member ended; another may start.
376 MemberEnd,
377 Done,
378}
379
380/// A streaming gzip decoder.
381///
382/// Multi-member streams (RFC 1952 section 2.2) are decoded as one by
383/// default, up to [`Limits::max_members`]; the output cap applies to all
384/// members together. With more input possible after a member, `decompress`
385/// returns `NeedsInput`; `finish` decides that the stream ended.
386///
387/// # Examples
388///
389/// ```
390/// use compress_lib::{deflate::Status, gzip};
391///
392/// let packed = gzip::compress(b"streamed");
393/// let mut dec = gzip::Decoder::new();
394/// let mut out = [0u8; 32];
395/// let p = dec.finish(&packed, &mut out)?;
396/// assert_eq!(p.status, Status::Done);
397/// assert_eq!(&out[..p.produced], b"streamed");
398/// # Ok::<(), compress_lib::Error>(())
399/// ```
400#[derive(Debug)]
401pub struct Decoder {
402 inflater: Inflater,
403 limits: Limits,
404 state: State,
405 multi: bool,
406 allow_trailing: bool,
407 keep_header: bool,
408 header: Option<GzipHeader>,
409 /// Header bytes and fields in progress.
410 fixed: [u8; 10],
411 index: usize,
412 extra_left: usize,
413 header_crc: Crc32,
414 header_bytes: usize,
415 field: Vec<u8>,
416 crc: Crc32,
417 member_len: u32,
418 trailer: [u8; 8],
419 members: u32,
420 out_before: u64,
421 consumed_total: u64,
422 failed: Option<Error>,
423}
424
425impl Default for Decoder {
426 fn default() -> Self {
427 Self::new()
428 }
429}
430
431impl Decoder {
432 /// A decoder with window bits 15 and the default limits, multi-member,
433 /// trailing data refused, header fields not kept.
434 ///
435 /// # Examples
436 ///
437 /// ```
438 /// assert!(!compress_lib::gzip::Decoder::new().is_done());
439 /// ```
440 #[must_use]
441 pub fn new() -> Self {
442 Self::from_inflater(Inflater::new(), Limits::new())
443 }
444
445 pub(crate) fn one_shot(limits: Limits) -> Self {
446 Self::from_inflater(Inflater::without_history(limits), limits)
447 }
448
449 fn from_inflater(inflater: Inflater, limits: Limits) -> Self {
450 Self {
451 inflater,
452 limits,
453 state: State::Fixed,
454 multi: true,
455 allow_trailing: false,
456 keep_header: false,
457 header: None,
458 fixed: [0; 10],
459 index: 0,
460 extra_left: 0,
461 header_crc: Crc32::new(),
462 header_bytes: 0,
463 field: Vec::new(),
464 crc: Crc32::new(),
465 member_len: 0,
466 trailer: [0; 8],
467 members: 0,
468 out_before: 0,
469 consumed_total: 0,
470 failed: None,
471 }
472 }
473
474 /// Sets the limits: the output cap (across members), the ratio cap, the
475 /// header field cap, and the member cap.
476 ///
477 /// # Examples
478 ///
479 /// ```
480 /// use compress_lib::{Limits, gzip};
481 ///
482 /// let dec = gzip::Decoder::new().with_limits(Limits::new().with_max_members(1));
483 /// # let _ = dec;
484 /// ```
485 #[must_use]
486 pub fn with_limits(mut self, limits: Limits) -> Self {
487 self.inflater = self.inflater.with_limits(limits);
488 self.limits = limits;
489 self
490 }
491
492 /// Decodes one member only; `Done` at its end, and what follows is left
493 /// to the caller.
494 ///
495 /// # Examples
496 ///
497 /// ```
498 /// use compress_lib::{deflate::Status, gzip};
499 ///
500 /// let mut two = gzip::compress(b"first");
501 /// let first_len = two.len();
502 /// two.extend_from_slice(&gzip::compress(b"second"));
503 /// let mut dec = gzip::Decoder::new().with_single_member();
504 /// let mut out = Vec::new();
505 /// let p = dec.decompress_vec(&two, &mut out)?;
506 /// assert_eq!((p.status, p.consumed), (Status::Done, first_len));
507 /// # Ok::<(), compress_lib::Error>(())
508 /// ```
509 #[must_use]
510 pub fn with_single_member(mut self) -> Self {
511 self.multi = false;
512 self
513 }
514
515 /// Allows bytes that are not a gzip member after the last member (some
516 /// tools pad files with zeros); `consumed` stops before them.
517 ///
518 /// # Examples
519 ///
520 /// ```
521 /// use compress_lib::gzip;
522 ///
523 /// let mut padded = gzip::compress(b"data");
524 /// padded.extend_from_slice(&[0; 8]);
525 /// let mut dec = gzip::Decoder::new().with_trailing_data(true);
526 /// let mut out = Vec::new();
527 /// dec.finish_vec(&padded, &mut out)?;
528 /// assert_eq!(out, b"data");
529 /// # Ok::<(), compress_lib::Error>(())
530 /// ```
531 #[must_use]
532 pub fn with_trailing_data(mut self, allow: bool) -> Self {
533 self.allow_trailing = allow;
534 self
535 }
536
537 /// Keeps the header fields (name, comment, extra) for [`header`]
538 /// (bounded by [`Limits::max_header`]); off by default, so no memory is
539 /// spent on them.
540 ///
541 /// [`header`]: Decoder::header
542 ///
543 /// # Examples
544 ///
545 /// ```
546 /// let dec = compress_lib::gzip::Decoder::new().with_keep_header(true);
547 /// assert!(dec.header().is_none());
548 /// ```
549 #[must_use]
550 pub fn with_keep_header(mut self, keep: bool) -> Self {
551 self.keep_header = keep;
552 self
553 }
554
555 /// The first member's header, once read, when headers are kept.
556 ///
557 /// # Examples
558 ///
559 /// ```
560 /// use compress_lib::gzip;
561 ///
562 /// let mut dec = gzip::Decoder::new().with_keep_header(true);
563 /// dec.finish_vec(&gzip::compress(b"x"), &mut Vec::new())?;
564 /// assert_eq!(dec.header().map(|h| h.os()), Some(255));
565 /// # Ok::<(), compress_lib::Error>(())
566 /// ```
567 #[must_use]
568 pub fn header(&self) -> Option<&GzipHeader> {
569 self.header.as_ref()
570 }
571
572 /// `true` once the stream ended.
573 ///
574 /// # Examples
575 ///
576 /// ```
577 /// use compress_lib::gzip;
578 ///
579 /// let mut dec = gzip::Decoder::new();
580 /// dec.finish_vec(&gzip::compress(b""), &mut Vec::new())?;
581 /// assert!(dec.is_done());
582 /// # Ok::<(), compress_lib::Error>(())
583 /// ```
584 #[must_use]
585 pub fn is_done(&self) -> bool {
586 self.state == State::Done
587 }
588
589 /// Members completed so far.
590 ///
591 /// # Examples
592 ///
593 /// ```
594 /// use compress_lib::gzip;
595 ///
596 /// let mut two = gzip::compress(b"a");
597 /// two.extend_from_slice(&gzip::compress(b"b"));
598 /// let mut dec = gzip::Decoder::new();
599 /// dec.finish_vec(&two, &mut Vec::new())?;
600 /// assert_eq!(dec.members(), 2);
601 /// # Ok::<(), compress_lib::Error>(())
602 /// ```
603 #[must_use]
604 pub fn members(&self) -> u32 {
605 self.members
606 }
607
608 /// Compressed bytes consumed.
609 ///
610 /// # Examples
611 ///
612 /// ```
613 /// use compress_lib::gzip;
614 ///
615 /// let packed = gzip::compress(b"x");
616 /// let mut dec = gzip::Decoder::new();
617 /// dec.finish_vec(&packed, &mut Vec::new())?;
618 /// assert_eq!(dec.total_in(), packed.len() as u64);
619 /// # Ok::<(), compress_lib::Error>(())
620 /// ```
621 #[must_use]
622 pub fn total_in(&self) -> u64 {
623 self.consumed_total
624 }
625
626 /// Decompressed bytes produced, all members.
627 ///
628 /// # Examples
629 ///
630 /// ```
631 /// use compress_lib::gzip;
632 ///
633 /// let mut dec = gzip::Decoder::new();
634 /// dec.finish_vec(&gzip::compress(b"four"), &mut Vec::new())?;
635 /// assert_eq!(dec.total_out(), 4);
636 /// # Ok::<(), compress_lib::Error>(())
637 /// ```
638 #[must_use]
639 pub fn total_out(&self) -> u64 {
640 self.out_before + self.inflater.total_out()
641 }
642
643 /// Heap bytes held.
644 ///
645 /// # Examples
646 ///
647 /// ```
648 /// assert!(compress_lib::gzip::Decoder::new().memory_in_use() > 32 * 1024);
649 /// ```
650 #[must_use]
651 pub fn memory_in_use(&self) -> usize {
652 self.inflater.memory_in_use() + self.field.capacity()
653 }
654
655 /// Starts over for a new stream, keeping allocations.
656 ///
657 /// # Examples
658 ///
659 /// ```
660 /// let mut dec = compress_lib::gzip::Decoder::new();
661 /// dec.reset();
662 /// ```
663 pub fn reset(&mut self) {
664 self.inflater.reset();
665 self.inflater.set_max_output(self.limits.max_output());
666 self.state = State::Fixed;
667 self.index = 0;
668 self.header = None;
669 self.field.clear();
670 self.crc = Crc32::new();
671 self.header_crc = Crc32::new();
672 self.header_bytes = 0;
673 self.members = 0;
674 self.out_before = 0;
675 self.consumed_total = 0;
676 self.failed = None;
677 }
678
679 /// Decompresses as much of `input` into `output` as fits; more input may
680 /// follow (and after a member, perhaps another member).
681 ///
682 /// # Errors
683 ///
684 /// Decoding errors, the header and trailer checks, and the limits.
685 ///
686 /// # Examples
687 ///
688 /// ```
689 /// use compress_lib::{deflate::Status, gzip};
690 ///
691 /// let packed = gzip::compress(b"split");
692 /// let mut dec = gzip::Decoder::new();
693 /// let mut out = [0u8; 16];
694 /// let a = dec.decompress(&packed[..5], &mut out)?;
695 /// assert_eq!(a.status, Status::NeedsInput);
696 /// let b = dec.finish(&packed[5..], &mut out[a.produced..])?;
697 /// assert_eq!(b.status, Status::Done);
698 /// # Ok::<(), compress_lib::Error>(())
699 /// ```
700 pub fn decompress(&mut self, input: &[u8], output: &mut [u8]) -> Result<Progress, Error> {
701 self.run(input, Sink::Slice(output), false)
702 }
703
704 /// As [`decompress`](Decoder::decompress), with `input` the last of the
705 /// stream.
706 ///
707 /// # Errors
708 ///
709 /// As `decompress`, plus `UnexpectedEnd`.
710 ///
711 /// # Examples
712 ///
713 /// ```
714 /// use compress_lib::{ErrorKind, gzip};
715 ///
716 /// let packed = gzip::compress(b"data");
717 /// let err = gzip::Decoder::new().finish(&packed[..packed.len() - 1], &mut [0u8; 8]).unwrap_err();
718 /// assert_eq!(err.kind(), ErrorKind::UnexpectedEnd);
719 /// ```
720 pub fn finish(&mut self, input: &[u8], output: &mut [u8]) -> Result<Progress, Error> {
721 self.run(input, Sink::Slice(output), true)
722 }
723
724 /// Decompresses `input`, appending to `out`.
725 ///
726 /// # Errors
727 ///
728 /// As [`decompress`](Decoder::decompress), plus `AllocationFailed`.
729 ///
730 /// # Examples
731 ///
732 /// ```
733 /// use compress_lib::gzip;
734 ///
735 /// let mut out = Vec::new();
736 /// gzip::Decoder::new().decompress_vec(&gzip::compress(b"vec"), &mut out)?;
737 /// assert_eq!(out, b"vec");
738 /// # Ok::<(), compress_lib::Error>(())
739 /// ```
740 pub fn decompress_vec(&mut self, input: &[u8], out: &mut Vec<u8>) -> Result<Progress, Error> {
741 self.run(input, Sink::Vec(out), false)
742 }
743
744 /// As [`decompress_vec`](Decoder::decompress_vec), with `input` the last
745 /// of the stream.
746 ///
747 /// # Errors
748 ///
749 /// As `decompress_vec`, plus `UnexpectedEnd`.
750 ///
751 /// # Examples
752 ///
753 /// ```
754 /// use compress_lib::gzip;
755 ///
756 /// let mut out = Vec::new();
757 /// gzip::Decoder::new().finish_vec(&gzip::compress(b"vec"), &mut out)?;
758 /// assert_eq!(out, b"vec");
759 /// # Ok::<(), compress_lib::Error>(())
760 /// ```
761 pub fn finish_vec(&mut self, input: &[u8], out: &mut Vec<u8>) -> Result<Progress, Error> {
762 self.run(input, Sink::Vec(out), true)
763 }
764
765 fn fail(&mut self, kind: ErrorKind, at: usize) -> Error {
766 let e = Error::new(kind, self.consumed_total + at as u64);
767 self.failed = Some(e);
768 e
769 }
770
771 /// Counts one byte of an optional header field against the cap.
772 fn header_byte(&mut self, b: u8) -> Result<(), ErrorKind> {
773 self.header_crc.update(&[b]);
774 self.header_bytes += 1;
775 if self.header_bytes > self.limits.max_header() {
776 return Err(ErrorKind::HeaderTooLarge);
777 }
778 if self.keep_header && self.members == 0 {
779 if self.field.try_reserve(1).is_err() {
780 return Err(ErrorKind::AllocationFailed);
781 }
782 self.field.push(b);
783 }
784 Ok(())
785 }
786
787 fn keep_field(&mut self, which: State) {
788 if !(self.keep_header && self.members == 0) {
789 return;
790 }
791 let value = core::mem::take(&mut self.field);
792 if let Some(h) = &mut self.header {
793 match which {
794 State::Extra => h.extra = Some(value),
795 State::Name => h.name = Some(value),
796 _ => h.comment = Some(value),
797 }
798 }
799 }
800
801 /// The header state after `from`, by the flags.
802 fn next_header_state(&self, from: State) -> State {
803 let flg = self.fixed[3];
804 let order = [
805 (State::ExtraLength, FEXTRA),
806 (State::Name, FNAME),
807 (State::Comment, FCOMMENT),
808 (State::HeaderCrc, FHCRC),
809 ];
810 let start = match from {
811 State::Fixed => 0,
812 State::Extra | State::ExtraLength => 1,
813 State::Name => 2,
814 State::Comment => 3,
815 _ => 4,
816 };
817 for &(state, bit) in &order[start..] {
818 if flg & bit != 0 {
819 return state;
820 }
821 }
822 State::Body
823 }
824
825 fn run(&mut self, input: &[u8], mut sink: Sink<'_>, last: bool) -> Result<Progress, Error> {
826 if let Some(e) = self.failed {
827 return Err(e);
828 }
829 let mut ip = 0;
830 let mut produced = 0;
831 let status = loop {
832 match self.state {
833 State::Fixed => {
834 while self.index < 10 && ip < input.len() {
835 self.fixed[self.index] = input[ip];
836 self.index += 1;
837 ip += 1;
838 // Reject a wrong magic number at once (also for the
839 // trailing-data check after a member).
840 if self.index == 2 && self.fixed[..2] != [ID1, ID2] {
841 return Err(self.fail(ErrorKind::InvalidHeader, ip));
842 }
843 }
844 if self.index < 10 {
845 break Status::NeedsInput;
846 }
847 let f = self.fixed;
848 if f[2] != 8 || f[3] & 0xE0 != 0 {
849 return Err(self.fail(ErrorKind::InvalidHeader, ip));
850 }
851 self.header_crc = Crc32::new();
852 self.header_crc.update(&f);
853 self.header_bytes = 0;
854 if self.keep_header && self.members == 0 {
855 self.header = Some(GzipHeader {
856 mtime: u32::from_le_bytes([f[4], f[5], f[6], f[7]]),
857 extra_flags: f[8],
858 os: f[9],
859 text: f[3] & FTEXT != 0,
860 name: None,
861 comment: None,
862 extra: None,
863 });
864 }
865 self.index = 0;
866 self.state = self.next_header_state(State::Fixed);
867 }
868 State::ExtraLength => {
869 while self.index < 2 && ip < input.len() {
870 self.trailer[self.index] = input[ip];
871 self.index += 1;
872 ip += 1;
873 }
874 if self.index < 2 {
875 break Status::NeedsInput;
876 }
877 let xlen = [self.trailer[0], self.trailer[1]];
878 self.header_crc.update(&xlen);
879 self.extra_left = usize::from(u16::from_le_bytes(xlen));
880 self.index = 0;
881 self.state = State::Extra;
882 }
883 State::Extra => {
884 while self.extra_left > 0 && ip < input.len() {
885 let b = input[ip];
886 ip += 1;
887 self.extra_left -= 1;
888 if let Err(kind) = self.header_byte(b) {
889 return Err(self.fail(kind, ip));
890 }
891 }
892 if self.extra_left > 0 {
893 break Status::NeedsInput;
894 }
895 self.keep_field(State::Extra);
896 self.state = self.next_header_state(State::Extra);
897 }
898 State::Name | State::Comment => {
899 let which = self.state;
900 let mut ended = false;
901 while ip < input.len() {
902 let b = input[ip];
903 ip += 1;
904 if b == 0 {
905 self.header_crc.update(&[0]);
906 ended = true;
907 break;
908 }
909 if let Err(kind) = self.header_byte(b) {
910 return Err(self.fail(kind, ip));
911 }
912 }
913 if !ended {
914 break Status::NeedsInput;
915 }
916 self.keep_field(which);
917 self.state = self.next_header_state(which);
918 }
919 State::HeaderCrc => {
920 while self.index < 2 && ip < input.len() {
921 self.trailer[self.index] = input[ip];
922 self.index += 1;
923 ip += 1;
924 }
925 if self.index < 2 {
926 break Status::NeedsInput;
927 }
928 self.index = 0;
929 let want = u16::from_le_bytes([self.trailer[0], self.trailer[1]]);
930 if want != (self.header_crc.finish() & 0xFFFF) as u16 {
931 return Err(self.fail(ErrorKind::ChecksumMismatch, ip));
932 }
933 self.state = State::Body;
934 }
935 State::Body => {
936 let r = match &mut sink {
937 Sink::Slice(out) => self
938 .inflater
939 .decompress(&input[ip..], &mut out[produced..])
940 .inspect(|p| {
941 self.crc.update(&out[produced..produced + p.produced]);
942 }),
943 Sink::Vec(out) => {
944 let at = out.len();
945 self.inflater
946 .decompress_vec(&input[ip..], out)
947 .inspect(|_| {
948 self.crc.update(&out[at..]);
949 })
950 }
951 };
952 let p = match r {
953 Ok(p) => p,
954 Err(e) => {
955 self.failed = Some(e);
956 return Err(e);
957 }
958 };
959 ip += p.consumed;
960 produced += p.produced;
961 self.member_len = self.member_len.wrapping_add(p.produced as u32);
962 match p.status {
963 Status::Done => {
964 self.index = 0;
965 self.state = State::Trailer;
966 }
967 Status::NeedsOutput => break Status::NeedsOutput,
968 _ => break Status::NeedsInput,
969 }
970 }
971 State::Trailer => {
972 while self.index < 8 && ip < input.len() {
973 self.trailer[self.index] = input[ip];
974 self.index += 1;
975 ip += 1;
976 }
977 if self.index < 8 {
978 break Status::NeedsInput;
979 }
980 let t = self.trailer;
981 if u32::from_le_bytes([t[0], t[1], t[2], t[3]]) != self.crc.finish() {
982 return Err(self.fail(ErrorKind::ChecksumMismatch, ip));
983 }
984 if u32::from_le_bytes([t[4], t[5], t[6], t[7]]) != self.member_len {
985 return Err(self.fail(ErrorKind::LengthMismatch, ip));
986 }
987 self.members += 1;
988 self.state = if self.multi {
989 State::MemberEnd
990 } else {
991 State::Done
992 };
993 }
994 State::MemberEnd => {
995 if ip == input.len() {
996 if last {
997 self.state = State::Done;
998 break Status::Done;
999 }
1000 break Status::NeedsInput;
1001 }
1002 if input[ip] != ID1 || (ip + 1 < input.len() && input[ip + 1] != ID2) {
1003 if self.allow_trailing {
1004 self.state = State::Done;
1005 break Status::Done;
1006 }
1007 return Err(self.fail(ErrorKind::TrailingData, ip));
1008 }
1009 if self.members >= self.limits.max_members() {
1010 return Err(self.fail(ErrorKind::TooManyMembers, ip));
1011 }
1012 // A new member: a new DEFLATE stream (no history across
1013 // members), with the output cap shared by all.
1014 self.out_before += self.inflater.total_out();
1015 self.inflater.reset();
1016 self.inflater
1017 .set_max_output(self.limits.max_output().saturating_sub(self.out_before));
1018 self.crc = Crc32::new();
1019 self.member_len = 0;
1020 self.index = 0;
1021 self.state = State::Fixed;
1022 }
1023 State::Done => break Status::Done,
1024 }
1025 };
1026 self.consumed_total += ip as u64;
1027 if status == Status::NeedsInput && last {
1028 // An empty stream, or one cut inside a member.
1029 return Err(self.fail(ErrorKind::UnexpectedEnd, 0));
1030 }
1031 Ok(Progress::new(ip, produced, status))
1032 }
1033}
1034
1035/// A streaming gzip encoder (one member).
1036///
1037/// # Examples
1038///
1039/// ```
1040/// use compress_lib::{Level, deflate::Flush, gzip};
1041///
1042/// let mut enc = gzip::Encoder::new(Level::DEFAULT);
1043/// let mut packed = Vec::new();
1044/// enc.compress_vec(b"part one, ", &mut packed, Flush::None)?;
1045/// enc.compress_vec(b"part two", &mut packed, Flush::Finish)?;
1046/// assert_eq!(gzip::decompress(&packed)?, b"part one, part two");
1047/// # Ok::<(), compress_lib::Error>(())
1048/// ```
1049#[derive(Debug)]
1050pub struct Encoder {
1051 deflater: Deflater,
1052 header: GzipHeader,
1053 crc: Crc32,
1054 pending: Vec<u8>,
1055 pending_pos: usize,
1056 header_written: bool,
1057 trailer_written: bool,
1058}
1059
1060impl Default for Encoder {
1061 fn default() -> Self {
1062 Self::new(Level::DEFAULT)
1063 }
1064}
1065
1066impl Encoder {
1067 /// An encoder at `level` with the plain header of [`GzipHeader::new`].
1068 ///
1069 /// # Examples
1070 ///
1071 /// ```
1072 /// let enc = compress_lib::gzip::Encoder::new(compress_lib::Level::BEST);
1073 /// # let _ = enc;
1074 /// ```
1075 #[must_use]
1076 pub fn new(level: Level) -> Self {
1077 Self {
1078 deflater: Deflater::new(level),
1079 header: GzipHeader::new(),
1080 crc: Crc32::new(),
1081 pending: Vec::new(),
1082 pending_pos: 0,
1083 header_written: false,
1084 trailer_written: false,
1085 }
1086 }
1087
1088 /// Writes `header` instead of the plain one.
1089 ///
1090 /// # Examples
1091 ///
1092 /// ```
1093 /// use compress_lib::{Level, gzip::{Encoder, GzipHeader}};
1094 ///
1095 /// let enc = Encoder::new(Level::DEFAULT).with_header(GzipHeader::new().with_name(b"a.txt"));
1096 /// # let _ = enc;
1097 /// ```
1098 #[must_use]
1099 pub fn with_header(mut self, header: GzipHeader) -> Self {
1100 self.header = header;
1101 self
1102 }
1103
1104 /// Sets the window to `2^bits` bytes, 8 to 15.
1105 ///
1106 /// # Errors
1107 ///
1108 /// [`ErrorKind::InvalidWindowBits`] outside 8 to 15.
1109 ///
1110 /// # Examples
1111 ///
1112 /// ```
1113 /// let enc = compress_lib::gzip::Encoder::default().with_window_bits(12)?;
1114 /// # let _ = enc;
1115 /// # Ok::<(), compress_lib::Error>(())
1116 /// ```
1117 pub fn with_window_bits(mut self, bits: u8) -> Result<Self, Error> {
1118 self.deflater = self.deflater.with_window_bits(bits)?;
1119 Ok(self)
1120 }
1121
1122 /// Starts a new member, keeping allocations and the header.
1123 ///
1124 /// # Examples
1125 ///
1126 /// ```
1127 /// let mut enc = compress_lib::gzip::Encoder::default();
1128 /// enc.reset();
1129 /// ```
1130 pub fn reset(&mut self) {
1131 self.deflater.reset();
1132 self.crc = Crc32::new();
1133 self.pending.clear();
1134 self.pending_pos = 0;
1135 self.header_written = false;
1136 self.trailer_written = false;
1137 }
1138
1139 /// Uncompressed bytes taken.
1140 ///
1141 /// # Examples
1142 ///
1143 /// ```
1144 /// use compress_lib::{deflate::Flush, gzip};
1145 ///
1146 /// let mut enc = gzip::Encoder::default();
1147 /// enc.compress_vec(b"abc", &mut Vec::new(), Flush::Finish)?;
1148 /// assert_eq!(enc.total_in(), 3);
1149 /// # Ok::<(), compress_lib::Error>(())
1150 /// ```
1151 #[must_use]
1152 pub fn total_in(&self) -> u64 {
1153 self.deflater.total_in()
1154 }
1155
1156 /// Heap bytes held.
1157 ///
1158 /// # Examples
1159 ///
1160 /// ```
1161 /// assert!(compress_lib::gzip::Encoder::default().memory_in_use() > 0);
1162 /// ```
1163 #[must_use]
1164 pub fn memory_in_use(&self) -> usize {
1165 self.deflater.memory_in_use() + self.pending.capacity()
1166 }
1167
1168 fn drain(&mut self, output: &mut [u8], produced: &mut usize) -> usize {
1169 let n = (self.pending.len() - self.pending_pos).min(output.len() - *produced);
1170 output[*produced..*produced + n]
1171 .copy_from_slice(&self.pending[self.pending_pos..self.pending_pos + n]);
1172 self.pending_pos += n;
1173 *produced += n;
1174 self.pending.len() - self.pending_pos
1175 }
1176
1177 /// As [`Deflater::compress`], with the header written first and the
1178 /// CRC-32 and length after `Finish`.
1179 ///
1180 /// # Errors
1181 ///
1182 /// As `Deflater::compress`.
1183 ///
1184 /// # Examples
1185 ///
1186 /// ```
1187 /// use compress_lib::{deflate::{Flush, Status}, gzip};
1188 ///
1189 /// let mut enc = gzip::Encoder::default();
1190 /// let mut out = [0u8; 64];
1191 /// let p = enc.compress(b"abc", &mut out, Flush::Finish)?;
1192 /// assert_eq!(p.status, Status::Done);
1193 /// assert_eq!(gzip::decompress(&out[..p.produced])?, b"abc");
1194 /// # Ok::<(), compress_lib::Error>(())
1195 /// ```
1196 pub fn compress(
1197 &mut self,
1198 input: &[u8],
1199 output: &mut [u8],
1200 flush: Flush,
1201 ) -> Result<Progress, Error> {
1202 let mut produced = 0;
1203 if !self.header_written {
1204 let xfl = match self.deflater.level().get() {
1205 9 => 2,
1206 1 => 4,
1207 _ => 0,
1208 };
1209 self.pending = self.header.encode(xfl);
1210 self.pending_pos = 0;
1211 self.header_written = true;
1212 }
1213 if self.drain(output, &mut produced) > 0 {
1214 return Ok(Progress::new(0, produced, Status::NeedsOutput));
1215 }
1216 if self.trailer_written {
1217 if !input.is_empty() {
1218 return Err(Error::config(ErrorKind::InvalidState));
1219 }
1220 return Ok(Progress::new(0, produced, Status::Done));
1221 }
1222 let p = self
1223 .deflater
1224 .compress(input, &mut output[produced..], flush)?;
1225 self.crc.update(&input[..p.consumed]);
1226 produced += p.produced;
1227 if p.status == Status::Done && flush == Flush::Finish {
1228 self.pending.clear();
1229 self.pending
1230 .extend_from_slice(&self.crc.finish().to_le_bytes());
1231 self.pending
1232 .extend_from_slice(&(self.crc.len() as u32).to_le_bytes());
1233 self.pending_pos = 0;
1234 self.trailer_written = true;
1235 let left = self.drain(output, &mut produced);
1236 let status = if left == 0 {
1237 Status::Done
1238 } else {
1239 Status::NeedsOutput
1240 };
1241 return Ok(Progress::new(p.consumed, produced, status));
1242 }
1243 Ok(Progress::new(p.consumed, produced, p.status))
1244 }
1245
1246 /// As [`compress`](Encoder::compress), appending to `out` until the
1247 /// flush point is written.
1248 ///
1249 /// # Errors
1250 ///
1251 /// As `compress`, plus `AllocationFailed`.
1252 ///
1253 /// # Examples
1254 ///
1255 /// ```
1256 /// use compress_lib::{deflate::Flush, gzip};
1257 ///
1258 /// let mut out = Vec::new();
1259 /// gzip::Encoder::default().compress_vec(b"abc", &mut out, Flush::Finish)?;
1260 /// assert_eq!(gzip::decompress(&out)?, b"abc");
1261 /// # Ok::<(), compress_lib::Error>(())
1262 /// ```
1263 pub fn compress_vec(
1264 &mut self,
1265 input: &[u8],
1266 out: &mut Vec<u8>,
1267 flush: Flush,
1268 ) -> Result<Progress, Error> {
1269 let start = out.len();
1270 let mut consumed = 0;
1271 loop {
1272 let have = out.len();
1273 let room = (input.len() - consumed) / 2 + 1024;
1274 if out.capacity() - have < room && out.try_reserve(room).is_err() {
1275 return Err(Error::config(ErrorKind::AllocationFailed));
1276 }
1277 // Zero-fill only this step's room, not all spare capacity: the
1278 // spare grows with the output, so filling it on every call
1279 // would make many small calls quadratic (T-5).
1280 out.resize(have + room, 0);
1281 match self.compress(&input[consumed..], &mut out[have..], flush) {
1282 Ok(p) => {
1283 out.truncate(have + p.produced);
1284 consumed += p.consumed;
1285 if p.status != Status::NeedsOutput {
1286 return Ok(Progress::new(consumed, out.len() - start, p.status));
1287 }
1288 }
1289 Err(e) => {
1290 out.truncate(have);
1291 return Err(e);
1292 }
1293 }
1294 }
1295 }
1296}