Skip to main content

openreadout_core/
gzip.rs

1//! Random access into gzip-compressed files (`.mzML.gz`, `.mzXML.gz`, ...).
2//!
3//! gzip (RFC 1952) wraps DEFLATE (RFC 1951), which can only be decoded from the start. Readers
4//! of large XML files need random access (an mzML's offset index points into the *decompressed*
5//! bytes), so [`GzipSource`] decompresses the file once when it is opened and records
6//! **checkpoints** on the way: at DEFLATE block boundaries roughly every megabyte of output it
7//! keeps the decoder's bit position and the last 32 KiB of output (the most a back-reference can
8//! reach). A later read starts from the nearest checkpoint at or before the wanted offset
9//! instead of from the beginning, and a read that follows the previous one continues the live
10//! decoder, so reading a file front to back costs one decompression. The same technique is
11//! used by zlib's `zran.c` example and by indexed-gzip tools.
12//!
13//! Costs: opening decompresses the whole file once (hundreds of MB/s); memory is at most
14//! [`MAX_CHECKPOINTS`] × 32 KiB (the spacing grows with the file to stay under it) plus a
15//! 1 MiB buffer for each of up to four live decoders. Every member's CRC-32 and length are
16//! verified during that pass.
17//! Concatenated members (`cat a.gz b.gz`, bgzip) are one stream, as `gzip -d` treats them.
18//! A truncated or damaged file keeps the bytes that decoded cleanly readable and says why in
19//! [`GzipSource::problem`].
20
21use std::io;
22use std::sync::{Arc, Mutex, PoisonError};
23
24use miniz_oxide::inflate::TINFLStatus;
25use miniz_oxide::inflate::core::inflate_flags::{
26    TINFL_FLAG_HAS_MORE_INPUT, TINFL_FLAG_STOP_ON_BLOCK_BOUNDARY,
27    TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF,
28};
29use miniz_oxide::inflate::core::{BlockBoundaryState, DecompressorOxide, decompress};
30
31use crate::error::{Error, Result};
32use crate::source::ByteSource;
33
34/// Most checkpoints kept; beyond it every other one is dropped and the spacing doubles.
35pub const MAX_CHECKPOINTS: usize = 1024;
36/// Output between two checkpoints to begin with (bytes).
37const FIRST_SPACING: u64 = 1 << 20;
38/// Largest back-reference distance of DEFLATE.
39const WINDOW: usize = 32 * 1024;
40/// Wrapping output buffer (a power of two, larger than the window): recent output stays in it,
41/// so a read slightly behind the previous one is served without decoding again.
42const WRAP: usize = 1 << 20;
43/// Compressed bytes read at a time.
44const IN_CHUNK: usize = 256 * 1024;
45/// Live decoders kept (least recently used is replaced).
46const LIVE_DECODERS: usize = 4;
47
48/// Live decoders, each with the tick of its last use.
49#[derive(Default)]
50struct Pool {
51    tick: u64,
52    streams: Vec<(u64, Stream)>,
53}
54
55/// True when `head` starts like a gzip member (magic `1F 8B`, method 8 = deflate).
56pub fn is_gzip(head: &[u8]) -> bool {
57    head.len() >= 3 && head[0] == 0x1f && head[1] == 0x8b && head[2] == 8
58}
59
60/// The first up to `max_out` decompressed bytes of a gzip file whose first bytes are `head`
61/// (for format detection from a sniffed head). `None` when `head` is not gzip or its header
62/// is incomplete; a truncated or damaged stream yields whatever decoded before the problem.
63pub fn peek(head: &[u8], max_out: usize) -> Option<Vec<u8>> {
64    let hdr = parse_header(head).ok()??;
65    let data = head.get(hdr.len..)?;
66    let mut r = DecompressorOxide::new();
67    let mut out = vec![0u8; max_out];
68    let (_, _, n) = decompress(
69        &mut r,
70        data,
71        &mut out,
72        0,
73        TINFL_FLAG_HAS_MORE_INPUT | TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF,
74    );
75    out.truncate(n);
76    Some(out)
77}
78
79/// A parsed gzip member header.
80#[derive(Debug, Clone, Default, PartialEq, Eq)]
81pub struct GzipHeader {
82    /// Header length in bytes (where the DEFLATE data starts).
83    pub len: usize,
84    /// `FNAME`: the original file name, when recorded.
85    pub name: Option<String>,
86    /// `MTIME`: modification time of the original (Unix seconds; 0 = not recorded).
87    pub mtime: u32,
88    /// `FCOMMENT`, when recorded.
89    pub comment: Option<String>,
90}
91
92/// Parse a member header at the start of `b`. `Ok(None)`: `b` ends inside the header.
93fn parse_header(b: &[u8]) -> std::result::Result<Option<GzipHeader>, String> {
94    if b.len() < 10 {
95        return Ok(None);
96    }
97    if b[0] != 0x1f || b[1] != 0x8b {
98        return Err("not a gzip member (no 1F 8B signature)".into());
99    }
100    if b[2] != 8 {
101        return Err(format!(
102            "gzip compression method {} (only 8, deflate, exists)",
103            b[2]
104        ));
105    }
106    let flg = b[3];
107    if flg & 0xE0 != 0 {
108        return Err(format!("gzip header flags {flg:#04x} set reserved bits"));
109    }
110    let mtime = u32::from_le_bytes([b[4], b[5], b[6], b[7]]);
111    let mut at = 10usize;
112    if flg & 0x04 != 0 {
113        let Some(x) = b.get(at..at + 2) else {
114            return Ok(None);
115        };
116        at += 2 + usize::from(u16::from_le_bytes([x[0], x[1]]));
117    }
118    let zstring = |at: &mut usize| -> Option<String> {
119        let rest = b.get(*at..)?;
120        let end = rest.iter().position(|&c| c == 0)?;
121        let s = String::from_utf8_lossy(&rest[..end]).into_owned();
122        *at += end + 1;
123        Some(s)
124    };
125    let mut name = None;
126    if flg & 0x08 != 0 {
127        match zstring(&mut at) {
128            Some(s) => name = Some(s),
129            None => return Ok(None),
130        }
131    }
132    let mut comment = None;
133    if flg & 0x10 != 0 {
134        match zstring(&mut at) {
135            Some(s) => comment = Some(s),
136            None => return Ok(None),
137        }
138    }
139    if flg & 0x02 != 0 {
140        at += 2;
141    }
142    if at > b.len() {
143        return Ok(None);
144    }
145    Ok(Some(GzipHeader {
146        len: at,
147        name,
148        mtime,
149        comment,
150    }))
151}
152
153/// Where decoding can restart.
154#[derive(Clone)]
155struct Checkpoint {
156    /// Decompressed offset.
157    out: u64,
158    /// Compressed offset of the next byte to feed the decoder.
159    input: u64,
160    /// Decoder state between blocks; `None` at the start of a member (fresh decoder).
161    state: Option<BlockBoundaryState>,
162    /// Up to 32 KiB of output before `out` (empty at the start of a member).
163    window: Arc<[u8]>,
164}
165
166impl std::fmt::Debug for Checkpoint {
167    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
168        f.debug_struct("Checkpoint")
169            .field("out", &self.out)
170            .field("input", &self.input)
171            .field("member_start", &self.state.is_none())
172            .finish_non_exhaustive()
173    }
174}
175
176/// What one decoding step produced.
177enum Step {
178    /// `n` bytes of output were written at the buffer position before the step.
179    Output { at: usize, n: usize },
180    /// A DEFLATE block ended (only when asked to stop there).
181    Block,
182    /// A new member started (its header has been read).
183    Member,
184    /// End of the data (clean, truncated or damaged: see `problem`).
185    End,
186}
187
188#[derive(Debug, Clone, Copy, PartialEq, Eq)]
189enum Phase {
190    Deflate,
191    Trailer,
192    Header,
193    End,
194}
195
196/// A running decoder.
197struct Stream {
198    r: Box<DecompressorOxide>,
199    buf: Box<[u8]>,
200    /// Write position in `buf`.
201    pos: usize,
202    /// Total output so far (decompressed offset of `buf[pos]`).
203    out: u64,
204    /// Buffered compressed bytes; `inbuf[ip]` is at compressed offset `next_in`.
205    inbuf: Vec<u8>,
206    ip: usize,
207    next_in: u64,
208    /// No more compressed bytes can be read.
209    eof: bool,
210    phase: Phase,
211    /// CRC-32 and length of the current member's output (verified when `verify`).
212    crc: crc32fast::Hasher,
213    member_out: u64,
214    verify: bool,
215    members: u32,
216    /// Output bytes available in `buf` before `pos` (at most `WRAP`).
217    history: usize,
218    problem: Option<String>,
219    header: Option<GzipHeader>,
220    /// A block ended in the step that returned output; report it next.
221    pending_block: bool,
222}
223
224impl Stream {
225    fn at_member_start(input: u64, out: u64, verify: bool) -> Self {
226        Stream {
227            r: Box::default(),
228            buf: vec![0u8; WRAP].into_boxed_slice(),
229            pos: 0,
230            out,
231            inbuf: Vec::new(),
232            ip: 0,
233            next_in: input,
234            eof: false,
235            phase: Phase::Header,
236            crc: crc32fast::Hasher::new(),
237            member_out: 0,
238            verify,
239            members: 0,
240            history: 0,
241            problem: None,
242            header: None,
243            pending_block: false,
244        }
245    }
246
247    fn from_checkpoint(c: &Checkpoint) -> Self {
248        let mut s = Stream::at_member_start(c.input, c.out, false);
249        match &c.state {
250            None => s.phase = Phase::Deflate,
251            Some(st) => {
252                s.r = Box::new(DecompressorOxide::from_block_boundary_state(st));
253                s.phase = Phase::Deflate;
254            }
255        }
256        let w = c.window.len().min(WINDOW);
257        s.buf[WINDOW - w..WINDOW].copy_from_slice(&c.window[c.window.len() - w..]);
258        s.pos = WINDOW;
259        s.history = w;
260        s
261    }
262
263    /// Up to the last `n` output bytes (for a checkpoint window).
264    fn tail(&self, n: usize) -> Vec<u8> {
265        let n = n.min(self.history);
266        let mut v = Vec::with_capacity(n);
267        let start = (self.pos + WRAP - n) & (WRAP - 1);
268        if start + n <= WRAP {
269            v.extend_from_slice(&self.buf[start..start + n]);
270        } else {
271            v.extend_from_slice(&self.buf[start..]);
272            v.extend_from_slice(&self.buf[..n - (WRAP - start)]);
273        }
274        v
275    }
276
277    /// Make at least one unread compressed byte available (unless at the end).
278    fn fill(&mut self, src: &dyn ByteSource) -> io::Result<()> {
279        if self.ip < self.inbuf.len() || self.eof {
280            return Ok(());
281        }
282        self.inbuf.resize(IN_CHUNK, 0);
283        let mut got = 0;
284        while got == 0 {
285            match src.read_at(self.next_in, &mut self.inbuf) {
286                Ok(0) => {
287                    self.eof = true;
288                    break;
289                }
290                Ok(n) => got = n,
291                Err(e) if e.kind() == io::ErrorKind::Interrupted => {}
292                Err(e) => return Err(e),
293            }
294        }
295        self.inbuf.truncate(got);
296        self.ip = 0;
297        Ok(())
298    }
299
300    /// Take exactly `n` compressed bytes (fewer at the end of the file).
301    fn take(&mut self, src: &dyn ByteSource, n: usize) -> io::Result<Vec<u8>> {
302        let mut v = Vec::with_capacity(n);
303        while v.len() < n {
304            self.fill(src)?;
305            if self.ip >= self.inbuf.len() {
306                break;
307            }
308            let k = (n - v.len()).min(self.inbuf.len() - self.ip);
309            v.extend_from_slice(&self.inbuf[self.ip..self.ip + k]);
310            self.ip += k;
311            self.next_in += k as u64;
312        }
313        Ok(v)
314    }
315
316    fn fail(&mut self, why: String) -> Step {
317        self.problem = Some(why);
318        self.phase = Phase::End;
319        Step::End
320    }
321
322    /// Decode a little further.
323    fn step(&mut self, src: &dyn ByteSource, stop_on_block: bool) -> io::Result<Step> {
324        if std::mem::take(&mut self.pending_block) {
325            return Ok(Step::Block);
326        }
327        loop {
328            match self.phase {
329                Phase::End => return Ok(Step::End),
330                Phase::Header => return self.read_header(src),
331                Phase::Trailer => return self.read_trailer(src),
332                Phase::Deflate => {}
333            }
334            self.fill(src)?;
335            let mut flags = 0;
336            if !self.eof {
337                flags |= TINFL_FLAG_HAS_MORE_INPUT;
338            }
339            if stop_on_block {
340                flags |= TINFL_FLAG_STOP_ON_BLOCK_BOUNDARY;
341            }
342            let at = self.pos;
343            let avail = self.inbuf.len() - self.ip;
344            let (status, nin, nout) = decompress(
345                &mut self.r,
346                &self.inbuf[self.ip..],
347                &mut self.buf,
348                at,
349                flags,
350            );
351            self.ip += nin;
352            self.next_in += nin as u64;
353            if nout > 0 {
354                if self.verify {
355                    self.crc.update(&self.buf[at..at + nout]);
356                }
357                self.member_out += nout as u64;
358                self.out += nout as u64;
359                self.pos = (at + nout) & (WRAP - 1);
360                self.history = (self.history + nout).min(WRAP);
361            }
362            let truncated = || {
363                format!(
364                    "the file ends inside compressed data (truncated after {} decompressed bytes)",
365                    self.out
366                )
367            };
368            match status {
369                TINFLStatus::Done => self.phase = Phase::Trailer,
370                TINFLStatus::BlockBoundary => {
371                    if nout == 0 {
372                        return Ok(Step::Block);
373                    }
374                    self.pending_block = true;
375                }
376                TINFLStatus::HasMoreOutput => {}
377                TINFLStatus::NeedsMoreInput if !self.eof && (nin > 0 || avail == 0) => {}
378                TINFLStatus::NeedsMoreInput | TINFLStatus::FailedCannotMakeProgress => {
379                    let why = truncated();
380                    return Ok(self.fail(why));
381                }
382                other => {
383                    let why = format!(
384                        "damaged deflate data near compressed byte {} ({other:?}); the first {} decompressed bytes are readable",
385                        self.next_in, self.out
386                    );
387                    return Ok(self.fail(why));
388                }
389            }
390            if nout > 0 {
391                return Ok(Step::Output { at, n: nout });
392            }
393        }
394    }
395
396    fn read_trailer(&mut self, src: &dyn ByteSource) -> io::Result<Step> {
397        let t = self.take(src, 8)?;
398        if t.len() < 8 {
399            return Ok(self.fail(format!(
400                "the file ends inside the gzip trailer of member {} (truncated)",
401                self.members.max(1)
402            )));
403        }
404        if self.verify {
405            let want = u32::from_le_bytes([t[0], t[1], t[2], t[3]]);
406            let size = u32::from_le_bytes([t[4], t[5], t[6], t[7]]);
407            let got = std::mem::replace(&mut self.crc, crc32fast::Hasher::new()).finalize();
408            if want != got {
409                return Ok(self.fail(format!(
410                    "CRC-32 mismatch in gzip member {} (recorded {want:08x}, data {got:08x}): the data is damaged",
411                    self.members.max(1)
412                )));
413            }
414            if u64::from(size) != self.member_out & 0xFFFF_FFFF {
415                return Ok(self.fail(format!(
416                    "gzip member {} length mismatch (recorded {size} mod 2^32, decoded {})",
417                    self.members.max(1),
418                    self.member_out
419                )));
420            }
421        }
422        self.phase = Phase::Header;
423        self.read_header(src)
424    }
425
426    fn read_header(&mut self, src: &dyn ByteSource) -> io::Result<Step> {
427        // Look at up to 64 KiB without consuming; headers are short unless FEXTRA is large.
428        self.fill(src)?;
429        if self.ip >= self.inbuf.len() {
430            if self.members == 0 {
431                return Ok(self.fail("the file is empty".into()));
432            }
433            self.phase = Phase::End;
434            return Ok(Step::End);
435        }
436        let mut peek = self.inbuf[self.ip..].to_vec();
437        if peek.len() < 64 * 1024 && !self.eof {
438            let mut more = vec![0u8; 64 * 1024 - peek.len()];
439            let n = src.read_at(self.next_in + peek.len() as u64, &mut more)?;
440            peek.extend_from_slice(&more[..n]);
441        }
442        if self.members > 0 && peek.iter().all(|&b| b == 0) {
443            // zero padding after the last member (tape blocks, some writers)
444            self.phase = Phase::End;
445            return Ok(Step::End);
446        }
447        match parse_header(&peek) {
448            Ok(Some(h)) => {
449                let n = h.len;
450                self.take(src, n)?;
451                if self.header.is_none() {
452                    self.header = Some(h);
453                }
454                self.members += 1;
455                *self.r = DecompressorOxide::new();
456                self.crc = crc32fast::Hasher::new();
457                self.member_out = 0;
458                self.phase = Phase::Deflate;
459                Ok(Step::Member)
460            }
461            Ok(None) => Ok(self.fail("the file ends inside a gzip header (truncated)".into())),
462            Err(e) if self.members > 0 => Ok(self.fail(format!(
463                "{} bytes after gzip member {} are not gzip data and were ignored ({e})",
464                peek.len(),
465                self.members
466            ))),
467            Err(e) => Ok(self.fail(e)),
468        }
469    }
470}
471
472/// Summary of a gzip file, from the pass made at open.
473#[derive(Debug, Clone, Default)]
474pub struct GzipSummary {
475    /// Number of gzip members (usually 1).
476    pub members: u32,
477    /// Compressed size in bytes.
478    pub compressed_len: u64,
479    /// Decompressed size in bytes (of the part that decoded cleanly).
480    pub decompressed_len: u64,
481    /// Restart points kept.
482    pub checkpoints: usize,
483    /// The first member's header (original name, time stamp, comment).
484    pub header: GzipHeader,
485    /// Why the data ends early or is suspect, when it does.
486    pub problem: Option<String>,
487}
488
489/// The decompressed bytes of a gzip file, with random access (see the module docs).
490pub struct GzipSource {
491    inner: Arc<dyn ByteSource>,
492    name: String,
493    points: Vec<Checkpoint>,
494    summary: GzipSummary,
495    live: Mutex<Pool>,
496}
497
498impl std::fmt::Debug for GzipSource {
499    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
500        f.debug_struct("GzipSource")
501            .field("name", &self.name)
502            .field("summary", &self.summary)
503            .finish_non_exhaustive()
504    }
505}
506
507impl GzipSource {
508    /// Decompress `inner` once, verifying every member, and keep restart points. `name` names
509    /// the decompressed content in messages. A file that is not gzip at all is an error
510    /// (corrupt, `format` names the caller); a truncated or damaged one opens with
511    /// [`problem`](GzipSource::problem) set and its clean prefix readable.
512    pub fn open(inner: Arc<dyn ByteSource>, name: String, format: &'static str) -> Result<Self> {
513        let io_err = |e: io::Error| Error::io(std::path::Path::new(&inner.name()), e);
514        let compressed_len = inner.size().map_err(io_err)?;
515        let mut s = Stream::at_member_start(0, 0, true);
516        let mut points: Vec<Checkpoint> = Vec::new();
517        let mut spacing = FIRST_SPACING;
518        loop {
519            match s.step(inner.as_ref(), true).map_err(io_err)? {
520                Step::Member => points.push(Checkpoint {
521                    out: s.out,
522                    input: s.next_in,
523                    state: None,
524                    window: Arc::from(Vec::new()),
525                }),
526                Step::Block => {
527                    let last = points.last().map_or(0, |p| p.out);
528                    if s.out.saturating_sub(last) >= spacing
529                        && let Some(state) = s.r.block_boundary_state()
530                    {
531                        points.push(Checkpoint {
532                            out: s.out,
533                            input: s.next_in,
534                            state: Some(state),
535                            window: Arc::from(s.tail(WINDOW)),
536                        });
537                        if points.len() > MAX_CHECKPOINTS {
538                            // keep member starts and every other block checkpoint
539                            let mut keep = false;
540                            points.retain(|p| {
541                                keep = !keep;
542                                keep || p.state.is_none()
543                            });
544                            spacing = spacing.saturating_mul(2);
545                        }
546                    }
547                }
548                Step::Output { .. } => {}
549                Step::End => break,
550            }
551        }
552        if s.members == 0 {
553            return Err(Error::corrupt(
554                format,
555                format!(
556                    "not a readable gzip file: {}",
557                    s.problem.unwrap_or_else(|| "no gzip member".into())
558                ),
559            ));
560        }
561        let summary = GzipSummary {
562            members: s.members,
563            compressed_len,
564            decompressed_len: s.out,
565            checkpoints: points.len(),
566            header: s.header.clone().unwrap_or_default(),
567            problem: s.problem.clone(),
568        };
569        Ok(GzipSource {
570            inner,
571            name,
572            points,
573            summary,
574            live: Mutex::new(Pool::default()),
575        })
576    }
577
578    /// What the pass at open found.
579    pub fn summary(&self) -> &GzipSummary {
580        &self.summary
581    }
582
583    /// Why the decompressed data ends early, when it does.
584    pub fn problem(&self) -> Option<&str> {
585        self.summary.problem.as_deref()
586    }
587
588    fn read_into(&self, offset: u64, buf: &mut [u8]) -> io::Result<usize> {
589        let len = self.summary.decompressed_len;
590        if offset >= len || buf.is_empty() {
591            return Ok(0);
592        }
593        let want = buf
594            .len()
595            .min(usize::try_from(len - offset).unwrap_or(usize::MAX));
596        let end = offset + want as u64;
597        let mut pool = self.live.lock().unwrap_or_else(PoisonError::into_inner);
598        pool.tick += 1;
599        let tick = pool.tick;
600        // Continue a live decoder when the wanted bytes are still in its buffer, or lie ahead of
601        // it but before the next restart point; otherwise restart at the nearest checkpoint.
602        // Several decoders are kept because readers alternate between places (an XML reader
603        // re-reads the document head before each element).
604        let cp = self.points.partition_point(|p| p.out <= offset);
605        let cp = &self.points[cp.saturating_sub(1)];
606        let best = pool
607            .streams
608            .iter()
609            .enumerate()
610            .filter(|(_, (_, s))| {
611                let behind = s.out.saturating_sub(s.history as u64);
612                // Continuing costs `offset - s.out`, restarting `offset - cp.out` plus the rest
613                // of the chunk the old decoder already produced (up to one buffer).
614                offset >= behind && s.out + WRAP as u64 >= cp.out
615            })
616            .min_by_key(|(_, (_, s))| offset.saturating_sub(s.out))
617            .map(|(i, _)| i);
618        let i = if let Some(i) = best {
619            i
620        } else {
621            let fresh = Stream::from_checkpoint(cp);
622            if pool.streams.len() < LIVE_DECODERS {
623                pool.streams.push((tick, fresh));
624                pool.streams.len() - 1
625            } else {
626                let lru = pool
627                    .streams
628                    .iter()
629                    .enumerate()
630                    .min_by_key(|(_, (t, _))| *t)
631                    .map_or(0, |(i, _)| i);
632                pool.streams[lru] = (tick, fresh);
633                lru
634            }
635        };
636        let Some((used, s)) = pool.streams.get_mut(i) else {
637            return Ok(0);
638        };
639        *used = tick;
640        let mut filled = 0usize;
641        // already decoded bytes in the buffer
642        if offset < s.out {
643            let back = (s.out - offset) as usize;
644            let n = back.min(want);
645            let start = (s.pos + WRAP - back) & (WRAP - 1);
646            copy_wrapped(&s.buf, start, &mut buf[..n]);
647            filled = n;
648        }
649        while filled < want {
650            match s.step(self.inner.as_ref(), false)? {
651                Step::Output { at, n } => {
652                    let first = s.out - n as u64; // decompressed offset of buf[at]
653                    let lo = (offset + filled as u64).max(first);
654                    let hi = end.min(s.out);
655                    if lo < hi {
656                        let from = at + (lo - first) as usize;
657                        let k = (hi - lo) as usize;
658                        buf[filled..filled + k].copy_from_slice(&s.buf[from..from + k]);
659                        filled += k;
660                    }
661                }
662                Step::Block | Step::Member => {}
663                Step::End => break,
664            }
665        }
666        Ok(filled)
667    }
668}
669
670fn copy_wrapped(ring: &[u8], start: usize, out: &mut [u8]) {
671    let n = out.len();
672    if start + n <= ring.len() {
673        out.copy_from_slice(&ring[start..start + n]);
674    } else {
675        let a = ring.len() - start;
676        out[..a].copy_from_slice(&ring[start..]);
677        out[a..].copy_from_slice(&ring[..n - a]);
678    }
679}
680
681impl ByteSource for GzipSource {
682    fn size(&self) -> io::Result<u64> {
683        Ok(self.summary.decompressed_len)
684    }
685
686    fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<usize> {
687        self.read_into(offset, buf)
688    }
689
690    fn name(&self) -> String {
691        self.name.clone()
692    }
693}
694
695/// `name` without a trailing `.gz` (any case): the name of the decompressed content.
696pub fn strip_gz(name: &str) -> &str {
697    let n = name.len();
698    if n > 3 && name.is_char_boundary(n - 3) && name[n - 3..].eq_ignore_ascii_case(".gz") {
699        &name[..n - 3]
700    } else {
701        name
702    }
703}
704
705#[cfg(test)]
706mod tests {
707    use super::*;
708    use crate::source::MemSource;
709    use std::io::Write;
710
711    fn gz(data: &[u8], level: u32) -> Vec<u8> {
712        let mut e = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::new(level));
713        e.write_all(data).unwrap();
714        e.finish().unwrap()
715    }
716
717    fn sample(n: usize) -> Vec<u8> {
718        // compressible but not trivial: pseudo-random words
719        let mut x = 0x1234_5678u32;
720        let words = [
721            &b"<spectrum "[..],
722            b"id=\"scan=",
723            b"\"/>\n",
724            b"cvParam ",
725            b"value=\"",
726        ];
727        let mut v = Vec::with_capacity(n + 16);
728        while v.len() < n {
729            x ^= x << 13;
730            x ^= x >> 17;
731            x ^= x << 5;
732            v.extend_from_slice(words[(x % 5) as usize]);
733            v.extend_from_slice((x % 100_000).to_string().as_bytes());
734        }
735        v.truncate(n);
736        v
737    }
738
739    fn open(bytes: Vec<u8>) -> GzipSource {
740        GzipSource::open(Arc::new(MemSource::new("t.gz", bytes)), "t".into(), "test").unwrap()
741    }
742
743    #[test]
744    fn random_and_sequential_reads_match() {
745        let data = sample(9 << 20);
746        let src = open(gz(&data, 6));
747        assert!(src.problem().is_none());
748        assert_eq!(src.size().unwrap(), data.len() as u64);
749        assert!(src.summary().checkpoints > 4, "{:?}", src.summary());
750        // random reads, backwards and forwards
751        let mut x = 7u64;
752        for _ in 0..200 {
753            x = x.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1);
754            let off = x % data.len() as u64;
755            let len = ((x >> 40) % 70_000) as usize;
756            let mut b = vec![0u8; len];
757            let n = src.read_at(off, &mut b).unwrap();
758            let want = len.min(data.len() - off as usize);
759            assert_eq!(n, want);
760            assert_eq!(&b[..n], &data[off as usize..off as usize + n]);
761        }
762        // front-to-back in small pieces, with small backward steps
763        let mut off = 0usize;
764        while off < data.len() {
765            let mut b = vec![0u8; 5000];
766            let n = src.read_at(off as u64, &mut b).unwrap();
767            assert_eq!(&b[..n], &data[off..off + n]);
768            off += n.saturating_sub(100).max(1);
769        }
770    }
771
772    #[test]
773    fn stored_and_multi_member_streams() {
774        let a = sample(3 << 20);
775        let b = sample(1 << 20);
776        let mut bytes = gz(&a, 0); // stored blocks
777        bytes.extend(gz(&b, 9));
778        bytes.extend([0u8; 16]); // zero padding
779        let src = open(bytes);
780        assert_eq!(src.summary().members, 2);
781        assert!(src.problem().is_none(), "{:?}", src.problem());
782        let mut all = a.clone();
783        all.extend_from_slice(&b);
784        let mut got = vec![0u8; all.len()];
785        src.read_exact_at(0, &mut got).unwrap();
786        assert_eq!(got, all);
787        let mut tail = vec![0u8; 1000];
788        src.read_exact_at(all.len() as u64 - 1000, &mut tail)
789            .unwrap();
790        assert_eq!(tail, all[all.len() - 1000..]);
791    }
792
793    #[test]
794    fn truncated_and_damaged_files_keep_their_prefix() {
795        let data = sample(4 << 20);
796        let full = gz(&data, 6);
797        let cut = full[..full.len() / 2].to_vec();
798        let src = open(cut);
799        let p = src.problem().unwrap();
800        assert!(p.contains("truncated"), "{p}");
801        let n = src.size().unwrap() as usize;
802        assert!(n > 0 && n < data.len());
803        let mut got = vec![0u8; n];
804        src.read_exact_at(0, &mut got).unwrap();
805        assert_eq!(got, data[..n]);
806
807        let mut bad = full.clone();
808        let l = bad.len();
809        bad[l - 6] ^= 0xFF; // CRC byte
810        let src = open(bad);
811        assert!(
812            src.problem().unwrap().contains("CRC"),
813            "{:?}",
814            src.problem()
815        );
816
817        assert!(
818            GzipSource::open(
819                Arc::new(MemSource::new("x", b"plain text".to_vec())),
820                "x".into(),
821                "t"
822            )
823            .is_err()
824        );
825    }
826
827    #[test]
828    fn headers_peek_and_names() {
829        let mut e = flate2::GzBuilder::new()
830            .filename("run.mzML")
831            .comment("c")
832            .extra(vec![1, 2, 3])
833            .write(Vec::new(), flate2::Compression::default());
834        e.write_all(b"<?xml version=\"1.0\"?><mzML>").unwrap();
835        let bytes = e.finish().unwrap();
836        assert!(is_gzip(&bytes));
837        let h = parse_header(&bytes).unwrap().unwrap();
838        assert_eq!(h.name.as_deref(), Some("run.mzML"));
839        assert_eq!(h.comment.as_deref(), Some("c"));
840        assert_eq!(peek(&bytes, 100).unwrap(), b"<?xml version=\"1.0\"?><mzML>");
841        assert_eq!(peek(&bytes[..5], 100), None);
842        assert_eq!(strip_gz("a.mzML.GZ"), "a.mzML");
843        assert_eq!(strip_gz("a.mzML"), "a.mzML");
844        for cut in 0..bytes.len() {
845            let _ = parse_header(&bytes[..cut]);
846            let _ = peek(&bytes[..cut], 64);
847        }
848    }
849}