use std::io;
use std::sync::{Arc, Mutex, PoisonError};
use miniz_oxide::inflate::TINFLStatus;
use miniz_oxide::inflate::core::inflate_flags::{
TINFL_FLAG_HAS_MORE_INPUT, TINFL_FLAG_STOP_ON_BLOCK_BOUNDARY,
TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF,
};
use miniz_oxide::inflate::core::{BlockBoundaryState, DecompressorOxide, decompress};
use crate::error::{Error, Result};
use crate::source::ByteSource;
pub const MAX_CHECKPOINTS: usize = 1024;
const FIRST_SPACING: u64 = 1 << 20;
const WINDOW: usize = 32 * 1024;
const WRAP: usize = 1 << 20;
const IN_CHUNK: usize = 256 * 1024;
const LIVE_DECODERS: usize = 4;
#[derive(Default)]
struct Pool {
tick: u64,
streams: Vec<(u64, Stream)>,
}
pub fn is_gzip(head: &[u8]) -> bool {
head.len() >= 3 && head[0] == 0x1f && head[1] == 0x8b && head[2] == 8
}
pub fn peek(head: &[u8], max_out: usize) -> Option<Vec<u8>> {
let hdr = parse_header(head).ok()??;
let data = head.get(hdr.len..)?;
let mut r = DecompressorOxide::new();
let mut out = vec![0u8; max_out];
let (_, _, n) = decompress(
&mut r,
data,
&mut out,
0,
TINFL_FLAG_HAS_MORE_INPUT | TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF,
);
out.truncate(n);
Some(out)
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct GzipHeader {
pub len: usize,
pub name: Option<String>,
pub mtime: u32,
pub comment: Option<String>,
}
fn parse_header(b: &[u8]) -> std::result::Result<Option<GzipHeader>, String> {
if b.len() < 10 {
return Ok(None);
}
if b[0] != 0x1f || b[1] != 0x8b {
return Err("not a gzip member (no 1F 8B signature)".into());
}
if b[2] != 8 {
return Err(format!(
"gzip compression method {} (only 8, deflate, exists)",
b[2]
));
}
let flg = b[3];
if flg & 0xE0 != 0 {
return Err(format!("gzip header flags {flg:#04x} set reserved bits"));
}
let mtime = u32::from_le_bytes([b[4], b[5], b[6], b[7]]);
let mut at = 10usize;
if flg & 0x04 != 0 {
let Some(x) = b.get(at..at + 2) else {
return Ok(None);
};
at += 2 + usize::from(u16::from_le_bytes([x[0], x[1]]));
}
let zstring = |at: &mut usize| -> Option<String> {
let rest = b.get(*at..)?;
let end = rest.iter().position(|&c| c == 0)?;
let s = String::from_utf8_lossy(&rest[..end]).into_owned();
*at += end + 1;
Some(s)
};
let mut name = None;
if flg & 0x08 != 0 {
match zstring(&mut at) {
Some(s) => name = Some(s),
None => return Ok(None),
}
}
let mut comment = None;
if flg & 0x10 != 0 {
match zstring(&mut at) {
Some(s) => comment = Some(s),
None => return Ok(None),
}
}
if flg & 0x02 != 0 {
at += 2;
}
if at > b.len() {
return Ok(None);
}
Ok(Some(GzipHeader {
len: at,
name,
mtime,
comment,
}))
}
#[derive(Clone)]
struct Checkpoint {
out: u64,
input: u64,
state: Option<BlockBoundaryState>,
window: Arc<[u8]>,
}
impl std::fmt::Debug for Checkpoint {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Checkpoint")
.field("out", &self.out)
.field("input", &self.input)
.field("member_start", &self.state.is_none())
.finish_non_exhaustive()
}
}
enum Step {
Output { at: usize, n: usize },
Block,
Member,
End,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Phase {
Deflate,
Trailer,
Header,
End,
}
struct Stream {
r: Box<DecompressorOxide>,
buf: Box<[u8]>,
pos: usize,
out: u64,
inbuf: Vec<u8>,
ip: usize,
next_in: u64,
eof: bool,
phase: Phase,
crc: crc32fast::Hasher,
member_out: u64,
verify: bool,
members: u32,
history: usize,
problem: Option<String>,
header: Option<GzipHeader>,
pending_block: bool,
}
impl Stream {
fn at_member_start(input: u64, out: u64, verify: bool) -> Self {
Stream {
r: Box::default(),
buf: vec![0u8; WRAP].into_boxed_slice(),
pos: 0,
out,
inbuf: Vec::new(),
ip: 0,
next_in: input,
eof: false,
phase: Phase::Header,
crc: crc32fast::Hasher::new(),
member_out: 0,
verify,
members: 0,
history: 0,
problem: None,
header: None,
pending_block: false,
}
}
fn from_checkpoint(c: &Checkpoint) -> Self {
let mut s = Stream::at_member_start(c.input, c.out, false);
match &c.state {
None => s.phase = Phase::Deflate,
Some(st) => {
s.r = Box::new(DecompressorOxide::from_block_boundary_state(st));
s.phase = Phase::Deflate;
}
}
let w = c.window.len().min(WINDOW);
s.buf[WINDOW - w..WINDOW].copy_from_slice(&c.window[c.window.len() - w..]);
s.pos = WINDOW;
s.history = w;
s
}
fn tail(&self, n: usize) -> Vec<u8> {
let n = n.min(self.history);
let mut v = Vec::with_capacity(n);
let start = (self.pos + WRAP - n) & (WRAP - 1);
if start + n <= WRAP {
v.extend_from_slice(&self.buf[start..start + n]);
} else {
v.extend_from_slice(&self.buf[start..]);
v.extend_from_slice(&self.buf[..n - (WRAP - start)]);
}
v
}
fn fill(&mut self, src: &dyn ByteSource) -> io::Result<()> {
if self.ip < self.inbuf.len() || self.eof {
return Ok(());
}
self.inbuf.resize(IN_CHUNK, 0);
let mut got = 0;
while got == 0 {
match src.read_at(self.next_in, &mut self.inbuf) {
Ok(0) => {
self.eof = true;
break;
}
Ok(n) => got = n,
Err(e) if e.kind() == io::ErrorKind::Interrupted => {}
Err(e) => return Err(e),
}
}
self.inbuf.truncate(got);
self.ip = 0;
Ok(())
}
fn take(&mut self, src: &dyn ByteSource, n: usize) -> io::Result<Vec<u8>> {
let mut v = Vec::with_capacity(n);
while v.len() < n {
self.fill(src)?;
if self.ip >= self.inbuf.len() {
break;
}
let k = (n - v.len()).min(self.inbuf.len() - self.ip);
v.extend_from_slice(&self.inbuf[self.ip..self.ip + k]);
self.ip += k;
self.next_in += k as u64;
}
Ok(v)
}
fn fail(&mut self, why: String) -> Step {
self.problem = Some(why);
self.phase = Phase::End;
Step::End
}
fn step(&mut self, src: &dyn ByteSource, stop_on_block: bool) -> io::Result<Step> {
if std::mem::take(&mut self.pending_block) {
return Ok(Step::Block);
}
loop {
match self.phase {
Phase::End => return Ok(Step::End),
Phase::Header => return self.read_header(src),
Phase::Trailer => return self.read_trailer(src),
Phase::Deflate => {}
}
self.fill(src)?;
let mut flags = 0;
if !self.eof {
flags |= TINFL_FLAG_HAS_MORE_INPUT;
}
if stop_on_block {
flags |= TINFL_FLAG_STOP_ON_BLOCK_BOUNDARY;
}
let at = self.pos;
let avail = self.inbuf.len() - self.ip;
let (status, nin, nout) = decompress(
&mut self.r,
&self.inbuf[self.ip..],
&mut self.buf,
at,
flags,
);
self.ip += nin;
self.next_in += nin as u64;
if nout > 0 {
if self.verify {
self.crc.update(&self.buf[at..at + nout]);
}
self.member_out += nout as u64;
self.out += nout as u64;
self.pos = (at + nout) & (WRAP - 1);
self.history = (self.history + nout).min(WRAP);
}
let truncated = || {
format!(
"the file ends inside compressed data (truncated after {} decompressed bytes)",
self.out
)
};
match status {
TINFLStatus::Done => self.phase = Phase::Trailer,
TINFLStatus::BlockBoundary => {
if nout == 0 {
return Ok(Step::Block);
}
self.pending_block = true;
}
TINFLStatus::HasMoreOutput => {}
TINFLStatus::NeedsMoreInput if !self.eof && (nin > 0 || avail == 0) => {}
TINFLStatus::NeedsMoreInput | TINFLStatus::FailedCannotMakeProgress => {
let why = truncated();
return Ok(self.fail(why));
}
other => {
let why = format!(
"damaged deflate data near compressed byte {} ({other:?}); the first {} decompressed bytes are readable",
self.next_in, self.out
);
return Ok(self.fail(why));
}
}
if nout > 0 {
return Ok(Step::Output { at, n: nout });
}
}
}
fn read_trailer(&mut self, src: &dyn ByteSource) -> io::Result<Step> {
let t = self.take(src, 8)?;
if t.len() < 8 {
return Ok(self.fail(format!(
"the file ends inside the gzip trailer of member {} (truncated)",
self.members.max(1)
)));
}
if self.verify {
let want = u32::from_le_bytes([t[0], t[1], t[2], t[3]]);
let size = u32::from_le_bytes([t[4], t[5], t[6], t[7]]);
let got = std::mem::replace(&mut self.crc, crc32fast::Hasher::new()).finalize();
if want != got {
return Ok(self.fail(format!(
"CRC-32 mismatch in gzip member {} (recorded {want:08x}, data {got:08x}): the data is damaged",
self.members.max(1)
)));
}
if u64::from(size) != self.member_out & 0xFFFF_FFFF {
return Ok(self.fail(format!(
"gzip member {} length mismatch (recorded {size} mod 2^32, decoded {})",
self.members.max(1),
self.member_out
)));
}
}
self.phase = Phase::Header;
self.read_header(src)
}
fn read_header(&mut self, src: &dyn ByteSource) -> io::Result<Step> {
self.fill(src)?;
if self.ip >= self.inbuf.len() {
if self.members == 0 {
return Ok(self.fail("the file is empty".into()));
}
self.phase = Phase::End;
return Ok(Step::End);
}
let mut peek = self.inbuf[self.ip..].to_vec();
if peek.len() < 64 * 1024 && !self.eof {
let mut more = vec![0u8; 64 * 1024 - peek.len()];
let n = src.read_at(self.next_in + peek.len() as u64, &mut more)?;
peek.extend_from_slice(&more[..n]);
}
if self.members > 0 && peek.iter().all(|&b| b == 0) {
self.phase = Phase::End;
return Ok(Step::End);
}
match parse_header(&peek) {
Ok(Some(h)) => {
let n = h.len;
self.take(src, n)?;
if self.header.is_none() {
self.header = Some(h);
}
self.members += 1;
*self.r = DecompressorOxide::new();
self.crc = crc32fast::Hasher::new();
self.member_out = 0;
self.phase = Phase::Deflate;
Ok(Step::Member)
}
Ok(None) => Ok(self.fail("the file ends inside a gzip header (truncated)".into())),
Err(e) if self.members > 0 => Ok(self.fail(format!(
"{} bytes after gzip member {} are not gzip data and were ignored ({e})",
peek.len(),
self.members
))),
Err(e) => Ok(self.fail(e)),
}
}
}
#[derive(Debug, Clone, Default)]
pub struct GzipSummary {
pub members: u32,
pub compressed_len: u64,
pub decompressed_len: u64,
pub checkpoints: usize,
pub header: GzipHeader,
pub problem: Option<String>,
}
pub struct GzipSource {
inner: Arc<dyn ByteSource>,
name: String,
points: Vec<Checkpoint>,
summary: GzipSummary,
live: Mutex<Pool>,
}
impl std::fmt::Debug for GzipSource {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("GzipSource")
.field("name", &self.name)
.field("summary", &self.summary)
.finish_non_exhaustive()
}
}
impl GzipSource {
pub fn open(inner: Arc<dyn ByteSource>, name: String, format: &'static str) -> Result<Self> {
let io_err = |e: io::Error| Error::io(std::path::Path::new(&inner.name()), e);
let compressed_len = inner.size().map_err(io_err)?;
let mut s = Stream::at_member_start(0, 0, true);
let mut points: Vec<Checkpoint> = Vec::new();
let mut spacing = FIRST_SPACING;
loop {
match s.step(inner.as_ref(), true).map_err(io_err)? {
Step::Member => points.push(Checkpoint {
out: s.out,
input: s.next_in,
state: None,
window: Arc::from(Vec::new()),
}),
Step::Block => {
let last = points.last().map_or(0, |p| p.out);
if s.out.saturating_sub(last) >= spacing
&& let Some(state) = s.r.block_boundary_state()
{
points.push(Checkpoint {
out: s.out,
input: s.next_in,
state: Some(state),
window: Arc::from(s.tail(WINDOW)),
});
if points.len() > MAX_CHECKPOINTS {
let mut keep = false;
points.retain(|p| {
keep = !keep;
keep || p.state.is_none()
});
spacing = spacing.saturating_mul(2);
}
}
}
Step::Output { .. } => {}
Step::End => break,
}
}
if s.members == 0 {
return Err(Error::corrupt(
format,
format!(
"not a readable gzip file: {}",
s.problem.unwrap_or_else(|| "no gzip member".into())
),
));
}
let summary = GzipSummary {
members: s.members,
compressed_len,
decompressed_len: s.out,
checkpoints: points.len(),
header: s.header.clone().unwrap_or_default(),
problem: s.problem.clone(),
};
Ok(GzipSource {
inner,
name,
points,
summary,
live: Mutex::new(Pool::default()),
})
}
pub fn summary(&self) -> &GzipSummary {
&self.summary
}
pub fn problem(&self) -> Option<&str> {
self.summary.problem.as_deref()
}
fn read_into(&self, offset: u64, buf: &mut [u8]) -> io::Result<usize> {
let len = self.summary.decompressed_len;
if offset >= len || buf.is_empty() {
return Ok(0);
}
let want = buf
.len()
.min(usize::try_from(len - offset).unwrap_or(usize::MAX));
let end = offset + want as u64;
let mut pool = self.live.lock().unwrap_or_else(PoisonError::into_inner);
pool.tick += 1;
let tick = pool.tick;
let cp = self.points.partition_point(|p| p.out <= offset);
let cp = &self.points[cp.saturating_sub(1)];
let best = pool
.streams
.iter()
.enumerate()
.filter(|(_, (_, s))| {
let behind = s.out.saturating_sub(s.history as u64);
offset >= behind && s.out + WRAP as u64 >= cp.out
})
.min_by_key(|(_, (_, s))| offset.saturating_sub(s.out))
.map(|(i, _)| i);
let i = if let Some(i) = best {
i
} else {
let fresh = Stream::from_checkpoint(cp);
if pool.streams.len() < LIVE_DECODERS {
pool.streams.push((tick, fresh));
pool.streams.len() - 1
} else {
let lru = pool
.streams
.iter()
.enumerate()
.min_by_key(|(_, (t, _))| *t)
.map_or(0, |(i, _)| i);
pool.streams[lru] = (tick, fresh);
lru
}
};
let Some((used, s)) = pool.streams.get_mut(i) else {
return Ok(0);
};
*used = tick;
let mut filled = 0usize;
if offset < s.out {
let back = (s.out - offset) as usize;
let n = back.min(want);
let start = (s.pos + WRAP - back) & (WRAP - 1);
copy_wrapped(&s.buf, start, &mut buf[..n]);
filled = n;
}
while filled < want {
match s.step(self.inner.as_ref(), false)? {
Step::Output { at, n } => {
let first = s.out - n as u64; let lo = (offset + filled as u64).max(first);
let hi = end.min(s.out);
if lo < hi {
let from = at + (lo - first) as usize;
let k = (hi - lo) as usize;
buf[filled..filled + k].copy_from_slice(&s.buf[from..from + k]);
filled += k;
}
}
Step::Block | Step::Member => {}
Step::End => break,
}
}
Ok(filled)
}
}
fn copy_wrapped(ring: &[u8], start: usize, out: &mut [u8]) {
let n = out.len();
if start + n <= ring.len() {
out.copy_from_slice(&ring[start..start + n]);
} else {
let a = ring.len() - start;
out[..a].copy_from_slice(&ring[start..]);
out[a..].copy_from_slice(&ring[..n - a]);
}
}
impl ByteSource for GzipSource {
fn size(&self) -> io::Result<u64> {
Ok(self.summary.decompressed_len)
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<usize> {
self.read_into(offset, buf)
}
fn name(&self) -> String {
self.name.clone()
}
}
pub fn strip_gz(name: &str) -> &str {
let n = name.len();
if n > 3 && name.is_char_boundary(n - 3) && name[n - 3..].eq_ignore_ascii_case(".gz") {
&name[..n - 3]
} else {
name
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::source::MemSource;
use std::io::Write;
fn gz(data: &[u8], level: u32) -> Vec<u8> {
let mut e = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::new(level));
e.write_all(data).unwrap();
e.finish().unwrap()
}
fn sample(n: usize) -> Vec<u8> {
let mut x = 0x1234_5678u32;
let words = [
&b"<spectrum "[..],
b"id=\"scan=",
b"\"/>\n",
b"cvParam ",
b"value=\"",
];
let mut v = Vec::with_capacity(n + 16);
while v.len() < n {
x ^= x << 13;
x ^= x >> 17;
x ^= x << 5;
v.extend_from_slice(words[(x % 5) as usize]);
v.extend_from_slice((x % 100_000).to_string().as_bytes());
}
v.truncate(n);
v
}
fn open(bytes: Vec<u8>) -> GzipSource {
GzipSource::open(Arc::new(MemSource::new("t.gz", bytes)), "t".into(), "test").unwrap()
}
#[test]
fn random_and_sequential_reads_match() {
let data = sample(9 << 20);
let src = open(gz(&data, 6));
assert!(src.problem().is_none());
assert_eq!(src.size().unwrap(), data.len() as u64);
assert!(src.summary().checkpoints > 4, "{:?}", src.summary());
let mut x = 7u64;
for _ in 0..200 {
x = x.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1);
let off = x % data.len() as u64;
let len = ((x >> 40) % 70_000) as usize;
let mut b = vec![0u8; len];
let n = src.read_at(off, &mut b).unwrap();
let want = len.min(data.len() - off as usize);
assert_eq!(n, want);
assert_eq!(&b[..n], &data[off as usize..off as usize + n]);
}
let mut off = 0usize;
while off < data.len() {
let mut b = vec![0u8; 5000];
let n = src.read_at(off as u64, &mut b).unwrap();
assert_eq!(&b[..n], &data[off..off + n]);
off += n.saturating_sub(100).max(1);
}
}
#[test]
fn stored_and_multi_member_streams() {
let a = sample(3 << 20);
let b = sample(1 << 20);
let mut bytes = gz(&a, 0); bytes.extend(gz(&b, 9));
bytes.extend([0u8; 16]); let src = open(bytes);
assert_eq!(src.summary().members, 2);
assert!(src.problem().is_none(), "{:?}", src.problem());
let mut all = a.clone();
all.extend_from_slice(&b);
let mut got = vec![0u8; all.len()];
src.read_exact_at(0, &mut got).unwrap();
assert_eq!(got, all);
let mut tail = vec![0u8; 1000];
src.read_exact_at(all.len() as u64 - 1000, &mut tail)
.unwrap();
assert_eq!(tail, all[all.len() - 1000..]);
}
#[test]
fn truncated_and_damaged_files_keep_their_prefix() {
let data = sample(4 << 20);
let full = gz(&data, 6);
let cut = full[..full.len() / 2].to_vec();
let src = open(cut);
let p = src.problem().unwrap();
assert!(p.contains("truncated"), "{p}");
let n = src.size().unwrap() as usize;
assert!(n > 0 && n < data.len());
let mut got = vec![0u8; n];
src.read_exact_at(0, &mut got).unwrap();
assert_eq!(got, data[..n]);
let mut bad = full.clone();
let l = bad.len();
bad[l - 6] ^= 0xFF; let src = open(bad);
assert!(
src.problem().unwrap().contains("CRC"),
"{:?}",
src.problem()
);
assert!(
GzipSource::open(
Arc::new(MemSource::new("x", b"plain text".to_vec())),
"x".into(),
"t"
)
.is_err()
);
}
#[test]
fn headers_peek_and_names() {
let mut e = flate2::GzBuilder::new()
.filename("run.mzML")
.comment("c")
.extra(vec![1, 2, 3])
.write(Vec::new(), flate2::Compression::default());
e.write_all(b"<?xml version=\"1.0\"?><mzML>").unwrap();
let bytes = e.finish().unwrap();
assert!(is_gzip(&bytes));
let h = parse_header(&bytes).unwrap().unwrap();
assert_eq!(h.name.as_deref(), Some("run.mzML"));
assert_eq!(h.comment.as_deref(), Some("c"));
assert_eq!(peek(&bytes, 100).unwrap(), b"<?xml version=\"1.0\"?><mzML>");
assert_eq!(peek(&bytes[..5], 100), None);
assert_eq!(strip_gz("a.mzML.GZ"), "a.mzML");
assert_eq!(strip_gz("a.mzML"), "a.mzML");
for cut in 0..bytes.len() {
let _ = parse_header(&bytes[..cut]);
let _ = peek(&bytes[..cut], 64);
}
}
}