use crate::crc32::Crc32;
use crate::{AnalysisErrorKind, AnalysisResource, DecodeError, GzipErrorKind, ReadAt};
use std::io::{self, Read};
const FLAG_HEADER_CRC: u8 = 0x02;
const FLAG_EXTRA: u8 = 0x04;
const FLAG_NAME: u8 = 0x08;
const FLAG_COMMENT: u8 = 0x10;
const RESERVED_FLAGS: u8 = 0xE0;
pub(crate) trait InputCursor {
fn position(&self) -> u64;
fn is_at_end(&mut self) -> Result<bool, DecodeError>;
fn available(&mut self) -> Result<&[u8], DecodeError>;
fn advance(&mut self, count: usize);
fn verify_source_unchanged(&self) -> Result<(), DecodeError>;
fn peek_two(&mut self) -> Result<Option<[u8; 2]>, DecodeError>;
fn byte(&mut self, truncated_at: u64) -> Result<u8, DecodeError> {
let Some(&byte) = self.available()?.first() else {
return Err(DecodeError::InvalidGzip {
offset: truncated_at,
reason: GzipErrorKind::Truncated,
});
};
self.advance(1);
Ok(byte)
}
fn read_exact<const N: usize>(&mut self, truncated_at: u64) -> Result<[u8; N], DecodeError> {
let mut result = [0_u8; N];
for byte in &mut result {
*byte = self.byte(truncated_at)?;
}
Ok(result)
}
}
impl<T: InputCursor + ?Sized> InputCursor for &mut T {
fn position(&self) -> u64 {
(**self).position()
}
fn is_at_end(&mut self) -> Result<bool, DecodeError> {
(**self).is_at_end()
}
fn available(&mut self) -> Result<&[u8], DecodeError> {
(**self).available()
}
fn advance(&mut self, count: usize) {
(**self).advance(count);
}
fn verify_source_unchanged(&self) -> Result<(), DecodeError> {
(**self).verify_source_unchanged()
}
fn peek_two(&mut self) -> Result<Option<[u8; 2]>, DecodeError> {
(**self).peek_two()
}
}
pub(crate) struct SourceCursor<'a, R: ReadAt + ?Sized> {
source: &'a R,
length: u64,
position: u64,
page: Vec<u8>,
page_start: u64,
page_length: usize,
}
impl<'a, R: ReadAt + ?Sized> SourceCursor<'a, R> {
pub(crate) fn new(source: &'a R, page_size: usize) -> Result<Self, DecodeError> {
let length = source
.len()
.map_err(|error| DecodeError::input_io(0, error))?;
Ok(Self {
source,
length,
position: 0,
page: vec![0; page_size],
page_start: u64::MAX,
page_length: 0,
})
}
pub(crate) const fn length(&self) -> u64 {
self.length
}
pub(crate) const fn at_end(&self) -> bool {
self.position >= self.length
}
pub(crate) fn seek(&mut self, position: u64) -> Result<(), DecodeError> {
if position > self.length {
return Err(DecodeError::InvalidGzip {
offset: self.position,
reason: GzipErrorKind::Truncated,
});
}
self.position = position;
Ok(())
}
fn refill(&mut self) -> Result<(), DecodeError> {
if self.at_end() {
self.page_start = self.position;
self.page_length = 0;
return Ok(());
}
self.page_start = self.position;
let remaining = usize::try_from(self.length - self.position).unwrap_or(usize::MAX);
let wanted = remaining.min(self.page.len());
let read = self
.source
.read_at(self.position, &mut self.page[..wanted])
.map_err(|error| DecodeError::input_io(self.position, error))?;
if read == 0 {
return Err(DecodeError::input_io(
self.position,
io::Error::new(
io::ErrorKind::UnexpectedEof,
"positional source ended before its snapshotted length",
),
));
}
self.page_length = read;
Ok(())
}
}
impl<R: ReadAt + ?Sized> InputCursor for SourceCursor<'_, R> {
fn position(&self) -> u64 {
self.position
}
fn is_at_end(&mut self) -> Result<bool, DecodeError> {
Ok(self.at_end())
}
fn available(&mut self) -> Result<&[u8], DecodeError> {
let page_end = self.page_start.saturating_add(self.page_length as u64);
if self.position < self.page_start || self.position >= page_end {
self.refill()?;
}
let relative = usize::try_from(self.position - self.page_start)
.expect("a page-relative offset always fits usize");
Ok(&self.page[relative..self.page_length])
}
fn advance(&mut self, count: usize) {
self.position = self.position.saturating_add(count as u64);
}
fn verify_source_unchanged(&self) -> Result<(), DecodeError> {
let final_length = self
.source
.len()
.map_err(|error| DecodeError::input_io(self.position, error))?;
if final_length != self.length {
return Err(DecodeError::input_io(
self.position,
io::Error::new(
io::ErrorKind::InvalidData,
"compressed source length changed during decoding",
),
));
}
Ok(())
}
fn peek_two(&mut self) -> Result<Option<[u8; 2]>, DecodeError> {
if self.length.saturating_sub(self.position) < 2 {
return Ok(None);
}
let mut prefix = [0_u8; 2];
let mut filled = 0;
while filled < prefix.len() {
let offset = self.position + filled as u64;
let read = self
.source
.read_at(offset, &mut prefix[filled..])
.map_err(|error| DecodeError::input_io(offset, error))?;
if read == 0 {
return Err(DecodeError::input_io(
offset,
io::Error::new(
io::ErrorKind::UnexpectedEof,
"positional source ended before its snapshotted length",
),
));
}
filled += read;
}
Ok(Some(prefix))
}
}
pub(crate) struct StreamCursor<R> {
source: R,
buffer: Vec<u8>,
consumed: usize,
filled: usize,
position: u64,
at_end: bool,
}
impl<R: Read> StreamCursor<R> {
pub(crate) fn new(source: R, page_size: usize) -> Self {
Self {
source,
buffer: vec![0; page_size.max(2)],
consumed: 0,
filled: 0,
position: 0,
at_end: false,
}
}
fn refill(&mut self) -> Result<(), DecodeError> {
if self.consumed != 0 {
self.buffer.copy_within(self.consumed..self.filled, 0);
self.filled -= self.consumed;
self.consumed = 0;
}
if self.filled == self.buffer.len() {
return Ok(());
}
loop {
match self.source.read(&mut self.buffer[self.filled..]) {
Ok(0) => {
self.at_end = true;
return Ok(());
}
Ok(count) => {
self.filled += count;
return Ok(());
}
Err(error) if error.kind() == io::ErrorKind::Interrupted => {}
Err(error) => return Err(DecodeError::input_io(self.position, error)),
}
}
}
pub(crate) fn buffered(&self) -> &[u8] {
&self.buffer[self.consumed..self.filled]
}
pub(crate) const fn stream_ended(&self) -> bool {
self.at_end
}
}
impl<R: Read> InputCursor for StreamCursor<R> {
fn position(&self) -> u64 {
self.position
}
fn is_at_end(&mut self) -> Result<bool, DecodeError> {
Ok(self.available()?.is_empty())
}
fn available(&mut self) -> Result<&[u8], DecodeError> {
if self.consumed == self.filled && !self.at_end {
self.refill()?;
}
Ok(&self.buffer[self.consumed..self.filled])
}
fn advance(&mut self, count: usize) {
debug_assert!(count <= self.filled - self.consumed);
let count = count.min(self.filled - self.consumed);
self.consumed += count;
self.position = self.position.saturating_add(count as u64);
}
fn verify_source_unchanged(&self) -> Result<(), DecodeError> {
Ok(())
}
fn peek_two(&mut self) -> Result<Option<[u8; 2]>, DecodeError> {
while self.filled - self.consumed < 2 && !self.at_end {
self.refill()?;
}
if self.filled - self.consumed < 2 {
Ok(None)
} else {
Ok(Some([
self.buffer[self.consumed],
self.buffer[self.consumed + 1],
]))
}
}
}
pub(crate) struct SliceCursor<'a> {
bytes: &'a [u8],
position: usize,
}
impl<'a> SliceCursor<'a> {
pub(crate) const fn new(bytes: &'a [u8]) -> Self {
Self { bytes, position: 0 }
}
}
impl InputCursor for SliceCursor<'_> {
fn position(&self) -> u64 {
self.position as u64
}
fn is_at_end(&mut self) -> Result<bool, DecodeError> {
Ok(self.position >= self.bytes.len())
}
fn available(&mut self) -> Result<&[u8], DecodeError> {
Ok(&self.bytes[self.position.min(self.bytes.len())..])
}
fn advance(&mut self, count: usize) {
self.position = self.position.saturating_add(count).min(self.bytes.len());
}
fn verify_source_unchanged(&self) -> Result<(), DecodeError> {
Ok(())
}
fn peek_two(&mut self) -> Result<Option<[u8; 2]>, DecodeError> {
let remaining = &self.bytes[self.position.min(self.bytes.len())..];
Ok((remaining.len() >= 2).then(|| [remaining[0], remaining[1]]))
}
}
#[derive(Clone, Copy, Debug)]
pub(crate) struct MemberHeader {
pub(crate) start: u64,
pub(crate) deflate_start: u64,
pub(crate) bgzf_block_size: Option<u16>,
}
pub(crate) struct DetailedMemberHeader {
pub(crate) member: MemberHeader,
pub(crate) flags: u8,
pub(crate) modification_time: u32,
pub(crate) extra_flags: u8,
pub(crate) operating_system: u8,
pub(crate) file_name: Option<Vec<u8>>,
pub(crate) comment: Option<Vec<u8>>,
pub(crate) extra: Option<Vec<u8>>,
pub(crate) header_crc16: Option<u16>,
pub(crate) retained_metadata_bytes: usize,
}
struct HeaderDetailsBuilder {
maximum_bytes: usize,
retained_bytes: usize,
flags: u8,
modification_time: u32,
extra_flags: u8,
operating_system: u8,
file_name: Option<Vec<u8>>,
comment: Option<Vec<u8>>,
extra: Option<Vec<u8>>,
header_crc16: Option<u16>,
}
impl HeaderDetailsBuilder {
const fn new(maximum_bytes: usize, retained_bytes: usize) -> Self {
Self {
maximum_bytes,
retained_bytes,
flags: 0,
modification_time: 0,
extra_flags: 0,
operating_system: 0,
file_name: None,
comment: None,
extra: None,
header_crc16: None,
}
}
fn reserve_bytes(&mut self, additional: usize) -> Result<(), DecodeError> {
if self
.retained_bytes
.checked_add(additional)
.is_none_or(|total| total > self.maximum_bytes)
{
return Err(DecodeError::Analysis {
reason: AnalysisErrorKind::ResourceLimit {
resource: AnalysisResource::HeaderBytes,
limit: self.maximum_bytes,
},
});
}
self.retained_bytes += additional;
Ok(())
}
fn push_name_byte(&mut self, byte: u8) -> Result<(), DecodeError> {
self.reserve_bytes(1)?;
let name = self.file_name.get_or_insert_with(Vec::new);
name.try_reserve(1).map_err(|_| DecodeError::Analysis {
reason: AnalysisErrorKind::AllocationFailed {
resource: AnalysisResource::HeaderBytes,
additional: 1,
},
})?;
name.push(byte);
Ok(())
}
fn push_comment_byte(&mut self, byte: u8) -> Result<(), DecodeError> {
self.reserve_bytes(1)?;
let comment = self.comment.get_or_insert_with(Vec::new);
comment.try_reserve(1).map_err(|_| DecodeError::Analysis {
reason: AnalysisErrorKind::AllocationFailed {
resource: AnalysisResource::HeaderBytes,
additional: 1,
},
})?;
comment.push(byte);
Ok(())
}
fn finish(self, member: MemberHeader) -> DetailedMemberHeader {
DetailedMemberHeader {
member,
flags: self.flags,
modification_time: self.modification_time,
extra_flags: self.extra_flags,
operating_system: self.operating_system,
file_name: self.file_name,
comment: self.comment,
extra: self.extra,
header_crc16: self.header_crc16,
retained_metadata_bytes: self.retained_bytes,
}
}
}
fn checked_header_byte<C: InputCursor>(
cursor: &mut C,
crc: &mut Option<Crc32>,
start: u64,
) -> Result<u8, DecodeError> {
let byte = cursor.byte(start)?;
if let Some(crc) = crc {
crc.update(&[byte]);
}
Ok(byte)
}
pub(crate) fn parse_member_header<C: InputCursor>(
cursor: &mut C,
first_member: bool,
) -> Result<MemberHeader, DecodeError> {
parse_member_header_inner(cursor, first_member, None).map(|(header, _)| header)
}
pub(crate) fn parse_member_header_detailed<C: InputCursor>(
cursor: &mut C,
first_member: bool,
maximum_metadata_bytes: usize,
retained_metadata_bytes: usize,
) -> Result<DetailedMemberHeader, DecodeError> {
let details = HeaderDetailsBuilder::new(maximum_metadata_bytes, retained_metadata_bytes);
let (member, details) = parse_member_header_inner(cursor, first_member, Some(details))?;
Ok(details
.expect("the detailed parser preserves its builder")
.finish(member))
}
fn parse_member_header_inner<C: InputCursor>(
cursor: &mut C,
first_member: bool,
mut details: Option<HeaderDetailsBuilder>,
) -> Result<(MemberHeader, Option<HeaderDetailsBuilder>), DecodeError> {
let start = cursor.position();
let id1 = cursor.byte(start)?;
let id2 = cursor.byte(start)?;
if (id1, id2) != (0x1F, 0x8B) {
return Err(DecodeError::InvalidGzip {
offset: start,
reason: if first_member {
GzipErrorKind::BadMagic
} else {
GzipErrorKind::TrailingGarbage
},
});
}
let compression_method = cursor.byte(start)?;
if compression_method != 8 {
return Err(DecodeError::InvalidGzip {
offset: start + 2,
reason: GzipErrorKind::UnsupportedCompressionMethod(compression_method),
});
}
let flags = cursor.byte(start)?;
if flags & RESERVED_FLAGS != 0 {
return Err(DecodeError::InvalidGzip {
offset: start + 3,
reason: GzipErrorKind::ReservedFlags(flags),
});
}
let mut header_crc = if flags & FLAG_HEADER_CRC != 0 {
let mut crc = Crc32::new();
crc.update(&[id1, id2, compression_method, flags]);
Some(crc)
} else {
None
};
if let Some(details) = details.as_mut() {
details.flags = flags;
}
let mut fixed_metadata = [0_u8; 6];
for byte in &mut fixed_metadata {
*byte = checked_header_byte(cursor, &mut header_crc, start)?;
}
if let Some(details) = details.as_mut() {
details.modification_time = u32::from_le_bytes(
fixed_metadata[..4]
.try_into()
.expect("the metadata prefix has four bytes"),
);
details.extra_flags = fixed_metadata[4];
details.operating_system = fixed_metadata[5];
}
let mut bgzf_block_size = None;
if flags & FLAG_EXTRA != 0 {
let low = checked_header_byte(cursor, &mut header_crc, start)?;
let high = checked_header_byte(cursor, &mut header_crc, start)?;
let extra_length = usize::from(u16::from_le_bytes([low, high]));
if let Some(details) = details.as_mut() {
details.reserve_bytes(extra_length)?;
}
let analyzing = details.is_some();
let mut extra = Vec::new();
extra.try_reserve_exact(extra_length).map_err(|_| {
if analyzing {
DecodeError::Analysis {
reason: AnalysisErrorKind::AllocationFailed {
resource: AnalysisResource::HeaderBytes,
additional: extra_length,
},
}
} else {
DecodeError::input_io(
start,
io::Error::new(
io::ErrorKind::OutOfMemory,
"could not allocate the gzip extra field",
),
)
}
})?;
for _ in 0..extra_length {
extra.push(checked_header_byte(cursor, &mut header_crc, start)?);
}
let mut offset: usize = 0;
while offset.saturating_add(4) <= extra.len() {
let subfield_length =
usize::from(u16::from_le_bytes([extra[offset + 2], extra[offset + 3]]));
let data_start = offset + 4;
let data_end = data_start.saturating_add(subfield_length);
if data_end > extra.len() {
break;
}
if &extra[offset..offset + 2] == b"BC" && subfield_length == 2 {
bgzf_block_size = Some(u16::from_le_bytes([
extra[data_start],
extra[data_start + 1],
]));
}
offset = data_end;
}
if let Some(details) = details.as_mut() {
details.extra = Some(extra);
}
}
for (flag, is_name) in [(FLAG_NAME, true), (FLAG_COMMENT, false)] {
if flags & flag != 0 {
if let Some(details) = details.as_mut() {
if is_name {
details.file_name = Some(Vec::new());
} else {
details.comment = Some(Vec::new());
}
}
loop {
if cursor.is_at_end()? {
return Err(DecodeError::InvalidGzip {
offset: start,
reason: GzipErrorKind::UnterminatedHeaderField,
});
}
let byte = checked_header_byte(cursor, &mut header_crc, start)?;
if byte == 0 {
break;
}
if let Some(details) = details.as_mut() {
if is_name {
details.push_name_byte(byte)?;
} else {
details.push_comment_byte(byte)?;
}
}
}
}
}
if flags & FLAG_HEADER_CRC != 0 {
let expected = u16::from_le_bytes(cursor.read_exact::<2>(start)?);
let actual = header_crc
.expect("FHCRC initialized a header checksum")
.finish() as u16;
if actual != expected {
return Err(DecodeError::InvalidGzip {
offset: cursor.position() - 2,
reason: GzipErrorKind::HeaderChecksumMismatch { expected, actual },
});
}
if let Some(details) = details.as_mut() {
details.header_crc16 = Some(expected);
}
}
Ok((
MemberHeader {
start,
deflate_start: cursor.position(),
bgzf_block_size,
},
details,
))
}
#[cfg(test)]
pub(crate) fn validate_initial_stream_header<R: Read>(
cursor: &mut StreamCursor<R>,
) -> Result<(), DecodeError> {
cursor.available()?;
let stream_ended = cursor.at_end;
let mut prefix = SliceCursor::new(cursor.buffered());
match parse_member_header(&mut prefix, true) {
Ok(_) => Ok(()),
Err(DecodeError::InvalidGzip {
reason: GzipErrorKind::Truncated | GzipErrorKind::UnterminatedHeaderField,
..
}) if !stream_ended => Ok(()),
Err(error) => Err(error),
}
}
#[cfg(test)]
mod tests {
use super::{
InputCursor, SourceCursor, StreamCursor, parse_member_header,
validate_initial_stream_header,
};
use crate::GzipErrorKind;
struct Trickle<'a> {
bytes: &'a [u8],
step: usize,
}
impl super::Read for Trickle<'_> {
fn read(&mut self, output: &mut [u8]) -> std::io::Result<usize> {
let count = self.bytes.len().min(output.len()).min(self.step);
output[..count].copy_from_slice(&self.bytes[..count]);
self.bytes = &self.bytes[count..];
Ok(count)
}
}
#[test]
fn parses_minimal_header() {
let bytes = b"\x1f\x8b\x08\x00\0\0\0\0\x00\xff";
let mut cursor = SourceCursor::new(bytes.as_slice(), 4).unwrap();
let header = parse_member_header(&mut cursor, true).unwrap();
assert_eq!(header.start, 0);
assert_eq!(header.deflate_start, 10);
}
#[test]
fn reports_trailing_garbage_after_a_member() {
let bytes = b"not gzip";
let mut cursor = SourceCursor::new(bytes.as_slice(), 4).unwrap();
let error = parse_member_header(&mut cursor, false).unwrap_err();
assert!(matches!(
error,
crate::DecodeError::InvalidGzip {
reason: GzipErrorKind::TrailingGarbage,
..
}
));
}
#[test]
fn stream_cursor_reads_across_window_refills() {
let bytes: Vec<u8> = (0..=255_u8).collect();
let mut cursor = StreamCursor::new(
Trickle {
bytes: &bytes,
step: 3,
},
7,
);
let mut seen = Vec::new();
while !cursor.is_at_end().unwrap() {
let available = cursor.available().unwrap();
let take = available.len().min(2);
seen.extend_from_slice(&available[..take]);
cursor.advance(take);
}
assert_eq!(seen, bytes);
assert_eq!(cursor.position(), bytes.len() as u64);
}
#[test]
fn stream_cursor_parses_a_header_from_a_trickling_source() {
let bytes = b"\x1f\x8b\x08\x00\0\0\0\0\x00\xffrest";
let mut cursor = StreamCursor::new(
Trickle {
bytes: bytes.as_slice(),
step: 1,
},
4,
);
let header = parse_member_header(&mut cursor, true).unwrap();
assert_eq!(header.deflate_start, 10);
assert_eq!(cursor.position(), 10);
}
#[test]
fn initial_stream_validation_rejects_bad_magic_without_consuming() {
let bytes = b"not gzip at all";
let mut cursor = StreamCursor::new(
Trickle {
bytes: bytes.as_slice(),
step: 4,
},
16,
);
let error = validate_initial_stream_header(&mut cursor).unwrap_err();
assert!(matches!(
error,
crate::DecodeError::InvalidGzip {
reason: GzipErrorKind::BadMagic,
..
}
));
assert_eq!(cursor.position(), 0);
}
#[test]
fn initial_stream_validation_defers_a_header_longer_than_the_window() {
let mut bytes = b"\x1f\x8b\x08\x08\0\0\0\0\x00\xff".to_vec();
bytes.extend_from_slice(&[b'n'; 64]);
bytes.push(0);
let mut cursor = StreamCursor::new(
Trickle {
bytes: &bytes,
step: 8,
},
8,
);
validate_initial_stream_header(&mut cursor).unwrap();
assert_eq!(cursor.position(), 0);
let header = parse_member_header(&mut cursor, true).unwrap();
assert_eq!(header.deflate_start, bytes.len() as u64);
}
}