use crate::algorithms::zex_native;
use crate::format::block::{BlockHeader, BlockType};
use crate::format::header::{HEADER_SIZE, HeaderFlags, ZexHeader};
use crate::utils::crc::Crc32;
use crate::utils::security::SecurityPolicy;
use crate::{Error, Result};
use std::io::{self, Read, Write};
use super::encoder::CompressOptions;
fn encode_block(chunk: &[u8], level: u8, dict: Option<&[u8]>) -> Result<(BlockType, Vec<u8>)> {
use crate::algorithms::turbo;
if chunk.is_empty() {
return Ok((BlockType::Raw, Vec::new()));
}
if chunk.len() < 16 && dict.is_none() {
return Ok((BlockType::Raw, chunk.to_vec()));
}
if turbo::should_use_turbo(level) && dict.is_none() && turbo::looks_incompressible(chunk) {
return Ok((BlockType::Raw, chunk.to_vec()));
}
let payload = if turbo::should_use_turbo(level) {
let mut buf = Vec::new();
turbo::compress_into_level(chunk, &mut buf, dict, level)?;
buf
} else {
zex_native::compress_with_history(chunk, level, dict)?
};
if payload.len() >= chunk.len() {
Ok((BlockType::Raw, chunk.to_vec()))
} else {
Ok((BlockType::Zex, payload))
}
}
fn write_block<W: Write>(out: &mut W, ty: BlockType, unc: u32, payload: &[u8]) -> Result<()> {
let bh = BlockHeader {
block_type: ty,
uncompressed_size: unc,
compressed_size: payload.len() as u32,
};
bh.write_to(out)?;
out.write_all(payload)?;
Ok(())
}
pub struct StreamEncoder<W: Write> {
writer: Option<W>,
opts: CompressOptions,
level: u8,
block_size: usize,
buf: Vec<u8>,
header_written: bool,
finished: bool,
crc: Crc32,
pub bytes_in: u64,
pub bytes_out: u64,
}
impl<W: Write> StreamEncoder<W> {
pub fn new(writer: W, opts: CompressOptions) -> Self {
let level = opts.mode.effective_level(opts.level);
let block_size = (opts.block_size as usize)
.min(opts.security.max_block_uncompressed)
.max(1024);
Self {
writer: Some(writer),
opts,
level,
block_size,
buf: Vec::with_capacity(block_size),
header_written: false,
finished: false,
crc: Crc32::new(),
bytes_in: 0,
bytes_out: 0,
}
}
pub fn with_defaults(writer: W) -> Self {
Self::new(writer, CompressOptions::default())
}
fn ensure_header(&mut self) -> io::Result<()> {
if self.header_written {
return Ok(());
}
let w = self
.writer
.as_mut()
.ok_or_else(|| io::Error::other("encoder finished"))?;
let mut header = ZexHeader::new(u64::MAX, self.opts.mode, self.level);
header.block_size = self.block_size as u32;
header.flags.set(HeaderFlags::CHECKSUM, self.opts.checksum);
header.flags.set(HeaderFlags::INDEPENDENT, true);
header.flags.set(HeaderFlags::CONTENT_SIZE, false);
if let Some(ref d) = self.opts.dictionary {
header.dictionary_id = d.id;
header.flags.insert(HeaderFlags::DICTIONARY);
}
header.write_to(w).map_err(io::Error::other)?;
self.bytes_out += HEADER_SIZE as u64;
self.header_written = true;
Ok(())
}
fn flush_full_blocks(&mut self) -> io::Result<()> {
self.ensure_header()?;
while self.buf.len() >= self.block_size {
let chunk: Vec<u8> = self.buf.drain(..self.block_size).collect();
self.emit_block(&chunk)?;
}
Ok(())
}
fn emit_block(&mut self, chunk: &[u8]) -> io::Result<()> {
let w = self
.writer
.as_mut()
.ok_or_else(|| io::Error::other("encoder finished"))?;
let dict = self.opts.dictionary.as_ref().map(|d| d.data.as_slice());
let (ty, payload) = encode_block(chunk, self.level, dict).map_err(io::Error::other)?;
write_block(w, ty, chunk.len() as u32, &payload).map_err(io::Error::other)?;
self.bytes_out += 9 + payload.len() as u64;
Ok(())
}
pub fn finish(mut self) -> Result<W> {
if self.finished {
return self
.writer
.take()
.ok_or_else(|| Error::Compression("already finished".into()));
}
self.ensure_header().map_err(Error::Io)?;
if !self.buf.is_empty() {
let chunk = std::mem::take(&mut self.buf);
self.emit_block(&chunk).map_err(Error::Io)?;
} else if self.bytes_in == 0 {
self.emit_block(&[]).map_err(Error::Io)?;
}
let w = self
.writer
.as_mut()
.ok_or_else(|| Error::Compression("encoder finished".into()))?;
BlockHeader::end().write_to(w)?;
self.bytes_out += 9;
if self.opts.checksum {
let c = self.crc.clone_finalize();
w.write_all(&c.to_le_bytes())?;
self.bytes_out += 4;
}
self.finished = true;
self.writer
.take()
.ok_or_else(|| Error::Compression("encoder finished".into()))
}
pub fn options(&self) -> &CompressOptions {
&self.opts
}
}
impl<W: Write> Write for StreamEncoder<W> {
fn write(&mut self, data: &[u8]) -> io::Result<usize> {
if self.finished || self.writer.is_none() {
return Err(io::Error::other("stream encoder already finished"));
}
if data.is_empty() {
return Ok(0);
}
self.crc.update(data);
self.bytes_in += data.len() as u64;
self.buf.extend_from_slice(data);
self.flush_full_blocks()?;
Ok(data.len())
}
fn flush(&mut self) -> io::Result<()> {
self.flush_full_blocks()?;
if let Some(w) = self.writer.as_mut() {
w.flush()?;
}
Ok(())
}
}
pub struct StreamDecoder<R: Read> {
reader: R,
header: Option<ZexHeader>,
out: Vec<u8>,
out_pos: usize,
finished: bool,
crc: Crc32,
verify_checksum: bool,
security: SecurityPolicy,
dictionary: Option<crate::core::dictionary::Dictionary>,
pending_crc_check: bool,
}
impl<R: Read> StreamDecoder<R> {
pub fn new(reader: R) -> Self {
Self {
reader,
header: None,
out: Vec::new(),
out_pos: 0,
finished: false,
crc: Crc32::new(),
verify_checksum: true,
security: SecurityPolicy::default(),
dictionary: None,
pending_crc_check: false,
}
}
pub fn security(mut self, policy: SecurityPolicy) -> Self {
self.security = policy;
self
}
pub fn verify_checksum(mut self, on: bool) -> Self {
self.verify_checksum = on;
self
}
pub fn dictionary(mut self, dict: crate::core::dictionary::Dictionary) -> Self {
self.dictionary = Some(dict);
self
}
pub fn header(&self) -> Option<&ZexHeader> {
self.header.as_ref()
}
fn ensure_header(&mut self) -> io::Result<()> {
if self.header.is_some() {
return Ok(());
}
let h = ZexHeader::read_from(&mut self.reader)
.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
if h.is_encrypted() {
return Err(io::Error::new(
io::ErrorKind::Unsupported,
"stream decode does not support encrypted frames; use one-shot decompress",
));
}
if h.dictionary_id != 0 {
match &self.dictionary {
None => {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
format!("frame needs dictionary id {:#x}", h.dictionary_id),
));
}
Some(d) if d.id != h.dictionary_id => {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
"dictionary id mismatch",
));
}
Some(_) => {}
}
}
self.security
.check_frame_size(h.content_size)
.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
self.pending_crc_check = h.has_checksum() && self.verify_checksum;
self.header = Some(h);
Ok(())
}
fn pull_block(&mut self) -> io::Result<bool> {
if self.finished {
return Ok(false);
}
self.ensure_header()?;
let bh = BlockHeader::read_from(&mut self.reader)
.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
if bh.block_type == BlockType::End {
if self.pending_crc_check {
let mut crc_buf = [0u8; 4];
self.reader.read_exact(&mut crc_buf)?;
let expected = u32::from_le_bytes(crc_buf);
let actual = self.crc.clone_finalize();
if actual != expected {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("checksum mismatch: expected 0x{expected:08x}, got 0x{actual:08x}"),
));
}
}
self.finished = true;
return Ok(false);
}
self.security
.check_block(bh.compressed_size as usize, bh.uncompressed_size as usize)
.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
let mut payload = vec![0u8; bh.compressed_size as usize];
self.reader.read_exact(&mut payload)?;
let unc = bh.uncompressed_size as usize;
let plain = match bh.block_type {
BlockType::Raw => {
if payload.len() != unc {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"raw block size mismatch",
));
}
payload
}
BlockType::Zex => {
let dict = self
.dictionary
.as_ref()
.map(|d| d.data.as_slice())
.filter(|_| {
self.header
.as_ref()
.map(|h| h.dictionary_id != 0)
.unwrap_or(false)
});
zex_native::decompress_with_history(&payload, unc, dict)
.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?
}
BlockType::End => unreachable!(),
};
self.crc.update(&plain);
if self.out_pos > 0 {
self.out.drain(..self.out_pos);
self.out_pos = 0;
}
self.out.extend_from_slice(&plain);
Ok(true)
}
}
impl<R: Read> Read for StreamDecoder<R> {
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
if buf.is_empty() {
return Ok(0);
}
loop {
let available = self.out.len().saturating_sub(self.out_pos);
if available > 0 {
let n = available.min(buf.len());
buf[..n].copy_from_slice(&self.out[self.out_pos..self.out_pos + n]);
self.out_pos += n;
return Ok(n);
}
if self.finished {
return Ok(0);
}
if !self.pull_block()? {
return Ok(0);
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::core::decoder::decompress;
use crate::format::header::CompressionMode;
use std::io::{Cursor, Read, Write};
#[test]
fn stream_encode_decode_roundtrip() {
let src = b"streaming ZEX test data! ".repeat(10_000);
let mut sink = Vec::new();
{
let mut enc = StreamEncoder::new(
&mut sink,
CompressOptions::default()
.mode(CompressionMode::Balanced)
.level(3)
.block_size(16 * 1024),
);
let mut off = 0;
while off < src.len() {
let n = (off % 97 + 1).min(src.len() - off);
enc.write_all(&src[off..off + n]).unwrap();
off += n;
}
enc.finish().unwrap();
}
assert_eq!(&sink[..4], b"ZEX1");
let d = decompress(&sink).unwrap();
assert_eq!(d, src);
let mut dec = StreamDecoder::new(Cursor::new(&sink));
let mut out = Vec::new();
dec.read_to_end(&mut out).unwrap();
assert_eq!(out, src);
}
#[test]
fn stream_empty() {
let mut sink = Vec::new();
let enc = StreamEncoder::with_defaults(&mut sink);
enc.finish().unwrap();
let d = decompress(&sink).unwrap();
assert!(d.is_empty());
}
#[test]
fn stream_small_writes() {
let mut sink = Vec::new();
{
let mut enc = StreamEncoder::with_defaults(&mut sink);
for b in b"hello stream" {
enc.write_all(&[*b]).unwrap();
}
enc.finish().unwrap();
}
let mut out = Vec::new();
StreamDecoder::new(Cursor::new(&sink))
.read_to_end(&mut out)
.unwrap();
assert_eq!(out, b"hello stream");
}
}