use alloc::vec::Vec;
use crate::Result;
use crate::algorithms::{turbo, zex_native};
use crate::core::dictionary::Dictionary;
use crate::format::block::{BlockHeader, BlockType};
use crate::format::header::{
CompressionMode, DEFAULT_BLOCK_SIZE, HEADER_SIZE, HeaderFlags, ZexHeader,
};
use crate::utils::crc::crc32;
use crate::utils::security::SecurityPolicy;
#[cfg(feature = "std")]
use rayon::prelude::*;
#[cfg(feature = "std")]
use std::io::Write;
#[cfg(feature = "encryption")]
use crate::crypto::{
EncryptionKey, NONCE_LEN, SALT_LEN, derive_key_from_password, encrypt_body, fill_random,
};
#[derive(Debug, Clone)]
pub struct CompressOptions {
pub mode: CompressionMode,
pub level: u8,
pub block_size: u32,
pub checksum: bool,
pub threads: bool,
pub independent: bool,
pub dictionary: Option<Dictionary>,
pub security: SecurityPolicy,
#[cfg(feature = "encryption")]
pub encryption: Option<EncryptionKey>,
}
impl Default for CompressOptions {
fn default() -> Self {
Self {
mode: CompressionMode::Balanced,
level: 3,
block_size: DEFAULT_BLOCK_SIZE,
checksum: true,
threads: true,
independent: true,
dictionary: None,
security: SecurityPolicy::default(),
#[cfg(feature = "encryption")]
encryption: None,
}
}
}
impl CompressOptions {
pub fn mode(mut self, mode: CompressionMode) -> Self {
self.mode = mode;
self
}
pub fn level(mut self, level: u8) -> Self {
self.level = level.clamp(1, 22);
self
}
pub fn block_size(mut self, size: u32) -> Self {
self.block_size = size.clamp(1024, 16 * 1024 * 1024);
self
}
pub fn checksum(mut self, on: bool) -> Self {
self.checksum = on;
self
}
pub fn threads(mut self, on: bool) -> Self {
self.threads = on;
self
}
pub fn dictionary(mut self, dict: Dictionary) -> Self {
self.dictionary = Some(dict);
self
}
pub fn secure(mut self) -> Self {
self.security = SecurityPolicy::strict();
self.checksum = true;
self.independent = true;
self
}
#[cfg(feature = "encryption")]
pub fn encrypt(mut self, key: EncryptionKey) -> Self {
self.encryption = Some(key);
self
}
}
pub struct Encoder {
opts: CompressOptions,
}
impl Encoder {
pub fn new(opts: CompressOptions) -> Self {
Self { opts }
}
pub fn compress(&self, src: &[u8]) -> Result<Vec<u8>> {
let mut out = Vec::with_capacity(src.len() / 2 + HEADER_SIZE + 64);
self.compress_into(src, &mut out)?;
Ok(out)
}
#[cfg(feature = "std")]
pub fn compress_to_writer<W: Write>(&self, src: &[u8], out: &mut W) -> Result<()> {
out.write_all(&self.compress(src)?)?;
Ok(())
}
fn compress_into(&self, src: &[u8], out: &mut Vec<u8>) -> Result<()> {
let opts = &self.opts;
let level = opts.mode.effective_level(opts.level);
let block_size = (opts.block_size as usize).min(opts.security.max_block_uncompressed);
let mut header = ZexHeader::new(src.len() as u64, opts.mode, level);
header.block_size = block_size as u32;
header.flags.set(HeaderFlags::CHECKSUM, opts.checksum);
header.flags.set(HeaderFlags::INDEPENDENT, opts.independent);
header.flags.set(
HeaderFlags::MULTITHREAD,
opts.threads && src.len() > block_size,
);
header.flags.set(
HeaderFlags::SECURE,
opts.security.max_output <= 512 * 1024 * 1024,
);
if let Some(ref d) = opts.dictionary {
header.dictionary_id = d.id;
header.flags.insert(HeaderFlags::DICTIONARY);
}
#[cfg(feature = "encryption")]
let want_enc = opts.encryption.is_some();
#[cfg(not(feature = "encryption"))]
let want_enc = false;
if want_enc {
header.flags.insert(HeaderFlags::ENCRYPTED);
}
let mut body = Vec::with_capacity(src.len() / 2 + 128);
let chunks: Vec<&[u8]> = if src.is_empty() {
vec![src]
} else {
src.chunks(block_size).collect()
};
let dict_bytes: Option<&[u8]> = opts.dictionary.as_ref().map(|d| d.data.as_slice());
#[cfg(feature = "std")]
let compressed: Vec<(BlockType, Vec<u8>, u32)> = if opts.threads && chunks.len() > 1 {
chunks
.par_iter()
.map(|chunk| compress_one_block(chunk, level, dict_bytes))
.collect::<Result<Vec<_>>>()?
} else {
chunks
.iter()
.map(|chunk| compress_one_block(chunk, level, dict_bytes))
.collect::<Result<Vec<_>>>()?
};
#[cfg(not(feature = "std"))]
let compressed: Vec<(BlockType, Vec<u8>, u32)> = chunks
.iter()
.map(|chunk| compress_one_block(chunk, level, dict_bytes))
.collect::<Result<Vec<_>>>()?;
for (ty, payload, unc) in compressed {
opts.security.check_block(payload.len(), unc as usize)?;
let bh = BlockHeader {
block_type: ty,
uncompressed_size: unc,
compressed_size: payload.len() as u32,
};
body.extend_from_slice(&bh.to_bytes());
body.extend_from_slice(&payload);
}
body.extend_from_slice(&BlockHeader::end().to_bytes());
if opts.checksum {
body.extend_from_slice(&crc32(src).to_le_bytes());
}
out.extend_from_slice(&header.to_bytes());
#[cfg(feature = "encryption")]
if let Some(ref ek) = opts.encryption {
let mut salt = [0u8; SALT_LEN];
let mut nonce = [0u8; NONCE_LEN];
fill_random(&mut salt)?;
fill_random(&mut nonce)?;
let key = match ek {
EncryptionKey::Raw(k) => {
salt = [0u8; SALT_LEN]; *k
}
EncryptionKey::Password(pw) => derive_key_from_password(pw, &salt)?,
};
let ct = encrypt_body(&key, &nonce, &body)?;
out.extend_from_slice(&salt);
out.extend_from_slice(&nonce);
out.extend_from_slice(&ct);
return Ok(());
}
let _ = want_enc;
out.extend_from_slice(&body);
Ok(())
}
}
fn compress_one_block(
chunk: &[u8],
level: u8,
dict: Option<&[u8]>,
) -> Result<(BlockType, Vec<u8>, u32)> {
let unc = chunk.len() as u32;
if chunk.is_empty() {
return Ok((BlockType::Raw, Vec::new(), 0));
}
if chunk.len() < 16 && dict.is_none() {
return Ok((BlockType::Raw, chunk.to_vec(), unc));
}
if turbo::should_use_turbo(level) && dict.is_none() && turbo::looks_incompressible(chunk) {
return Ok((BlockType::Raw, chunk.to_vec(), unc));
}
let payload = if turbo::should_use_turbo(level) {
let mut buf = Vec::with_capacity(chunk.len() / 2 + 32);
turbo::compress_into_level(chunk, &mut buf, dict, level)?;
buf
} else {
zex_native::compress_with_history(chunk, level, dict)?
};
if payload.len() >= chunk.len() {
Ok((BlockType::Raw, chunk.to_vec(), unc))
} else {
Ok((BlockType::Zex, payload, unc))
}
}
pub fn compress(src: &[u8], opts: &CompressOptions) -> Result<Vec<u8>> {
Encoder::new(opts.clone()).compress(src)
}
pub fn compress_with_dict(src: &[u8], opts: &CompressOptions, dict: Dictionary) -> Result<Vec<u8>> {
let mut o = opts.clone();
o.dictionary = Some(dict);
Encoder::new(o).compress(src)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::core::decoder::{DecompressOptions, decompress, decompress_with_options};
use crate::core::dictionary::Dictionary;
#[test]
fn encode_decode_native() {
let src = b"ZEX native encoder roundtrip. ".repeat(500);
let opts = CompressOptions::default()
.mode(CompressionMode::Balanced)
.level(3);
let c = compress(&src, &opts).unwrap();
assert_eq!(&c[..4], b"ZEX1");
let d = decompress(&c).unwrap();
assert_eq!(d, src);
}
#[test]
fn profiles_roundtrip() {
let src = b"ABCDEFGHIJ".repeat(1000);
for mode in [
CompressionMode::Turbo,
CompressionMode::Balanced,
CompressionMode::Max,
] {
let opts = CompressOptions::default()
.mode(mode)
.level(5)
.threads(false);
let c = compress(&src, &opts).unwrap();
let d = decompress(&c).unwrap();
assert_eq!(d, src, "mode {mode}");
}
}
#[test]
fn empty_input() {
let c = compress(b"", &CompressOptions::default()).unwrap();
let d = decompress(&c).unwrap();
assert!(d.is_empty());
}
#[test]
fn secure_preset() {
let src = b"secure mode data ".repeat(100);
let c = compress(&src, &CompressOptions::default().secure()).unwrap();
assert_eq!(decompress(&c).unwrap(), src);
}
#[test]
fn dictionary_history_frame_roundtrip() {
let dict = Dictionary::from_bytes(
b"SHARED-PREFIX-HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n\r\n".to_vec(),
)
.unwrap();
let msg =
b"SHARED-PREFIX-HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n\r\nhello dict history";
let without = compress(msg, &CompressOptions::default().level(6).threads(false)).unwrap();
let with = compress(
msg,
&CompressOptions::default()
.level(6)
.threads(false)
.dictionary(dict.clone()),
)
.unwrap();
assert!(
with.len() <= without.len(),
"dict frame {} vs plain {}",
with.len(),
without.len()
);
let h = ZexHeader::from_bytes(&with).unwrap();
assert_eq!(h.dictionary_id, dict.id);
assert!(h.flags.contains(HeaderFlags::DICTIONARY));
let d = decompress_with_options(
&with,
&DecompressOptions {
dictionary: Some(dict),
..Default::default()
},
)
.unwrap();
assert_eq!(d, msg);
}
#[cfg(feature = "encryption")]
#[test]
fn encrypt_password_roundtrip() {
let src = b"top secret payload for ZEX AEAD test!!!!".repeat(20);
let opts = CompressOptions::default()
.level(4)
.encrypt(EncryptionKey::Password("hunter2".into()));
let c = compress(&src, &opts).unwrap();
let h = ZexHeader::from_bytes(&c).unwrap();
assert!(h.is_encrypted());
let dopts = DecompressOptions {
encryption: Some(EncryptionKey::Password("hunter2".into())),
..Default::default()
};
let d = decompress_with_options(&c, &dopts).unwrap();
assert_eq!(d, src);
let bad = DecompressOptions {
encryption: Some(EncryptionKey::Password("wrong".into())),
..Default::default()
};
assert!(decompress_with_options(&c, &bad).is_err());
}
}