#[cfg(feature = "zstd")]
mod zstd_backend;
use crate::coding::{Decode, Encode};
use crate::io::{Read, ReadBytesExt, Write, WriteBytesExt};
#[cfg(zstd_any)]
use alloc::sync::Arc;
#[cfg(feature = "zstd")]
use once_cell::race::OnceBox;
#[cfg(zstd_any)]
pub trait CompressionProvider {
fn compress(data: &[u8], level: i32, two_pass_seed: bool) -> crate::Result<Vec<u8>>;
fn compress_with_layout(
data: &[u8],
level: i32,
two_pass_seed: bool,
) -> crate::Result<(Vec<u8>, Vec<u32>)>;
fn decompress(data: &[u8], capacity: usize) -> crate::Result<Vec<u8>>;
fn compress_with_dict(
data: &[u8],
level: i32,
dict: &ZstdDictionary,
two_pass_seed: bool,
) -> crate::Result<Vec<u8>>;
fn decompress_with_dict(
data: &[u8],
dict: &ZstdDictionary,
capacity: usize,
) -> crate::Result<Vec<u8>>;
}
#[cfg(feature = "zstd")]
pub type ZstdBackend = zstd_backend::ZstdProvider;
#[cfg(zstd_any)]
pub struct ZstdDictionary {
id: u64,
raw: Arc<[u8]>,
#[cfg(feature = "zstd")]
prepared: Arc<OnceBox<structured_zstd::decoding::DictionaryHandle>>,
#[cfg(feature = "zstd")]
encoder: Arc<OnceBox<structured_zstd::encoding::EncoderDictionary>>,
}
#[cfg(zstd_any)]
impl Clone for ZstdDictionary {
fn clone(&self) -> Self {
Self {
id: self.id,
raw: Arc::clone(&self.raw),
#[cfg(feature = "zstd")]
prepared: Arc::clone(&self.prepared),
#[cfg(feature = "zstd")]
encoder: Arc::clone(&self.encoder),
}
}
}
#[cfg(zstd_any)]
impl PartialEq for ZstdDictionary {
fn eq(&self, other: &Self) -> bool {
self.id == other.id
}
}
#[cfg(zstd_any)]
impl Eq for ZstdDictionary {}
#[cfg(zstd_any)]
impl ZstdDictionary {
#[must_use]
pub fn new(raw: &[u8]) -> Self {
Self {
id: compute_dict_id(raw),
raw: Arc::from(raw),
#[cfg(feature = "zstd")]
prepared: Arc::new(OnceBox::new()),
#[cfg(feature = "zstd")]
encoder: Arc::new(OnceBox::new()),
}
}
#[cfg(test)]
#[must_use]
pub(crate) fn with_id_for_test(&self, id: u32) -> Self {
Self {
id: u64::from(id),
raw: Arc::clone(&self.raw),
#[cfg(feature = "zstd")]
prepared: Arc::new(OnceBox::new()),
#[cfg(feature = "zstd")]
encoder: Arc::new(OnceBox::new()),
}
}
#[cfg(feature = "zstd")]
pub(crate) fn frame_id(&self) -> u32 {
self.id().max(1)
}
#[cfg(feature = "zstd")]
pub(crate) fn prepared_encoder(
&self,
) -> crate::Result<structured_zstd::encoding::EncoderDictionary> {
use structured_zstd::decoding::{DICTIONARY_MAGIC, Dictionary};
use structured_zstd::encoding::EncoderDictionary;
self.encoder
.get_or_try_init(|| -> crate::Result<Box<EncoderDictionary>> {
let dictionary = if self.raw.starts_with(&DICTIONARY_MAGIC) {
EncoderDictionary::from_bytes(&self.raw)
} else {
Dictionary::from_raw_content(self.frame_id(), self.raw.to_vec())
.map(EncoderDictionary::from_dictionary)
}
.map_err(|e| crate::Error::Io(crate::io::Error::other(e.to_string())))?;
Ok(Box::new(dictionary))
})
.cloned()
}
#[cfg(feature = "zstd")]
pub(crate) fn prepared_handle(
&self,
) -> crate::Result<structured_zstd::decoding::DictionaryHandle> {
use structured_zstd::decoding::{DICTIONARY_MAGIC, Dictionary, DictionaryHandle};
self.prepared
.get_or_try_init(|| -> crate::Result<Box<DictionaryHandle>> {
let handle = if self.raw.starts_with(&DICTIONARY_MAGIC) {
DictionaryHandle::decode_dict(&self.raw)
.map_err(|e| crate::Error::Io(crate::io::Error::other(e.to_string())))?
} else {
let dict = Dictionary::from_raw_content(self.frame_id(), self.raw.to_vec())
.map_err(|e| crate::Error::Io(crate::io::Error::other(e.to_string())))?;
DictionaryHandle::from_dictionary(dict)
};
Ok(Box::new(handle))
})
.cloned()
}
#[must_use]
#[expect(
clippy::cast_possible_truncation,
reason = "intentional: public API returns 32-bit fingerprint"
)]
pub fn id(&self) -> u32 {
self.id as u32
}
#[cfg(feature = "zstd")]
#[must_use]
pub(crate) fn id64(&self) -> u64 {
self.id
}
#[must_use]
pub fn raw(&self) -> &[u8] {
&self.raw
}
}
#[cfg(zstd_any)]
impl core::fmt::Debug for ZstdDictionary {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
f.debug_struct("ZstdDictionary")
.field("id", &format_args!("{:#018x}", self.id))
.field("size", &self.raw.len())
.finish_non_exhaustive() }
}
#[cfg(zstd_any)]
fn compute_dict_id(raw: &[u8]) -> u64 {
xxhash_rust::xxh3::xxh3_64(raw)
}
#[cfg(zstd_any)]
#[derive(Clone, Default)]
pub struct ZstdDictionaries {
by_id: Option<Arc<crate::HashMap<u32, Arc<ZstdDictionary>>>>,
}
#[cfg(zstd_any)]
impl ZstdDictionaries {
#[must_use]
pub fn new() -> Self {
Self { by_id: None }
}
#[must_use]
pub fn get(&self, id: u32) -> Option<&Arc<ZstdDictionary>> {
self.by_id.as_ref()?.get(&id)
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.by_id.as_ref().is_none_or(|m| m.is_empty())
}
#[must_use]
pub fn len(&self) -> usize {
self.by_id.as_ref().map_or(0, |m| m.len())
}
pub fn ids(&self) -> impl Iterator<Item = u32> + '_ {
self.by_id.iter().flat_map(|m| m.keys().copied())
}
pub fn iter(&self) -> impl Iterator<Item = &Arc<ZstdDictionary>> + '_ {
self.by_id.iter().flat_map(|m| m.values())
}
#[must_use]
pub fn with(&self, dict: Arc<ZstdDictionary>) -> Self {
let id = dict.id();
if self.get(id).is_some() {
return self.clone();
}
let mut next = self
.by_id
.as_ref()
.map_or_else(crate::HashMap::default, |m| (**m).clone());
next.insert(id, dict);
Self {
by_id: Some(Arc::new(next)),
}
}
pub fn for_compression(
&self,
compression: CompressionType,
) -> crate::Result<Option<Arc<ZstdDictionary>>> {
match compression {
CompressionType::ZstdDict { dict_id, .. } => match self.get(dict_id) {
Some(dict) => Ok(Some(dict.clone())),
None => Err(crate::Error::ZstdDictMismatch {
expected: dict_id,
got: None,
}),
},
_ => Ok(None),
}
}
#[must_use]
pub fn without(&self, id: u32) -> Self {
if self.get(id).is_none() {
return self.clone();
}
let mut next = self
.by_id
.as_ref()
.map_or_else(crate::HashMap::default, |m| (**m).clone());
next.remove(&id);
Self {
by_id: (!next.is_empty()).then(|| Arc::new(next)),
}
}
}
#[cfg(zstd_any)]
impl core::fmt::Debug for ZstdDictionaries {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
f.debug_struct("ZstdDictionaries")
.field("len", &self.len())
.finish_non_exhaustive()
}
}
#[derive(Copy, Clone, Debug, Eq, PartialEq)]
#[non_exhaustive]
pub enum CompressionType {
None,
#[cfg(feature = "lz4")]
Lz4,
#[cfg(zstd_any)]
Zstd(i32),
#[cfg(zstd_any)]
ZstdDict {
level: i32,
dict_id: u32,
},
}
impl CompressionType {
#[must_use]
pub fn dict_id(&self) -> u32 {
#[cfg(zstd_any)]
if let Self::ZstdDict { dict_id, .. } = self {
return *dict_id;
}
0
}
#[cfg(zstd_any)]
fn validate_zstd_level(level: i32) -> crate::Result<()> {
if !(i32::from(i8::MIN)..=22).contains(&level) {
return Err(crate::Error::Io(crate::io::Error::other(format!(
"invalid zstd compression level {level}, expected -128..=22"
))));
}
Ok(())
}
#[cfg(zstd_any)]
pub fn zstd(level: i32) -> crate::Result<Self> {
Self::validate_zstd_level(level)?;
Ok(Self::Zstd(level))
}
#[cfg(zstd_any)]
pub fn zstd_dict(level: i32, dict_id: u32) -> crate::Result<Self> {
Self::validate_zstd_level(level)?;
Ok(Self::ZstdDict { level, dict_id })
}
}
impl core::fmt::Display for CompressionType {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
write!(
f,
"{}",
match self {
Self::None => "none",
#[cfg(feature = "lz4")]
Self::Lz4 => "lz4",
#[cfg(zstd_any)]
Self::Zstd(_) => "zstd",
#[cfg(zstd_any)]
Self::ZstdDict { .. } => "zstd+dict",
}
)
}
}
impl Encode for CompressionType {
fn encode_into<W: Write>(&self, writer: &mut W) -> Result<(), crate::Error> {
match self {
Self::None => {
writer.write_u8(0)?;
}
#[cfg(feature = "lz4")]
Self::Lz4 => {
writer.write_u8(1)?;
}
#[cfg(zstd_any)]
Self::Zstd(level) => {
writer.write_u8(3)?;
debug_assert!(
(i32::from(i8::MIN)..=22).contains(level),
"zstd level {level} outside valid range -128..=22"
);
#[expect(
clippy::cast_possible_truncation,
reason = "level range -128..=22 maps exactly to i8"
)]
writer.write_i8(*level as i8)?;
}
#[cfg(zstd_any)]
Self::ZstdDict { level, dict_id } => {
writer.write_u8(4)?;
debug_assert!(
(i32::from(i8::MIN)..=22).contains(level),
"zstd level {level} outside valid range -128..=22"
);
#[expect(
clippy::cast_possible_truncation,
reason = "level range -128..=22 maps exactly to i8"
)]
writer.write_i8(*level as i8)?;
crate::io::WriteBytesExt::write_u32::<crate::io::LittleEndian>(writer, *dict_id)?;
}
}
Ok(())
}
}
impl Decode for CompressionType {
fn decode_from<R: Read>(reader: &mut R) -> Result<Self, crate::Error> {
let tag = reader.read_u8()?;
match tag {
0 => Ok(Self::None),
#[cfg(feature = "lz4")]
1 => Ok(Self::Lz4),
#[cfg(zstd_any)]
3 => {
let level = i32::from(reader.read_i8()?);
Self::validate_zstd_level(level)?;
Ok(Self::Zstd(level))
}
#[cfg(zstd_any)]
4 => {
let level = i32::from(reader.read_i8()?);
Self::validate_zstd_level(level)?;
let dict_id = crate::io::ReadBytesExt::read_u32::<crate::io::LittleEndian>(reader)?;
Ok(Self::ZstdDict { level, dict_id })
}
tag => Err(crate::Error::InvalidTag(("CompressionType", tag))),
}
}
}
#[cfg(test)]
#[allow(
clippy::unwrap_used,
clippy::indexing_slicing,
clippy::useless_vec,
clippy::expect_used,
reason = "test code"
)]
mod tests;