use super::block::{CODEC_STORED, CODEC_ZSTD, CODEC_ZSTD_DICT};
use anyhow::{bail, Result};
use std::borrow::Cow;
mod z {
#[cfg(not(target_arch = "wasm32"))]
use anyhow::Context;
use anyhow::Result;
#[cfg(not(target_arch = "wasm32"))]
pub fn compress(raw: &[u8], dict: Option<&[u8]>, level: i32) -> Result<Vec<u8>> {
match dict {
Some(d) => zstd::bulk::Compressor::with_dictionary(level, d)
.context("zstd compressor with dictionary")?
.compress(raw)
.context("zstd dictionary compress"),
None => zstd::bulk::compress(raw, level).context("zstd compress"),
}
}
#[cfg(not(target_arch = "wasm32"))]
pub fn decompress_into(payload: &[u8], dict: Option<&[u8]>, out: &mut [u8]) -> Result<usize> {
let v = match dict {
Some(d) => zstd::bulk::Decompressor::with_dictionary(d)
.context("zstd decompressor with dictionary")?
.decompress(payload, out.len())
.context("zstd dictionary decompress")?,
None => zstd::bulk::decompress(payload, out.len()).context("zstd decompress")?,
};
if v.len() > out.len() {
anyhow::bail!("decoded {} exceeds declared {}", v.len(), out.len());
}
out[..v.len()].copy_from_slice(&v);
Ok(v.len())
}
#[cfg(target_arch = "wasm32")]
pub fn compress(raw: &[u8], dict: Option<&[u8]>, level: i32) -> Result<Vec<u8>> {
use structured_zstd::encoding::{CompressionLevel, FrameCompressor};
match dict {
Some(d) => {
let mut c: FrameCompressor = FrameCompressor::new(CompressionLevel::Level(level));
c.set_dictionary_from_bytes(d)
.map_err(|e| anyhow::anyhow!("zstd compressor with dictionary: {e:?}"))?;
Ok(c.compress_independent_frame(raw))
}
None => Ok(structured_zstd::encoding::compress_slice_to_vec(
raw,
CompressionLevel::Level(level),
)),
}
}
#[cfg(target_arch = "wasm32")]
pub fn decompress_into(payload: &[u8], dict: Option<&[u8]>, out: &mut [u8]) -> Result<usize> {
let mut d = structured_zstd::decoding::FrameDecoder::new();
if let Some(dict) = dict {
d.add_dict_from_bytes(dict)
.map_err(|e| anyhow::anyhow!("zstd decompressor with dictionary: {e:?}"))?;
}
d.decode_all(payload, out).map_err(|e| anyhow::anyhow!("zstd decompress: {e:?}"))
}
}
pub const LEVEL_DEFAULT: i32 = 19;
pub fn encode<'a>(raw: &'a [u8], dict: Option<&[u8]>, level: i32) -> Result<(u8, Cow<'a, [u8]>)> {
let compressed = z::compress(raw, dict, level)?;
if compressed.len() >= raw.len() {
return Ok((CODEC_STORED, Cow::Borrowed(raw)));
}
let tag = if dict.is_some() { CODEC_ZSTD_DICT } else { CODEC_ZSTD };
Ok((tag, Cow::Owned(compressed)))
}
pub fn decode(codec: u8, payload: &[u8], raw_len: u32, dict: Option<&[u8]>) -> Result<Vec<u8>> {
let n = raw_len as usize;
match codec {
CODEC_STORED => {
if payload.len() != n {
bail!("stored block payload {} != raw {}", payload.len(), n);
}
Ok(payload.to_vec())
}
CODEC_ZSTD | CODEC_ZSTD_DICT => {
let d =
if codec == CODEC_ZSTD_DICT {
Some(dict.ok_or_else(|| {
anyhow::anyhow!("dictionary block but no dictionary loaded")
})?)
} else {
None
};
let mut out = vec![0u8; n];
let got = z::decompress_into(payload, d, &mut out)?;
if got != n {
bail!("decoded length {got} != declared raw {n}");
}
Ok(out)
}
other => bail!("unknown block codec {other}"),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn compressible_input_uses_zstd_and_roundtrips() {
let raw = "the shared system prompt, repeated. ".repeat(64).into_bytes();
let (codec, payload) = encode(&raw, None, LEVEL_DEFAULT).unwrap();
assert_eq!(codec, CODEC_ZSTD);
assert!(payload.len() < raw.len());
assert_eq!(decode(codec, &payload, raw.len() as u32, None).unwrap(), raw);
}
#[test]
fn incompressible_input_falls_back_to_stored() {
let mut raw: Vec<u8> = Vec::with_capacity(4096);
let mut seed = [0u8; 32];
while raw.len() < 4096 {
seed = blake3::hash(&seed).into();
raw.extend_from_slice(&seed);
}
let (codec, payload) = encode(&raw, None, LEVEL_DEFAULT).unwrap();
assert_eq!(codec, CODEC_STORED, "must fall back rather than store an expanded payload");
assert_eq!(payload.len(), raw.len(), "stored <= raw is structural");
assert_eq!(decode(codec, &payload, raw.len() as u32, None).unwrap(), raw);
}
#[test]
fn empty_piece_roundtrips() {
let raw: Vec<u8> = Vec::new();
let (codec, payload) = encode(&raw, None, LEVEL_DEFAULT).unwrap();
assert_eq!(codec, CODEC_STORED);
assert_eq!(decode(codec, &payload, 0, None).unwrap(), raw);
}
#[test]
fn dictionary_roundtrips() {
let dict = "gen_ai.request.model gen_ai.usage tool_use assistant ".repeat(16).into_bytes();
let raw = b"{\"gen_ai.request.model\":\"claude\",\"tool_use\":true}".to_vec();
let (codec, payload) = encode(&raw, Some(&dict), LEVEL_DEFAULT).unwrap();
assert_eq!(decode(codec, &payload, raw.len() as u32, Some(&dict)).unwrap(), raw);
}
#[test]
fn wrong_declared_length_refuses() {
let raw = b"abcdefghij".to_vec();
let (codec, payload) = encode(&raw, None, LEVEL_DEFAULT).unwrap();
assert!(decode(codec, &payload, 9, None).is_err(), "length mismatch must fail loud");
}
}