use crate::page_buffer::{zeroize_bytes, ZeroizingBytes};
use std::cell::RefCell;
use zstd_complete::encoding::{CompressionLevel, EncoderWorkspace, StaticEncoderWorkspace};
const MAX_RETAINED: usize = 8 * 1024 * 1024;
struct Buffers {
scratch: ZeroizingBytes,
output: ZeroizingBytes,
}
thread_local! { static POOL: RefCell<Option<Buffers>> = const { RefCell::new(None) }; }
struct Lease(Option<Buffers>);
impl Lease {
fn acquire(required: usize, output_size: usize) -> Self {
let cached = POOL
.try_with(|slot| slot.try_borrow_mut().ok().and_then(|mut slot| slot.take()))
.ok()
.flatten();
let buffers = cached
.filter(|b| b.scratch.len() == required && b.output.len() == output_size)
.unwrap_or_else(|| Buffers {
scratch: ZeroizingBytes::new(vec![0; required]),
output: ZeroizingBytes::new(vec![0; output_size]),
});
Self(Some(buffers))
}
}
impl Drop for Lease {
fn drop(&mut self) {
let mut buffers = self.0.take();
if let Some(buffers) = &mut buffers {
zeroize_bytes(&mut buffers.scratch);
zeroize_bytes(&mut buffers.output);
if buffers
.scratch
.capacity()
.saturating_add(buffers.output.capacity())
> MAX_RETAINED
{
return;
}
}
let _ = POOL.try_with(|slot| {
if let Ok(mut slot) = slot.try_borrow_mut() {
if slot.is_none() {
*slot = buffers.take();
}
}
});
}
}
pub(crate) struct EncoderFrame {
len: usize,
incompressible: bool,
}
impl EncoderFrame {
#[cfg(test)]
pub(crate) fn new(input: &[u8], probe: bool) -> Self {
Self::new_sensitive(input, probe, false)
}
pub(crate) fn new_sensitive(input: &[u8], probe: bool, sensitive: bool) -> Self {
Self {
len: input.len(),
incompressible: !probe
|| !(64 * 1024..=super::MAX_DECOMPRESSED_COMPRESSION_FRAME_BYTES as usize)
.contains(&input.len())
|| (!sensitive && super::looks_incompressible(input)),
}
}
}
#[cfg(test)]
fn with_encoder<R>(
first_frame: EncoderFrame,
compression: Option<crate::Compression>,
default_level: i32,
operation: impl FnOnce(&mut dyn FnMut(&[u8]) -> (u8, Vec<u8>)) -> R,
) -> R {
with_frame_encoder(first_frame, compression, default_level, false, |encode| {
operation(&mut |input| encode(input).into_encoded(input))
})
}
pub(crate) fn with_frame_encoder<R>(
first_frame: EncoderFrame,
compression: Option<crate::Compression>,
default_level: i32,
sensitive: bool,
operation: impl FnOnce(&mut dyn FnMut(&[u8]) -> super::FrameEncoding) -> R,
) -> R {
let mut fallback = |input: &[u8]| {
if sensitive {
super::encode_sensitive_outcome(input, compression, default_level)
} else {
match compression {
Some(compression) => super::encode_with_compression_outcome(input, compression),
None => super::encode_frame_outcome(input, default_level),
}
}
};
let Some(crate::Compression::Zstd { level }) = compression else {
return operation(&mut fallback);
};
if !(64 * 1024..=super::MAX_DECOMPRESSED_COMPRESSION_FRAME_BYTES as usize)
.contains(&first_frame.len)
|| first_frame.incompressible
{
return operation(&mut fallback);
}
if level.get() >= 18 {
return operation(&mut fallback);
}
let level = CompressionLevel::new(level.get()).expect("validated compression level");
let Ok(required) = EncoderWorkspace::required_size(level, first_frame.len) else {
return operation(&mut fallback);
};
let Ok(output_size) = EncoderWorkspace::required_output_size(first_frame.len) else {
return operation(&mut fallback);
};
if required.saturating_add(output_size) > MAX_RETAINED {
return operation(&mut fallback);
}
let mut lease = Lease::acquire(required, output_size);
let Buffers { scratch, output } = lease.0.as_mut().expect("lease owns buffers");
let Ok(mut encoder) = StaticEncoderWorkspace::new(scratch, level, first_frame.len) else {
return operation(&mut fallback);
};
operation(&mut |input| {
if input.len() != first_frame.len || (!sensitive && super::looks_incompressible(input)) {
return fallback(input);
}
match encoder.encode_into(input, output) {
Ok(encoded) if encoded.len() < input.len() => {
super::FrameEncoding::Compressed(ZeroizingBytes::new(encoded.to_vec()))
}
Ok(_) => super::FrameEncoding::Raw,
Err(_) => fallback(input),
}
})
}
pub(crate) fn encode_sensitive(input: &[u8], level: CompressionLevel) -> super::FrameEncoding {
let Some((scratch_size, output_size)) = EncoderWorkspace::required_size(level, input.len())
.ok()
.zip(EncoderWorkspace::required_output_size(input.len()).ok())
else {
return super::FrameEncoding::Raw;
};
if scratch_size
.checked_add(output_size)
.is_none_or(|total| total > 64 * 1024 * 1024)
{
return super::FrameEncoding::Raw;
}
let allocate = |size: usize| -> Option<ZeroizingBytes> {
let mut bytes = Vec::new();
bytes.try_reserve_exact(size).ok()?;
bytes.resize(size, 0);
Some(ZeroizingBytes::new(bytes))
};
let Some(mut scratch) = allocate(scratch_size) else {
return super::FrameEncoding::Raw;
};
let Some(mut output) = allocate(output_size) else {
return super::FrameEncoding::Raw;
};
let result = {
let Ok(mut encoder) = StaticEncoderWorkspace::new(&mut scratch, level, input.len()) else {
return super::FrameEncoding::Raw;
};
match encoder.encode_into(input, &mut output) {
Ok(encoded) if encoded.len() < input.len() => Some(encoded.len()),
_ => None,
}
};
match result {
Some(length) => {
output.truncate(length);
super::FrameEncoding::Compressed(output)
}
None => super::FrameEncoding::Raw,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn pooled_buffers_are_wiped_and_reused_after_success_error_and_unwind() {
POOL.with(|slot| slot.borrow_mut().take());
let input = vec![73; 65536];
let mut pointers = None;
for exit in 0..3 {
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
with_encoder(
EncoderFrame::new(&input, true),
Some(crate::Compression::default()),
3,
|encode| {
let encoded = encode(&input);
assert_eq!(
super::super::decode_compression_frame(
encoded.0,
&encoded.1,
input.len() as u64
)
.unwrap(),
input
);
match exit {
0 => Ok(()),
1 => Err(()),
_ => panic!("injected callback failure"),
}
},
)
}));
match exit {
0 => assert_eq!(result.unwrap(), Ok(())),
1 => assert_eq!(result.unwrap(), Err(())),
_ => assert!(result.is_err()),
}
POOL.with(|slot| {
let slot = slot.borrow();
let buffers = slot.as_ref().unwrap();
assert!(buffers.scratch.iter().all(|b| *b == 0));
assert!(buffers.output.iter().all(|b| *b == 0));
assert!(buffers.scratch.capacity() + buffers.output.capacity() <= MAX_RETAINED);
let current = (buffers.scratch.as_ptr(), buffers.output.as_ptr());
if let Some(previous) = pointers {
assert_eq!(previous, current);
}
pointers = Some(current);
});
}
POOL.with(|slot| slot.borrow_mut().take());
drop(Lease::acquire(MAX_RETAINED + 1, 0));
POOL.with(|slot| {
assert!(
slot.borrow().is_none(),
"over-budget allocations cannot be retained"
)
});
}
#[test]
fn pool_reentry_and_unavailable_borrow_preserve_encoding_and_cleanup() {
POOL.with(|slot| slot.borrow_mut().take());
let input = vec![61; 65536];
with_encoder(
EncoderFrame::new(&input, true),
Some(crate::Compression::default()),
3,
|outer| {
let first = outer(&input);
with_encoder(
EncoderFrame::new(&input, true),
Some(crate::Compression::default()),
3,
|inner| assert_eq!(inner(&input), first),
);
assert_eq!(outer(&input), first);
},
);
POOL.with(|slot| {
let mut held = slot.borrow_mut();
held.take();
with_encoder(
EncoderFrame::new(&input, true),
Some(crate::Compression::default()),
3,
|encode| {
let encoded = encode(&input);
assert_eq!(
super::super::decode_compression_frame(
encoded.0,
&encoded.1,
input.len() as u64
)
.unwrap(),
input
);
},
);
assert!(held.is_none());
});
}
#[test]
fn full_sized_frames_preserve_numeric_bytes_across_reuse() {
let pattern: Vec<u8> = (0..2 * 1024 * 1024).map(|i| (i % 251) as u8).collect();
let repeated = vec![37; pattern.len()];
for level in [1, 3, 6] {
let compression = crate::Compression::Zstd {
level: crate::ZstdLevel::new(level).unwrap(),
};
for _ in 0..2 {
with_encoder(
EncoderFrame::new(&pattern, true),
Some(compression),
3,
|encode| {
for input in [&pattern[..], &repeated[..], &pattern[..]] {
let actual = encode(input);
assert_eq!(
actual,
super::super::encode_with_compression(input, compression)
);
assert_eq!(
super::super::decode_compression_frame(
actual.0,
&actual.1,
input.len() as u64
)
.unwrap(),
input
);
}
},
);
}
}
}
#[test]
fn retained_state_matches_fresh_numeric_encoding_including_size_changes() {
let mut state = 17u64;
let mixed: Vec<u8> = (0..131073)
.map(|i| {
state ^= state << 13;
state ^= state >> 7;
state ^= state << 17;
if i % 131072 < 65536 {
(i % 19) as u8
} else {
state as u8
}
})
.collect();
let repetitive = vec![5; mixed.len()];
let short = &mixed[..65536];
for level in 1..=22 {
let compression = crate::Compression::Zstd {
level: crate::ZstdLevel::new(level).unwrap(),
};
with_encoder(
EncoderFrame::new(&mixed, true),
Some(compression),
1,
|encode| {
for (frame, input) in [&mixed[..], &repetitive[..], short, &mixed[..], &[]]
.into_iter()
.enumerate()
{
let result = encode(input);
let decoded = super::super::decode_compression_frame(
result.0,
&result.1,
input.len() as u64,
)
.unwrap();
assert!(
decoded == input,
"round trip differs: level {level}, frame {frame}"
);
let fresh = super::super::encode_with_compression(input, compression);
assert!(result == fresh, "encoded bytes differ: level {level}, frame {frame}, reused {} bytes, fresh {} bytes", result.1.len(), fresh.1.len());
}
},
);
}
}
}