use crate::common::error::{ErrorCode, ERROR};
use crate::common::mem::{MEM_writeLE16, MEM_writeLE32};
use crate::decompress::zstd_decompress_block::SymbolEncodingType_e;
pub fn ZSTD_literalsCompressionIsDisabled(
literalCompressionMode: crate::compress::zstd_ldm::ZSTD_ParamSwitch_e,
strategy: u32,
targetLength: u32,
) -> bool {
use crate::compress::zstd_compress_sequences::ZSTD_fast;
use crate::compress::zstd_ldm::ZSTD_ParamSwitch_e;
match literalCompressionMode {
ZSTD_ParamSwitch_e::ZSTD_ps_enable => false,
ZSTD_ParamSwitch_e::ZSTD_ps_disable => true,
ZSTD_ParamSwitch_e::ZSTD_ps_auto => {
strategy == ZSTD_fast && targetLength > 0
}
}
}
pub type ZSTD_strategy = u32;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum HUF_repeat {
#[default]
HUF_repeat_none,
HUF_repeat_check,
HUF_repeat_valid,
}
#[inline]
pub fn showHexa(src: &[u8]) -> usize {
src.len()
}
pub fn ZSTD_noCompressLiterals(dst: &mut [u8], src: &[u8]) -> usize {
let srcSize = src.len();
let flSize: usize = 1 + ((srcSize > 31) as usize) + ((srcSize > 4095) as usize);
if srcSize + flSize > dst.len() {
return ERROR(ErrorCode::DstSizeTooSmall);
}
match flSize {
1 => {
dst[0] = SymbolEncodingType_e::set_basic as u8 + ((srcSize as u8) << 3);
}
2 => {
let v = SymbolEncodingType_e::set_basic as u16 + (1u16 << 2) + ((srcSize as u16) << 4);
MEM_writeLE16(&mut dst[..2], v);
}
_ => {
let v = (SymbolEncodingType_e::set_basic as u32)
.wrapping_add(3u32 << 2)
.wrapping_add((srcSize as u32) << 4);
MEM_writeLE32(&mut dst[..4], v);
}
}
dst[flSize..flSize + srcSize].copy_from_slice(src);
srcSize + flSize
}
fn allBytesIdentical(src: &[u8]) -> bool {
debug_assert!(!src.is_empty());
let first = src[0];
src[1..].iter().all(|b| *b == first)
}
pub fn ZSTD_compressRleLiteralsBlock(dst: &mut [u8], src: &[u8]) -> usize {
let srcSize = src.len();
debug_assert!(
!src.is_empty(),
"ZSTD_compressRleLiteralsBlock requires srcSize >= 1"
);
debug_assert!(
allBytesIdentical(src),
"ZSTD_compressRleLiteralsBlock requires all bytes identical"
);
let flSize: usize = 1 + ((srcSize > 31) as usize) + ((srcSize > 4095) as usize);
debug_assert!(
dst.len() >= 4,
"ZSTD_compressRleLiteralsBlock requires dstCapacity >= 4"
);
if dst.len() < 4 {
return ERROR(ErrorCode::DstSizeTooSmall);
}
match flSize {
1 => dst[0] = SymbolEncodingType_e::set_rle as u8 + ((srcSize as u8) << 3),
2 => {
let v = SymbolEncodingType_e::set_rle as u16 + (1u16 << 2) + ((srcSize as u16) << 4);
MEM_writeLE16(&mut dst[..2], v);
}
_ => {
let v = (SymbolEncodingType_e::set_rle as u32)
.wrapping_add(3u32 << 2)
.wrapping_add((srcSize as u32) << 4);
MEM_writeLE32(&mut dst[..4], v);
}
}
dst[flSize] = src[0];
flSize + 1
}
pub fn ZSTD_minLiteralsToCompress(strategy: ZSTD_strategy, huf_repeat: HUF_repeat) -> usize {
debug_assert!(strategy <= 9);
let shift = core::cmp::min(9i32 - strategy as i32, 3) as u32;
if huf_repeat == HUF_repeat::HUF_repeat_valid {
6
} else {
8usize << shift
}
}
pub use crate::decompress::zstd_decompress_block::MIN_LITERALS_FOR_4_STREAMS;
pub const LitHufLog: u32 = 11;
#[inline]
pub fn ZSTD_minGain(srcSize: usize, strategy: ZSTD_strategy) -> usize {
use crate::compress::zstd_compress_sequences::ZSTD_btultra;
let minlog: u32 = if strategy >= ZSTD_btultra {
strategy - 1
} else {
6
};
(srcSize >> minlog) + 2
}
pub fn ZSTD_compressLiterals(
dst: &mut [u8],
src: &[u8],
disableLiteralCompression: i32,
strategy: ZSTD_strategy,
prevHufTable: Option<&mut [crate::compress::huf_compress::HUF_CElt]>,
prevRepeat: Option<&mut HUF_repeat>,
suspectUncompressible: i32,
bmi2: i32,
) -> usize {
use crate::common::error::{ErrorCode, ERROR};
use crate::common::mem::{MEM_writeLE24, MEM_writeLE32};
use crate::compress::huf_compress::{HUF_compress1X_repeat, HUF_compress4X_repeat};
use crate::compress::zstd_compress_sequences::{ZSTD_btultra, ZSTD_lazy};
let srcSize = src.len();
let mut prevRepeat = prevRepeat;
let mut prevHufTable = prevHufTable;
let lhSize = 3 + (srcSize >= 1024) as usize + (srcSize >= 16384) as usize;
let mut singleStream = srcSize < 256;
let mut hType = SymbolEncodingType_e::set_compressed;
if disableLiteralCompression != 0 {
return ZSTD_noCompressLiterals(dst, src);
}
let repeat_valid = prevRepeat
.as_ref()
.map(|r| **r == HUF_repeat::HUF_repeat_valid)
.unwrap_or(false);
let min_lits = ZSTD_minLiteralsToCompress(
strategy,
if repeat_valid {
HUF_repeat::HUF_repeat_valid
} else {
HUF_repeat::HUF_repeat_none
},
);
if srcSize < min_lits {
return ZSTD_noCompressLiterals(dst, src);
}
if dst.len() < lhSize + 1 {
return ERROR(ErrorCode::DstSizeTooSmall);
}
if repeat_valid && lhSize == 3 {
singleStream = true;
}
let flags: i32 = (if bmi2 != 0 {
crate::common::entropy_common::HUF_flags_bmi2
} else {
0
}) | (if strategy < ZSTD_lazy && srcSize <= 1024 {
crate::decompress::huf_decompress::HUF_flags_preferRepeat
} else {
0
}) | (if strategy >= ZSTD_btultra {
crate::decompress::huf_decompress::HUF_flags_optimalDepth
} else {
0
}) | (if suspectUncompressible != 0 {
crate::decompress::huf_decompress::HUF_flags_suspectUncompressible
} else {
0
});
let mut snapshot_huf_ctable = [crate::compress::huf_compress::HUF_CElt::default(); 257];
let snapshot_huf_ctable_len = prevHufTable.as_deref().map(|t| {
let n = t.len().min(snapshot_huf_ctable.len());
snapshot_huf_ctable[..n].copy_from_slice(&t[..n]);
n
});
let mut repeat_scratch = prevRepeat
.as_deref()
.copied()
.unwrap_or(HUF_repeat::HUF_repeat_none);
let initial_repeat = repeat_scratch;
let cLitSize = {
let (_, payload_dst) = dst.split_at_mut(lhSize);
let huf_table_arg = prevHufTable.as_deref_mut();
if singleStream {
HUF_compress1X_repeat(
payload_dst,
src,
crate::decompress::huf_decompress::HUF_SYMBOLVALUE_MAX,
LitHufLog,
huf_table_arg,
Some(&mut repeat_scratch),
flags,
)
} else if srcSize >= MIN_LITERALS_FOR_4_STREAMS {
HUF_compress4X_repeat(
payload_dst,
src,
crate::decompress::huf_decompress::HUF_SYMBOLVALUE_MAX,
LitHufLog,
huf_table_arg,
Some(&mut repeat_scratch),
flags,
)
} else {
0
}
};
if let Some(r) = prevRepeat.as_deref_mut() {
*r = repeat_scratch;
}
if repeat_scratch != HUF_repeat::HUF_repeat_none {
hType = SymbolEncodingType_e::set_repeat;
}
let restore_table = |slot: &mut Option<&mut [crate::compress::huf_compress::HUF_CElt]>| {
if let (Some(snap_len), Some(t)) = (snapshot_huf_ctable_len, slot.as_deref_mut()) {
let n = snap_len.min(t.len());
t[..n].copy_from_slice(&snapshot_huf_ctable[..n]);
}
};
let minGain = ZSTD_minGain(srcSize, strategy);
if cLitSize == 0
|| cLitSize >= srcSize.wrapping_sub(minGain)
|| crate::common::error::ERR_isError(cLitSize)
{
if let Some(r) = prevRepeat.as_deref_mut() {
*r = initial_repeat;
}
restore_table(&mut prevHufTable);
return ZSTD_noCompressLiterals(dst, src);
}
if cLitSize == 1 && (srcSize >= 8 || allBytesIdentical(src)) {
if let Some(r) = prevRepeat.as_deref_mut() {
*r = initial_repeat;
}
restore_table(&mut prevHufTable);
return ZSTD_compressRleLiteralsBlock(dst, src);
}
let lhc: u32 = match lhSize {
3 => {
hType as u32
| ((!singleStream as u32) << 2)
| ((srcSize as u32) << 4)
| ((cLitSize as u32) << 14)
}
4 => hType as u32 | (2 << 2) | ((srcSize as u32) << 4) | ((cLitSize as u32) << 18),
_ => {
hType as u32 | (3 << 2) | ((srcSize as u32) << 4) | ((cLitSize as u32) << 22)
}
};
match lhSize {
3 => MEM_writeLE24(&mut dst[..3], lhc),
4 => MEM_writeLE32(&mut dst[..4], lhc),
_ => {
MEM_writeLE32(&mut dst[..4], lhc);
dst[4] = (cLitSize >> 10) as u8;
}
}
if hType == SymbolEncodingType_e::set_compressed {
if let Some(r) = prevRepeat.as_deref_mut() {
*r = HUF_repeat::HUF_repeat_check;
}
}
lhSize + cLitSize
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn HUF_repeat_discriminants_match_upstream() {
assert_eq!(HUF_repeat::HUF_repeat_none as u32, 0);
assert_eq!(HUF_repeat::HUF_repeat_check as u32, 1);
assert_eq!(HUF_repeat::HUF_repeat_valid as u32, 2);
assert_eq!(HUF_repeat::default(), HUF_repeat::HUF_repeat_none);
}
#[test]
fn minLiteralsToCompress_matches_upstream_formula() {
for strat in 1u32..=9 {
let shift = core::cmp::min(9 - strat as i32, 3) as u32;
let expected: usize = 8usize << shift;
assert_eq!(
ZSTD_minLiteralsToCompress(strat, HUF_repeat::HUF_repeat_none),
expected,
"no-repeat minLiteralsToCompress({strat})",
);
}
for strat in 1u32..=9 {
assert_eq!(
ZSTD_minLiteralsToCompress(strat, HUF_repeat::HUF_repeat_valid),
6,
"repeat-valid minLiteralsToCompress({strat})",
);
}
}
#[test]
fn minGain_matches_upstream_formula_across_strategies() {
let cases: &[(u32, usize)] = &[
(1, 1024), (3, 1024), (7, 1024), (8, 1024), (9, 1024), (1, 65536), (9, 65536), ];
for &(strat, sz) in cases {
let minlog: u32 = if strat >= 8 { strat - 1 } else { 6 };
let expected = (sz >> minlog) + 2;
assert_eq!(
ZSTD_minGain(sz, strat),
expected,
"minGain(size={sz}, strat={strat})",
);
}
}
use crate::decompress::zstd_decompress_block::{
streaming_operation, ZSTD_DCtx, ZSTD_buildDefaultSeqTables, ZSTD_decodeLiteralsBlock,
};
fn roundtrip(src: &[u8]) -> Vec<u8> {
let mut buf = vec![0u8; src.len() + 16];
let written = ZSTD_noCompressLiterals(&mut buf, src);
assert!(!crate::common::error::ERR_isError(written));
let mut dctx = ZSTD_DCtx::new();
ZSTD_buildDefaultSeqTables(&mut dctx);
let mut dst_junk = vec![0u8; src.len().max(16)];
let consumed = ZSTD_decodeLiteralsBlock(
&mut dctx,
&buf[..written],
&mut dst_junk,
streaming_operation::not_streaming,
);
assert!(!crate::common::error::ERR_isError(consumed));
assert_eq!(consumed, written);
assert_eq!(dctx.litSize, src.len());
dctx.litExtraBuffer[..dctx.litSize].to_vec()
}
#[test]
fn literalsCompressionIsDisabled_respects_explicit_mode() {
use crate::compress::zstd_ldm::ZSTD_ParamSwitch_e;
assert!(!ZSTD_literalsCompressionIsDisabled(
ZSTD_ParamSwitch_e::ZSTD_ps_enable,
1,
32
));
assert!(ZSTD_literalsCompressionIsDisabled(
ZSTD_ParamSwitch_e::ZSTD_ps_disable,
5,
0
));
}
#[test]
fn literalsCompressionIsDisabled_auto_fast_plus_targetLength_turns_off() {
use crate::compress::zstd_ldm::ZSTD_ParamSwitch_e;
assert!(ZSTD_literalsCompressionIsDisabled(
ZSTD_ParamSwitch_e::ZSTD_ps_auto,
1,
32
));
assert!(!ZSTD_literalsCompressionIsDisabled(
ZSTD_ParamSwitch_e::ZSTD_ps_auto,
1,
0
));
assert!(!ZSTD_literalsCompressionIsDisabled(
ZSTD_ParamSwitch_e::ZSTD_ps_auto,
3,
32
));
}
#[test]
fn no_compress_short_literals_roundtrip() {
let src = b"HELLO";
let decoded = roundtrip(src);
assert_eq!(decoded, src);
}
#[test]
fn no_compress_medium_literals_roundtrip() {
let src: Vec<u8> = (0..200u16).map(|i| (i as u8).wrapping_add(b'0')).collect();
let decoded = roundtrip(&src);
assert_eq!(decoded, src);
}
#[test]
fn no_compress_large_literals_roundtrip() {
let src: Vec<u8> = (0..8000u32).map(|i| (i as u8).wrapping_mul(31)).collect();
let decoded = roundtrip(&src);
assert_eq!(decoded, src);
}
#[test]
fn no_compress_rejects_too_small_dst() {
let src = [1u8; 10];
let mut dst = [0u8; 5]; let rc = ZSTD_noCompressLiterals(&mut dst, &src);
assert!(crate::common::error::ERR_isError(rc));
}
#[test]
fn rle_literals_roundtrip_short() {
let mut buf = [0u8; 4];
let src = [b'A'; 20];
let written = ZSTD_compressRleLiteralsBlock(&mut buf, &src);
assert_eq!(written, 2);
let mut dctx = ZSTD_DCtx::new();
ZSTD_buildDefaultSeqTables(&mut dctx);
let mut dst_junk = vec![0u8; src.len().max(16)];
let consumed = ZSTD_decodeLiteralsBlock(
&mut dctx,
&buf[..written],
&mut dst_junk,
streaming_operation::not_streaming,
);
assert_eq!(consumed, written);
assert_eq!(dctx.litSize, 20);
assert_eq!(&dctx.litExtraBuffer[..20], &[b'A'; 20]);
}
#[test]
fn literal_special_case_headers_match_upstream() {
let mut dst = vec![0u8; 8200];
let n = ZSTD_noCompressLiterals(&mut dst, &[0xAB; 31]);
assert_eq!(n, 32);
assert_eq!(dst[0], 31 << 3);
let n = ZSTD_noCompressLiterals(&mut dst, &[0xCD; 32]);
assert_eq!(n, 34);
assert_eq!(&dst[..2], &((1u16 << 2) + (32u16 << 4)).to_le_bytes());
let n = ZSTD_noCompressLiterals(&mut dst, &[0xEF; 4096]);
assert_eq!(n, 4099);
assert_eq!(
&dst[..3],
&((3u32 << 2) + (4096u32 << 4)).to_le_bytes()[..3]
);
assert_eq!(dst[3], 0xEF);
let n = ZSTD_compressRleLiteralsBlock(&mut dst, &[0x42; 31]);
assert_eq!(n, 2);
assert_eq!(dst[0], (SymbolEncodingType_e::set_rle as u8) + (31 << 3));
assert_eq!(dst[1], 0x42);
let n = ZSTD_compressRleLiteralsBlock(&mut dst, &[0x42; 4096]);
assert_eq!(n, 4);
assert_eq!(
&dst[..3],
&((SymbolEncodingType_e::set_rle as u32) + (3u32 << 2) + (4096u32 << 4)).to_le_bytes()
[..3]
);
assert_eq!(dst[3], 0x42);
}
#[test]
fn compress_literals_fallback_restores_repeat_and_table() {
let src: Vec<u8> = (0..2048u32).map(|i| i as u8).collect();
let mut dst = vec![0u8; src.len() + 16];
let mut table = vec![crate::compress::huf_compress::HUF_CElt::MAX; 257];
let original = table.clone();
let mut repeat = HUF_repeat::HUF_repeat_check;
let written = ZSTD_compressLiterals(
&mut dst,
&src,
0,
crate::compress::zstd_compress_sequences::ZSTD_lazy2,
Some(&mut table),
Some(&mut repeat),
1,
0,
);
assert!(!crate::common::error::ERR_isError(written));
assert_eq!(dst[0] & 3, SymbolEncodingType_e::set_basic as u8);
assert_eq!(repeat, HUF_repeat::HUF_repeat_check);
assert_eq!(table, original);
}
#[cfg_attr(debug_assertions, should_panic(expected = "dstCapacity >= 4"))]
#[test]
fn rle_literals_requires_upstream_minimum_capacity() {
let src = [b'A'; 20];
let mut dst = [0u8; 2];
let written = ZSTD_compressRleLiteralsBlock(&mut dst, &src);
if !cfg!(debug_assertions) {
assert!(crate::common::error::ERR_isError(written));
}
}
#[test]
fn all_bytes_identical_helper() {
assert!(allBytesIdentical(b"A"));
assert!(allBytesIdentical(b"AAAA"));
assert!(!allBytesIdentical(b"AAAB"));
assert!(!allBytesIdentical(b"AB"));
}
#[test]
fn zstd_compress_literals_huf_roundtrip() {
let src: Vec<u8> = b"the quick brown fox jumps over the lazy dog. "
.iter()
.cycle()
.take(500)
.copied()
.collect();
let mut dst = vec![0u8; src.len() * 2 + 16];
let written = ZSTD_compressLiterals(&mut dst, &src, 0, 5, None, None, 0, 0);
assert!(
!crate::common::error::ERR_isError(written),
"compress err: {}",
crate::common::error::ERR_getErrorName(written)
);
assert!(
written > 0 && written < src.len(),
"no savings: wrote {written} bytes for {}-byte src",
src.len()
);
let mut dctx = ZSTD_DCtx::new();
ZSTD_buildDefaultSeqTables(&mut dctx);
let mut dst_scratch = vec![0u8; src.len()];
let consumed = ZSTD_decodeLiteralsBlock(
&mut dctx,
&dst[..written],
&mut dst_scratch,
streaming_operation::not_streaming,
);
assert!(
!crate::common::error::ERR_isError(consumed),
"decode err: {}",
crate::common::error::ERR_getErrorName(consumed)
);
assert_eq!(consumed, written);
assert_eq!(dctx.litSize, src.len());
assert_eq!(&dctx.litExtraBuffer[..dctx.litSize], &src[..]);
}
#[test]
fn zstd_compress_literals_disabled_falls_back_to_raw() {
let src = b"some content that would compress";
let mut dst = vec![0u8; src.len() + 16];
let written = ZSTD_compressLiterals(&mut dst, src, 1, 5, None, None, 0, 0);
assert!(!crate::common::error::ERR_isError(written));
assert_eq!(dst[0] & 3, 0, "first byte should flag set_basic");
}
#[test]
fn min_literals_heuristic_matches_upstream_formula() {
assert_eq!(
ZSTD_minLiteralsToCompress(9, HUF_repeat::HUF_repeat_none),
8
);
assert_eq!(
ZSTD_minLiteralsToCompress(6, HUF_repeat::HUF_repeat_none),
64
);
assert_eq!(
ZSTD_minLiteralsToCompress(5, HUF_repeat::HUF_repeat_none),
64
);
assert_eq!(
ZSTD_minLiteralsToCompress(1, HUF_repeat::HUF_repeat_valid),
6
);
}
}