use crate::bit_writer::BitWriter;
use crate::entropy::{
Histogram, OwnedEntropyCode, Token, build_huffman_codes, cluster_histograms, uint_encode,
write_entropy_code,
};
pub(crate) const LZ77_MIN_SYMBOL: u32 = 64;
pub(crate) const LZ77_MIN_LENGTH: u32 = 3;
const LZ77_DIST_VALUE: u32 = 1;
#[inline]
fn lz77_length_encode(length_value: u32) -> (u32, u32, u32) {
if length_value < 16 {
(length_value, 0, 0)
} else {
let n = 31 - length_value.leading_zeros();
let token = 16 + n - 4;
let nbits = n;
let bits = length_value - (1 << n);
(token, nbits, bits)
}
}
#[derive(Clone, Copy)]
pub(crate) enum AcLz {
Lit { context: u32, value: u32 },
Copy { context: u32, length_value: u32 },
}
pub(crate) fn lz77_compress_ac(tokens: &[Token]) -> Vec<AcLz> {
let mut out: Vec<AcLz> = Vec::with_capacity(tokens.len());
let mut i = 0;
while i < tokens.len() {
let t = tokens[i];
out.push(AcLz::Lit {
context: t.context,
value: t.value,
});
let mut j = i + 1;
while j < tokens.len() && tokens[j].context == t.context && tokens[j].value == t.value {
j += 1;
}
let run_extra = (j - i - 1) as u32; if run_extra >= LZ77_MIN_LENGTH {
out.push(AcLz::Copy {
context: t.context,
length_value: run_extra - LZ77_MIN_LENGTH,
});
i = j;
} else {
i += 1;
}
}
out
}
pub(crate) fn build_lz_code_no_cluster(
streams: &[Vec<AcLz>],
num_contexts: usize,
) -> OwnedEntropyCode {
let distance_context = (num_contexts - 1) as u32;
let mut histograms = vec![Histogram::new(); num_contexts];
for stream in streams {
for t in stream {
match *t {
AcLz::Lit { context, value } => {
let (sym, _, _) = uint_encode(value);
histograms[context as usize].add(sym);
}
AcLz::Copy {
context,
length_value,
} => {
let (len_tok, _, _) = lz77_length_encode(length_value);
histograms[context as usize].add(LZ77_MIN_SYMBOL + len_tok);
histograms[distance_context as usize].add(LZ77_DIST_VALUE);
}
}
}
}
let context_map: Vec<u8> = (0..num_contexts as u8).collect();
let mut prefix_codes = build_huffman_codes(&histograms);
for pc in &mut prefix_codes {
let mut nonzero = 0usize;
let mut idx = 0usize;
for (i, &d) in pc.depths.iter().enumerate() {
if d != 0 {
nonzero += 1;
idx = i;
if nonzero > 1 {
break;
}
}
}
if nonzero == 1 {
if idx == 0 {
pc.depths[0] = 0;
pc.bits[0] = 0;
} else {
pc.depths[0] = 1;
pc.bits[0] = 0;
pc.depths[idx] = 1;
pc.bits[idx] = 1;
}
}
pc.update_single_symbol();
}
OwnedEntropyCode {
context_map,
prefix_codes,
orig_context_map: None,
orig_num_contexts: num_contexts,
}
}
pub(crate) fn build_ac_lz_code(streams: &[Vec<AcLz>], num_contexts: usize) -> OwnedEntropyCode {
let distance_context = (num_contexts - 1) as u32;
let mut histograms = vec![Histogram::new(); num_contexts];
for stream in streams {
for t in stream {
match *t {
AcLz::Lit { context, value } => {
let (sym, _, _) = uint_encode(value);
histograms[context as usize].add(sym);
}
AcLz::Copy {
context,
length_value,
} => {
let (len_tok, _, _) = lz77_length_encode(length_value);
histograms[context as usize].add(LZ77_MIN_SYMBOL + len_tok);
histograms[distance_context as usize].add(LZ77_DIST_VALUE);
}
}
}
}
let mut context_map: Vec<u8> = Vec::new();
cluster_histograms(&mut histograms, &mut context_map);
let prefix_codes = build_huffman_codes(&histograms);
OwnedEntropyCode {
context_map,
prefix_codes,
orig_context_map: None,
orig_num_contexts: num_contexts,
}
}
pub(crate) fn estimate_ac_lz_bits(
streams: &[Vec<AcLz>],
code: &OwnedEntropyCode,
num_contexts: usize,
) -> u64 {
let distance_context = num_contexts - 1;
let mut bits: u64 = 0;
for stream in streams {
for t in stream {
match *t {
AcLz::Lit { context, value } => {
let (sym, nbits, _) = uint_encode(value);
let cl = code.context_map[context as usize] as usize;
let pc = &code.prefix_codes[cl];
let d = if pc.single_symbol {
0
} else {
pc.depths[sym as usize] as u64
};
bits += d + nbits as u64;
}
AcLz::Copy {
context,
length_value,
} => {
let (len_tok, len_nbits, _) = lz77_length_encode(length_value);
let sym = LZ77_MIN_SYMBOL + len_tok;
let cl = code.context_map[context as usize] as usize;
let pc = &code.prefix_codes[cl];
let d = if pc.single_symbol {
0
} else {
pc.depths[sym as usize] as u64
};
bits += d + len_nbits as u64;
let (dsym, dnbits, _) = uint_encode(LZ77_DIST_VALUE);
let dcl = code.context_map[distance_context] as usize;
let dpc = &code.prefix_codes[dcl];
let dd = if dpc.single_symbol {
0
} else {
dpc.depths[dsym as usize] as u64
};
bits += dd + dnbits as u64;
}
}
}
}
bits
}
pub(crate) fn estimate_ac_plain_bits(tokens: &[Token], code: &OwnedEntropyCode) -> u64 {
let mut bits: u64 = 0;
for t in tokens {
let (sym, nbits, _) = uint_encode(t.value);
let cl = code.context_map[t.context as usize] as usize;
bits += code.prefix_codes[cl].depths[sym as usize] as u64 + nbits as u64;
}
bits
}
#[inline]
pub(crate) fn write_ac_lz(
t: AcLz,
code: &OwnedEntropyCode,
num_contexts: usize,
w: &mut BitWriter,
) {
let distance_context = num_contexts - 1;
match t {
AcLz::Lit { context, value } => {
let (sym, nbits, bits) = uint_encode(value);
let cluster = code.context_map[context as usize] as usize;
let pc = &code.prefix_codes[cluster];
if pc.single_symbol {
w.write(nbits as usize, bits as u64);
} else {
let d = pc.depths[sym as usize] as usize;
let data = (pc.bits[sym as usize] as u64) | ((bits as u64) << d);
w.write(d + nbits as usize, data);
}
}
AcLz::Copy {
context,
length_value,
} => {
let (len_tok, len_nbits, len_bits) = lz77_length_encode(length_value);
let sym = LZ77_MIN_SYMBOL + len_tok;
let pcluster = code.context_map[context as usize] as usize;
let pc = &code.prefix_codes[pcluster];
if pc.single_symbol {
w.write(len_nbits as usize, len_bits as u64);
} else {
let d = pc.depths[sym as usize] as usize;
debug_assert!(d > 0, "LZ77 AC length symbol {} unrepresented", sym);
let data = (pc.bits[sym as usize] as u64) | ((len_bits as u64) << d);
w.write(d + len_nbits as usize, data);
}
let (dsym, dnbits, dbits) = uint_encode(LZ77_DIST_VALUE);
let dcluster = code.context_map[distance_context] as usize;
let dpc = &code.prefix_codes[dcluster];
if dpc.single_symbol {
w.write(dnbits as usize, dbits as u64);
} else {
let dd = dpc.depths[dsym as usize] as usize;
let ddata = (dpc.bits[dsym as usize] as u64) | ((dbits as u64) << dd);
w.write(dd + dnbits as usize, ddata);
}
}
}
}
pub(crate) fn write_ac_lz_header_and_code(code: &OwnedEntropyCode, w: &mut BitWriter) {
w.write(1, 1); w.write(2, 0b11);
w.write(15, (LZ77_MIN_SYMBOL - 8) as u64);
w.write(2, 0b00);
w.write(4, 4);
w.write(3, 0);
w.write(3, 0);
write_entropy_code(&code.as_ref(), w);
}